From ee3e000cd166b931faaaf0faa0b0d23feba13b37 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Fri, 26 Jan 2024 18:18:07 +0100 Subject: [PATCH] Add polygon merging --- Docs/SettingUpPSQL.md | 2 +- package.json | 2 +- scripts/osm2pgsql/generateBuildDbScript.ts | 274 +++++++++++------- src/Logic/FeatureSource/FeatureSource.ts | 6 + .../Sources/FeatureSourceMerger.ts | 32 +- src/Logic/FeatureSource/Sources/MvtSource.ts | 191 +++++++----- .../DynamicGeoJsonTileSource.ts | 4 +- .../DynamicMvtTileSource.ts | 61 +++- .../TiledFeatureSource/DynamicTileSource.ts | 99 ++++++- src/UI/Map/ShowDataLayer.ts | 1 - src/UI/Test.svelte | 93 +----- 11 files changed, 460 insertions(+), 305 deletions(-) diff --git a/Docs/SettingUpPSQL.md b/Docs/SettingUpPSQL.md index dfa9989c6..e3a4ae13b 100644 --- a/Docs/SettingUpPSQL.md +++ b/Docs/SettingUpPSQL.md @@ -29,7 +29,7 @@ Install osm2pgsql (hint: compile from source is painless) To seed the database: ```` -osm2pgsql -O flex -E 4326 -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi .osm.pbf +osm2pgsql -O flex -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi .osm.pbf ```` Storing properties to table '"public"."osm2pgsql_properties" takes about 25 minutes with planet.osm diff --git a/package.json b/package.json index 5f961226b..921eaee85 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "oauth_secret": "NBWGhWDrD3QDB35xtVuxv4aExnmIt4FA_WgeLtwxasg", "url": "https://www.openstreetmap.org" }, - "mvt_layer_server": "http://127.0.0.1:7800/public.{layer}/{z}/{x}/{y}.pbf", + "mvt_layer_server": "http://127.0.0.1:7800/public.{type}_{layer}/{z}/{x}/{y}.pbf", "disabled:oauth_credentials": { "##": "DEV", "#": "This client-id is registered by 'MapComplete' on https://master.apis.dev.openstreetmap.org/", diff --git a/scripts/osm2pgsql/generateBuildDbScript.ts b/scripts/osm2pgsql/generateBuildDbScript.ts index 0fce50b06..4559f4683 100644 --- a/scripts/osm2pgsql/generateBuildDbScript.ts +++ b/scripts/osm2pgsql/generateBuildDbScript.ts @@ -5,91 +5,57 @@ import Script from "../Script" import fs from "fs" import { Or } from "../../src/Logic/Tags/Or" import { RegexTag } from "../../src/Logic/Tags/RegexTag" -import { Utils } from "../../src/Utils" import { ValidateThemeEnsemble } from "../../src/Models/ThemeConfig/Conversion/Validation" import { AllKnownLayouts } from "../../src/Customizations/AllKnownLayouts" class LuaSnippets { - /** - * The main piece of code that calls `process_poi` - */ - static tail = [ - "function osm2pgsql.process_node(object)", - " process_poi(object, object:as_point())", + + public static helpers = [ + "function countTbl(tbl)\n" + + " local c = 0\n" + + " for n in pairs(tbl) do \n" + + " c = c + 1 \n" + + " end\n" + + " return c\n" + "end", - "", - "function osm2pgsql.process_way(object)", - " if object.is_closed then", - " process_poi(object, object:as_polygon():centroid())", - " end", - "end", - ""].join("\n") + ].join("\n") - public static combine(calls: string[]): string { - return [ - `function process_poi(object, geom)`, - ...calls.map(c => " " + c + "(object, geom)"), - `end`, - ].join("\n") - } -} - -class GenerateLayerLua { - private readonly _id: string - private readonly _tags: TagsFilter - private readonly _foundInThemes: string[] - - constructor(id: string, tags: TagsFilter, foundInThemes: string[] = []) { - this._tags = tags - this._id = id - this._foundInThemes = foundInThemes - } - - public functionName() { - if (!this._tags) { - return undefined + public static toLuaFilter(tag: TagsFilter, useParens: boolean = false): string { + if (tag instanceof Tag) { + return `object.tags["${tag.key}"] == "${tag.value}"` } - return `process_poi_${this._id}` - } - - public generateFunction(): string { - if (!this._tags) { - return undefined + if (tag instanceof And) { + const expr = tag.and.map(t => this.toLuaFilter(t, true)).join(" and ") + if (useParens) { + return "(" + expr + ")" + } + return expr } - return [ - `local pois_${this._id} = osm2pgsql.define_table({`, - this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "", - ` name = '${this._id}',`, - " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", - " columns = {", - " { column = 'tags', type = 'jsonb' },", - " { column = 'geom', type = 'point', projection = 4326, not_null = true },", - " }" + - "})", - "", - "", - `function ${this.functionName()}(object, geom)`, - " local matches_filter = " + this.toLuaFilter(this._tags), - " if( not matches_filter) then", - " return", - " end", - " local a = {", - " geom = geom,", - " tags = object.tags", - " }", - " ", - ` pois_${this._id}:insert(a)`, - "end", - "", - ].join("\n") + if (tag instanceof Or) { + const expr = tag.or.map(t => this.toLuaFilter(t, true)).join(" or ") + if (useParens) { + return "(" + expr + ")" + } + return expr + } + if (tag instanceof RegexTag) { + let expr = LuaSnippets.regexTagToLua(tag) + if (useParens) { + expr = "(" + expr + ")" + } + return expr + } + let msg = "Could not handle" + tag.asHumanString(false, false, {}) + console.error(msg) + throw msg } - private regexTagToLua(tag: RegexTag) { + private static regexTagToLua(tag: RegexTag) { if (typeof tag.value === "string" && tag.invert) { return `object.tags["${tag.key}"] ~= "${tag.value}"` } - const v = ( tag.value).source.replace(/\\\//g, "/") + const v = (tag.value).source.replace(/\\\//g, "/") if ("" + tag.value === "/.+/is" && !tag.invert) { return `object.tags["${tag.key}"] ~= nil` @@ -115,35 +81,58 @@ class GenerateLayerLua { return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${v}"))` } - private toLuaFilter(tag: TagsFilter, useParens: boolean = false): string { - if (tag instanceof Tag) { - return `object.tags["${tag.key}"] == "${tag.value}"` - } - if (tag instanceof And) { - const expr = tag.and.map(t => this.toLuaFilter(t, true)).join(" and ") - if (useParens) { - return "(" + expr + ")" - } - return expr - } - if (tag instanceof Or) { - const expr = tag.or.map(t => this.toLuaFilter(t, true)).join(" or ") - if (useParens) { - return "(" + expr + ")" - } - return expr - } - if (tag instanceof RegexTag) { - let expr = this.regexTagToLua(tag) - if (useParens) { - expr = "(" + expr + ")" - } - return expr - } - let msg = "Could not handle" + tag.asHumanString(false, false, {}) - console.error(msg) - throw msg +} + +class GenerateLayerLua { + private readonly _id: string + private readonly _tags: TagsFilter + private readonly _foundInThemes: string[] + + constructor(id: string, tags: TagsFilter, foundInThemes: string[] = []) { + this._tags = tags + this._id = id + this._foundInThemes = foundInThemes } + + public generateTables(): string { + if (!this._tags) { + return undefined + } + return [ + `db_tables.pois_${this._id} = osm2pgsql.define_table({`, + this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "", + ` name = 'pois_${this._id}',`, + " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", + " columns = {", + " { column = 'tags', type = 'jsonb' },", + " { column = 'geom', type = 'point', projection = 4326, not_null = true },", + " }", + "})", + "", + `db_tables.lines_${this._id} = osm2pgsql.define_table({`, + this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "", + ` name = 'lines_${this._id}',`, + " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", + " columns = {", + " { column = 'tags', type = 'jsonb' },", + " { column = 'geom', type = 'linestring', projection = 4326, not_null = true },", + " }", + "})", + + `db_tables.polygons_${this._id} = osm2pgsql.define_table({`, + this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "", + ` name = 'polygons_${this._id}',`, + " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", + " columns = {", + " { column = 'tags', type = 'jsonb' },", + " { column = 'geom', type = 'polygon', projection = 4326, not_null = true },", + " }", + "})", + "", + ].join("\n") + } + + } class GenerateBuildDbScript extends Script { @@ -163,14 +152,93 @@ class GenerateBuildDbScript extends Script { }) const script = [ - ...generators.map(g => g.generateFunction()), - LuaSnippets.combine(Utils.NoNull(generators.map(g => g.functionName()))), - LuaSnippets.tail, + "local db_tables = {}", + LuaSnippets.helpers, + ...generators.map(g => g.generateTables()), + this.generateProcessPoi(allNeededLayers), + this.generateProcessWay(allNeededLayers), ].join("\n\n\n") const path = "build_db.lua" fs.writeFileSync(path, script, "utf-8") console.log("Written", path) - console.log(allNeededLayers.size+" layers will be created. Make sure to set 'max_connections' to at least "+(10 + allNeededLayers.size) ) + console.log(allNeededLayers.size + " layers will be created with 3 tables each. Make sure to set 'max_connections' to at least " + (10 + 3 * allNeededLayers.size)) + } + + private earlyAbort() { + return [" if countTbl(object.tags) == 0 then", + " return", + " end", + ""].join("\n") + } + + private generateProcessPoi(allNeededLayers: Map) { + const body: string[] = [] + allNeededLayers.forEach(({ tags }, layerId) => { + body.push( + this.insertInto(tags, layerId, "pois_").join("\n"), + ) + }) + + return [ + "function osm2pgsql.process_node(object)", + this.earlyAbort(), + " local geom = object:as_point()", + " local matches_filter = false", + body.join("\n"), + "end", + ].join("\n") + } + + /** + * If matches_filter + * @param tags + * @param layerId + * @param tableprefix + * @private + */ + private insertInto(tags: TagsFilter, layerId: string, tableprefix: "pois_" | "lines_" | "polygons_") { + const filter = LuaSnippets.toLuaFilter(tags) + return [ + " matches_filter = " + filter, + " if matches_filter then", + " db_tables." + tableprefix + layerId + ":insert({", + " geom = geom,", + " tags = object.tags", + " })", + " end", + ] + } + + private generateProcessWay(allNeededLayers: Map) { + const bodyLines: string[] = [] + allNeededLayers.forEach(({ tags }, layerId) => { + bodyLines.push(this.insertInto(tags, layerId, "lines_").join("\n")) + }) + + const bodyPolygons: string[] = [] + allNeededLayers.forEach(({ tags }, layerId) => { + bodyPolygons.push(this.insertInto(tags, layerId, "polygons_").join("\n")) + }) + + return [ + "function process_polygon(object, geom)", + " local matches_filter", + ...bodyPolygons, + "end", + "function process_linestring(object, geom)", + " local matches_filter", + ...bodyLines, + "end", + "", + "function osm2pgsql.process_way(object)", + this.earlyAbort(), + " if object.is_closed then", + " process_polygon(object, object:as_polygon())", + " else", + " process_linestring(object, object:as_linestring())", + " end", + "end", + ].join("\n") } } diff --git a/src/Logic/FeatureSource/FeatureSource.ts b/src/Logic/FeatureSource/FeatureSource.ts index 3b9798dbb..bd5ecbfc2 100644 --- a/src/Logic/FeatureSource/FeatureSource.ts +++ b/src/Logic/FeatureSource/FeatureSource.ts @@ -16,6 +16,12 @@ export interface FeatureSourceForLayer extends Feat readonly layer: FilteredLayer } +export interface FeatureSourceForTile extends FeatureSource { + readonly x: number + readonly y: number + readonly z: number + +} /** * A feature source which is aware of the indexes it contains */ diff --git a/src/Logic/FeatureSource/Sources/FeatureSourceMerger.ts b/src/Logic/FeatureSource/Sources/FeatureSourceMerger.ts index 951b688c5..7ea4d1635 100644 --- a/src/Logic/FeatureSource/Sources/FeatureSourceMerger.ts +++ b/src/Logic/FeatureSource/Sources/FeatureSourceMerger.ts @@ -2,43 +2,49 @@ import { Store, UIEventSource } from "../../UIEventSource" import { FeatureSource, IndexedFeatureSource } from "../FeatureSource" import { Feature } from "geojson" import { Utils } from "../../../Utils" +import DynamicTileSource from "../TiledFeatureSource/DynamicTileSource" /** - * + * The featureSourceMerger receives complete geometries from various sources. + * If multiple sources contain the same object (as determined by 'id'), only one copy of them is retained */ -export default class FeatureSourceMerger implements IndexedFeatureSource { +export default class FeatureSourceMerger implements IndexedFeatureSource { public features: UIEventSource = new UIEventSource([]) public readonly featuresById: Store> - private readonly _featuresById: UIEventSource> - private readonly _sources: FeatureSource[] = [] + protected readonly _featuresById: UIEventSource> + private readonly _sources: Src[] = [] /** * Merges features from different featureSources. * In case that multiple features have the same id, the latest `_version_number` will be used. Otherwise, we will take the last one */ - constructor(...sources: FeatureSource[]) { + constructor(...sources: Src[]) { this._featuresById = new UIEventSource>(new Map()) this.featuresById = this._featuresById const self = this sources = Utils.NoNull(sources) for (let source of sources) { source.features.addCallback(() => { - self.addData(sources.map((s) => s.features.data)) + self.addDataFromSources(sources) }) } - this.addData(sources.map((s) => s.features.data)) + this.addDataFromSources(sources) this._sources = sources } - public addSource(source: FeatureSource) { + public addSource(source: Src) { if (!source) { return } this._sources.push(source) source.features.addCallbackAndRun(() => { - this.addData(this._sources.map((s) => s.features.data)) + this.addDataFromSources(this._sources) }) } + protected addDataFromSources(sources: Src[]){ + this.addData(sources.map(s => s.features.data)) + } + protected addData(sources: Feature[][]) { sources = Utils.NoNull(sources) let somethingChanged = false @@ -56,7 +62,7 @@ export default class FeatureSourceMerger implements IndexedFeatureSource { const id = f.properties.id unseen.delete(id) if (!all.has(id)) { - // This is a new feature + // This is a new, previously unseen feature somethingChanged = true all.set(id, f) continue @@ -81,10 +87,8 @@ export default class FeatureSourceMerger implements IndexedFeatureSource { return } - const newList = [] - all.forEach((value) => { - newList.push(value) - }) + const newList = Array.from(all.values()) + this.features.setData(newList) this._featuresById.setData(all) } diff --git a/src/Logic/FeatureSource/Sources/MvtSource.ts b/src/Logic/FeatureSource/Sources/MvtSource.ts index 8c93eee9c..3aeb34400 100644 --- a/src/Logic/FeatureSource/Sources/MvtSource.ts +++ b/src/Logic/FeatureSource/Sources/MvtSource.ts @@ -1,6 +1,6 @@ import { Feature, Geometry } from "geojson" import { Store, UIEventSource } from "../../UIEventSource" -import { FeatureSource } from "../FeatureSource" +import { FeatureSourceForTile } from "../FeatureSource" import Pbf from "pbf" import * as pbfCompile from "pbf/compile" import * as PbfSchema from "protocol-buffers-schema" @@ -19,8 +19,67 @@ class MvtFeatureBuilder { this._y0 = extent * y } - public toGeoJson(geometry, typeIndex, properties): Feature { - let coords: [number, number] | Coords | Coords[] = this.encodeGeometry(geometry) + private static signedArea(ring: Coords): number { + let sum = 0 + const len = ring.length + // J is basically (i - 1) % len + let j = len - 1 + let p1 + let p2 + for (let i = 0; i < len; i++) { + p1 = ring[i] + p2 = ring[j] + sum += (p2.x - p1.x) * (p1.y + p2.y) + j = i + } + return sum + } + + /** + * + * const rings = [ [ [ 3.208361864089966, 51.186908820014736 ], [ 3.2084155082702637, 51.18689537073311 ], [ 3.208436965942383, 51.186888646090836 ], [ 3.2084155082702637, 51.18686174751187 ], [ 3.2084155082702637, 51.18685502286465 ], [ 3.2083725929260254, 51.18686847215807 ], [ 3.2083404064178467, 51.18687519680333 ], [ 3.208361864089966, 51.186908820014736 ] ] ] + * MvtFeatureBuilder.classifyRings(rings) // => [rings] + */ + private static classifyRings(rings: Coords[]): Coords[][] { + if (rings.length <= 0) { + throw "Now rings in polygon found" + } + if (rings.length == 1) { + return [rings] + } + + const polygons: Coords[][] = [] + let currentPolygon: Coords[] + + for (let i = 0; i < rings.length; i++) { + let ring = rings[i] + const area = this.signedArea(ring) + if (area === 0) { + // Weird, degenerate ring + continue + } + const ccw = area < 0 + + if (ccw === (area < 0)) { + if (currentPolygon) { + polygons.push(currentPolygon) + } + currentPolygon = [ring] + + } else { + currentPolygon.push(ring) + } + } + if (currentPolygon) { + polygons.push(currentPolygon) + } + + return polygons + } + + public toGeoJson(geometry: number[], typeIndex: 1 | 2 | 3, properties: any): Feature { + let coords: Coords[] = this.encodeGeometry(geometry) + let classified = undefined switch (typeIndex) { case 1: const points = [] @@ -38,9 +97,9 @@ class MvtFeatureBuilder { break case 3: - let classified = this.classifyRings(coords) - for (let i = 0; i < coords.length; i++) { - for (let j = 0; j < coords[i].length; j++) { + classified = MvtFeatureBuilder.classifyRings(coords) + for (let i = 0; i < classified.length; i++) { + for (let j = 0; j < classified[i].length; j++) { this.project(classified[i][j]) } } @@ -48,9 +107,11 @@ class MvtFeatureBuilder { } let type: string = MvtFeatureBuilder.geom_types[typeIndex] + let polygonCoords: Coords | Coords[] | Coords[][] if (coords.length === 1) { - coords = coords[0] + polygonCoords = (classified ?? coords)[0] } else { + polygonCoords = classified ?? coords type = "Multi" + type } @@ -58,13 +119,22 @@ class MvtFeatureBuilder { type: "Feature", geometry: { type: type, - coordinates: coords, + coordinates: polygonCoords, }, properties, } } - private encodeGeometry(geometry: number[]) { + /** + * + * const geometry = [9,233,8704,130,438,1455,270,653,248,423,368,493,362,381,330,267,408,301,406,221,402,157,1078,429,1002,449,1036,577,800,545,1586,1165,164,79,40] + * const builder = new MvtFeatureBuilder(4096, 66705, 43755, 17) + * const expected = [[3.2106759399175644,51.213658395282124],[3.2108227908611298,51.21396418776169],[3.2109133154153824,51.21410154168976],[3.210996463894844,51.214190590500664],[3.211119845509529,51.214294340548975],[3.211241215467453,51.2143745681588],[3.2113518565893173,51.21443085341426],[3.211488649249077,51.21449427925393],[3.2116247713565826,51.214540903490956],[3.211759552359581,51.21457408647774],[3.2121209800243378,51.214664394485254],[3.212456926703453,51.21475890267553],[3.2128042727708817,51.214880292910834],[3.213072493672371,51.214994962285544],[3.2136042416095734,51.21523984134939],[3.2136592268943787,51.21525664260963],[3.213672637939453,51.21525664260963]] + * builder.project(builder.encodeGeometry(geometry)[0]) // => expected + * @param geometry + * @private + */ + private encodeGeometry(geometry: number[]): Coords[] { let cX = 0 let cY = 0 let coordss: Coords[] = [] @@ -86,7 +156,7 @@ class MvtFeatureBuilder { currentRing = [] } } - if (commandId === 1 || commandId === 2){ + if (commandId === 1 || commandId === 2) { for (let j = 0; j < commandCount; j++) { const dx = geometry[i + j * 2 + 1] cX += ((dx >> 1) ^ (-(dx & 1))) @@ -94,10 +164,11 @@ class MvtFeatureBuilder { cY += ((dy >> 1) ^ (-(dy & 1))) currentRing.push([cX, cY]) } - i = commandCount * 2 + i += commandCount * 2 } - if(commandId === 7){ + if (commandId === 7) { currentRing.push([...currentRing[0]]) + i++ } } @@ -107,62 +178,12 @@ class MvtFeatureBuilder { return coordss } - private signedArea(ring: Coords): number { - let sum = 0 - const len = ring.length - // J is basically (i - 1) % len - let j = len - 1 - let p1 - let p2 - for (let i = 0; i < len; i++) { - p1 = ring[i] - p2 = ring[j] - sum += (p2.x - p1.x) * (p1.y + p2.y) - j = i - } - return sum - } - - private classifyRings(rings: Coords[]): Coords[][] { - const len = rings.length - - if (len <= 1) return [rings] - - const polygons = [] - let polygon - // CounterClockWise - let ccw: boolean - - for (let i = 0; i < len; i++) { - const area = this.signedArea(rings[i]) - if (area === 0) continue - - if (ccw === undefined) { - ccw = area < 0 - } - if (ccw === (area < 0)) { - if (polygon) { - polygons.push(polygon) - } - polygon = [rings[i]] - - } else { - polygon.push(rings[i]) - } - } - if (polygon) { - polygons.push(polygon) - } - - return polygons - } - /** * Inline replacement of the location by projecting - * @param line - * @private + * @param line the line which will be rewritten inline + * @return line */ - private project(line: [number, number][]) { + private project(line: Coords) { const y0 = this._y0 const x0 = this._x0 const size = this._size @@ -174,12 +195,13 @@ class MvtFeatureBuilder { 360 / Math.PI * Math.atan(Math.exp(y2 * Math.PI / 180)) - 90, ] } + return line } } -export default class MvtSource implements FeatureSource { +export default class MvtSource implements FeatureSourceForTile { - private static readonly schemaSpec = ` + private static readonly schemaSpec21 = ` package vector_tile; option optimize_for = LITE_RUNTIME; @@ -259,26 +281,30 @@ message Tile { extensions 16 to 8191; } ` - private static readonly tile_schema = pbfCompile(PbfSchema.parse(MvtSource.schemaSpec)).Tile - - + private static readonly tile_schema = (pbfCompile.default ?? pbfCompile)(PbfSchema.parse(MvtSource.schemaSpec21)).Tile + public readonly features: Store[]> private readonly _url: string private readonly _layerName: string private readonly _features: UIEventSource[]> = new UIEventSource[]>([]) - public readonly features: Store[]> = this._features - private readonly x: number - private readonly y: number - private readonly z: number + public readonly x: number + public readonly y: number + public readonly z: number - constructor(url: string, x: number, y: number, z: number, layerName?: string) { + constructor(url: string, x: number, y: number, z: number, layerName?: string, isActive?: Store) { this._url = url this._layerName = layerName this.x = x this.y = y this.z = z this.downloadSync() + this.features = this._features.map(fs => { + if (fs === undefined || isActive?.data === false) { + return [] + } + return fs + }, [isActive]) } private getValue(v: { @@ -316,16 +342,23 @@ message Tile { } - private downloadSync(){ + private downloadSync() { this.download().then(d => { - if(d.length === 0){ + if (d.length === 0) { return } return this._features.setData(d) - }).catch(e => {console.error(e)}) + }).catch(e => { + console.error(e) + }) } + private async download(): Promise { const result = await fetch(this._url) + if (result.status !== 200) { + console.error("Could not download tile " + this._url) + return [] + } const buffer = await result.arrayBuffer() const data = MvtSource.tile_schema.read(new Pbf(buffer)) const layers = data.layers @@ -336,7 +369,7 @@ message Tile { } layer = layers.find(l => l.name === this._layerName) } - if(!layer){ + if (!layer) { return [] } const builder = new MvtFeatureBuilder(layer.extent, this.x, this.y, this.z) diff --git a/src/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts index 88455e9bb..010af52ff 100644 --- a/src/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts +++ b/src/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts @@ -1,4 +1,4 @@ -import { Store } from "../../UIEventSource" +import { ImmutableStore, Store } from "../../UIEventSource" import DynamicTileSource from "./DynamicTileSource" import { Utils } from "../../../Utils" import GeoJsonSource from "../Sources/GeoJsonSource" @@ -65,7 +65,7 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource { const blackList = new Set() super( - source.geojsonZoomLevel, + new ImmutableStore(source.geojsonZoomLevel), layer.minzoom, (zxy) => { if (whitelist !== undefined) { diff --git a/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts index ac8713332..3b7476768 100644 --- a/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts +++ b/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts @@ -1,13 +1,45 @@ import { Store } from "../../UIEventSource" -import DynamicTileSource from "./DynamicTileSource" +import DynamicTileSource, { PolygonSourceMerger } from "./DynamicTileSource" import { Utils } from "../../../Utils" import { BBox } from "../../BBox" import LayerConfig from "../../../Models/ThemeConfig/LayerConfig" import MvtSource from "../Sources/MvtSource" import { Tiles } from "../../../Models/TileRange" import Constants from "../../../Models/Constants" +import FeatureSourceMerger from "../Sources/FeatureSourceMerger" -export default class DynamicMvtileSource extends DynamicTileSource { + +class PolygonMvtSource extends PolygonSourceMerger{ + constructor( layer: LayerConfig, + mapProperties: { + zoom: Store + bounds: Store + }, + options?: { + isActive?: Store + }) { + const roundedZoom = mapProperties.zoom.mapD(z => Math.min(Math.floor(z/2)*2, 14)) + super( + roundedZoom, + layer.minzoom, + (zxy) => { + const [z, x, y] = Tiles.tile_from_index(zxy) + const url = Utils.SubstituteKeys(Constants.VectorTileServer, + { + z, x, y, layer: layer.id, + type: "polygons", + }) + return new MvtSource(url, x, y, z) + }, + mapProperties, + { + isActive: options?.isActive, + }) + } +} + + +class PointMvtSource extends DynamicTileSource { constructor( layer: LayerConfig, @@ -19,14 +51,16 @@ export default class DynamicMvtileSource extends DynamicTileSource { isActive?: Store }, ) { + const roundedZoom = mapProperties.zoom.mapD(z => Math.min(Math.floor(z/2)*2, 14)) super( - mapProperties.zoom, + roundedZoom, layer.minzoom, (zxy) => { const [z, x, y] = Tiles.tile_from_index(zxy) const url = Utils.SubstituteKeys(Constants.VectorTileServer, { z, x, y, layer: layer.id, + type: "pois", }) return new MvtSource(url, x, y, z) }, @@ -37,3 +71,24 @@ export default class DynamicMvtileSource extends DynamicTileSource { ) } } + +export default class DynamicMvtileSource extends FeatureSourceMerger { + + constructor( + layer: LayerConfig, + mapProperties: { + zoom: Store + bounds: Store + }, + options?: { + isActive?: Store + }, + ) { + const roundedZoom = mapProperties.zoom.mapD(z => Math.floor(z)) + super( + new PointMvtSource(layer, mapProperties, options), + new PolygonMvtSource(layer, mapProperties, options) + + ) + } +} diff --git a/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts index 3bb5affd9..f905ae893 100644 --- a/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts +++ b/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts @@ -1,25 +1,37 @@ import { Store, Stores } from "../../UIEventSource" import { Tiles } from "../../../Models/TileRange" import { BBox } from "../../BBox" -import { FeatureSource } from "../FeatureSource" +import { FeatureSource, FeatureSourceForTile } from "../FeatureSource" import FeatureSourceMerger from "../Sources/FeatureSourceMerger" +import { Feature } from "geojson" +import { Utils } from "../../../Utils" +import { GeoOperations } from "../../GeoOperations" + /*** * A tiled source which dynamically loads the required tiles at a fixed zoom level. * A single featureSource will be initialized for every tile in view; which will later be merged into this featureSource */ -export default class DynamicTileSource extends FeatureSourceMerger { +export default class DynamicTileSource extends FeatureSourceMerger { + /** + * + * @param zoomlevel If {z} is specified in the source, the 'zoomlevel' will be used as zoomlevel to download from + * @param minzoom Only activate this feature source if zoomed in further then this + * @param constructSource + * @param mapProperties + * @param options + */ constructor( zoomlevel: Store, minzoom: number, - constructSource: (tileIndex: number) => FeatureSource, + constructSource: (tileIndex: number) => Src, mapProperties: { bounds: Store zoom: Store }, options?: { isActive?: Store - } + }, ) { super() const loadedTiles = new Set() @@ -34,32 +46,32 @@ export default class DynamicTileSource extends FeatureSourceMerger { if (mapProperties.zoom.data < minzoom) { return undefined } - const z = Math.round(zoomlevel.data) + const z = Math.floor(zoomlevel.data) const tileRange = Tiles.TileRangeBetween( z, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), - bounds.getWest() + bounds.getWest(), ) if (tileRange.total > 500) { console.warn( - "Got a really big tilerange, bounds and location might be out of sync" + "Got a really big tilerange, bounds and location might be out of sync", ) return undefined } const needed = Tiles.MapRange(tileRange, (x, y) => - Tiles.tile_index(z, x, y) + Tiles.tile_index(z, x, y), ).filter((i) => !loadedTiles.has(i)) if (needed.length === 0) { return undefined } return needed }, - [options?.isActive, mapProperties.zoom] + [options?.isActive, mapProperties.zoom], ) - .stabilized(250) + .stabilized(250), ) neededTiles.addCallbackAndRunD((neededIndexes) => { @@ -70,3 +82,70 @@ export default class DynamicTileSource extends FeatureSourceMerger { }) } } + + +/** + * The PolygonSourceMerger receives various small pieces of bigger polygons and stitches them together. + * This is used to reconstruct polygons of vector tiles + */ +export class PolygonSourceMerger extends DynamicTileSource { + constructor( + zoomlevel: Store, + minzoom: number, + constructSource: (tileIndex: number) => FeatureSourceForTile, + mapProperties: { + bounds: Store + zoom: Store + }, + options?: { + isActive?: Store + }, + ) { + super(zoomlevel, minzoom, constructSource, mapProperties, options) + } + + protected addDataFromSources(sources: FeatureSourceForTile[]) { + sources = Utils.NoNull(sources) + const all: Map = new Map() + const zooms: Map = new Map() + + for (const source of sources) { + let z = source.z + for (const f of source.features.data) { + const id = f.properties.id + if(id.endsWith("146616907")){ + console.log("Horeca totaal") + } + if (!all.has(id)) { + // No other parts of this polygon have been seen before, simply add it + all.set(id, f) + zooms.set(id, z) + continue + } + + // A part of this object has been seen before, eventually from a different zoom level + const oldV = all.get(id) + const oldZ = zooms.get(id) + if (oldZ > z) { + // The store contains more detailed information, so we ignore this part which has a lower accuraccy + continue + } + if (oldZ < z) { + // The old value has worse accuracy then what we receive now, we throw it away + all.set(id, f) + zooms.set(id, z) + continue + } + const merged = GeoOperations.union(f, oldV) + merged.properties = oldV.properties + all.set(id, merged) + zooms.set(id, z) + } + } + + const newList = Array.from(all.values()) + this.features.setData(newList) + this._featuresById.setData(all) + } + +} diff --git a/src/UI/Map/ShowDataLayer.ts b/src/UI/Map/ShowDataLayer.ts index 64a3cfe94..75c7213b0 100644 --- a/src/UI/Map/ShowDataLayer.ts +++ b/src/UI/Map/ShowDataLayer.ts @@ -303,7 +303,6 @@ class LineRenderingLayer { type: "FeatureCollection", features, }, - cluster: true, promoteId: "id", }) const linelayer = this._layername + "_line" diff --git a/src/UI/Test.svelte b/src/UI/Test.svelte index 70f864f1b..4e278c4b1 100644 --- a/src/UI/Test.svelte +++ b/src/UI/Test.svelte @@ -4,20 +4,18 @@ import MaplibreMap from "./Map/MaplibreMap.svelte" import { Map as MlMap } from "maplibre-gl" import { MapLibreAdaptor } from "./Map/MapLibreAdaptor" - import Constants from "../Models/Constants" - import toilet from "../assets/generated/layers/toilet.json" + import shops from "../assets/generated/layers/shops.json" import LayerConfig from "../Models/ThemeConfig/LayerConfig" import DynamicMvtileSource from "../Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource" import ShowDataLayer from "./Map/ShowDataLayer" - const tl = new LayerConfig(toilet) + const tl = new LayerConfig(shops) let map: UIEventSource = new UIEventSource(undefined) let adaptor = new MapLibreAdaptor(map) const src = new DynamicMvtileSource(tl, adaptor) - src.features.addCallbackAndRun(f => console.log(">>> Features are", f)) new ShowDataLayer(map, { layer: tl, features: src @@ -27,94 +25,7 @@ lat: 51.2095, lon: 3.2260, }) adaptor.zoom.setData(13) - const loadedIcons = new Set() - async function loadImage(map: MlMap, url: string, name: string): Promise { - return new Promise((resolve, reject) => { - if (loadedIcons.has(name)) { - return new Promise((resolve, reject) => resolve()) - } - loadedIcons.add(name) - if (Constants.defaultPinIcons.indexOf(url) >= 0) { - url = "./assets/svg/" + url + ".svg" - } - map.loadImage( - url, - (error, image) => { - if (error) { - reject(error) - } - map.addImage(name, image) - resolve() - }) - }) - } - - map.addCallbackAndRunD(map => { - map.on("load", async () => { - console.log("Onload") - await loadImage(map, "https://upload.wikimedia.org/wikipedia/commons/7/7c/201408_cat.png", "cat") - - /* - map.addSource("drinking_water", { - "type": "vector", - "tiles": ["http://127.0.0.2:7800/public.drinking_water/{z}/{x}/{y}.pbf"], // http://127.0.0.2:7800/public.drinking_water.json", - }) - - map.addLayer( - { - "id": "drinking_water_layer", - "type": "circle", - "source": "drinking_water", - "source-layer": "public.drinking_water", - "paint": { - "circle-radius": 5, - "circle-color": "#ff00ff", - "circle-stroke-width": 2, - "circle-stroke-color": "#000000", - }, - }, - )*/ - /* - map.addSource("toilet", { - "type": "vector", - "tiles": ["http://127.0.0.2:7800/public.toilet/{z}/{x}/{y}.pbf"], // http://127.0.0.2:7800/public.drinking_water.json", - }) - - map.addLayer( - { - "id": "toilet_layer", - "type": "circle", - "source": "toilet", - "source-layer": "public.toilet", - "paint": { - "circle-radius": 5, - "circle-color": "#0000ff", - "circle-stroke-width": 2, - "circle-stroke-color": "#000000", - }, - }, - ) - map.addLayer({ - "id": "points", - "type": "symbol", - "source": "toilet", - "source-layer": "public.toilet", - "layout": { - "icon-overlap": "always", - "icon-image": "cat", - "icon-size": 0.05, - }, - })*/ - - - map.on("click", "drinking_water_layer", (e) => { -// Copy coordinates array. - console.log(e) - console.warn(">>>", e.features[0]) - }) - }) - })