Add polygon merging

Docs/SettingUpPSQL.md

@@ -29,7 +29,7 @@ Install osm2pgsql (hint: compile from source is painless)
 To seed the database:
 
 ````
-osm2pgsql -O flex -E 4326 -S build_db.lua -s  --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi <file>.osm.pbf 
+osm2pgsql -O flex -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi <file>.osm.pbf 
 ````
 Storing properties to table '"public"."osm2pgsql_properties" takes about 25 minutes with planet.osm
 

package.json

@@ -21,7 +21,7 @@
       "oauth_secret": "NBWGhWDrD3QDB35xtVuxv4aExnmIt4FA_WgeLtwxasg",
       "url": "https://www.openstreetmap.org"
     },
-    "mvt_layer_server": "http://127.0.0.1:7800/public.{layer}/{z}/{x}/{y}.pbf",
+    "mvt_layer_server": "http://127.0.0.1:7800/public.{type}_{layer}/{z}/{x}/{y}.pbf",
     "disabled:oauth_credentials": {
       "##": "DEV",
       "#": "This client-id is registered by 'MapComplete' on https://master.apis.dev.openstreetmap.org/",

scripts/osm2pgsql/generateBuildDbScript.ts

@@ -5,91 +5,57 @@ import Script from "../Script"
 import fs from "fs"
 import { Or } from "../../src/Logic/Tags/Or"
 import { RegexTag } from "../../src/Logic/Tags/RegexTag"
-import { Utils } from "../../src/Utils"
 import { ValidateThemeEnsemble } from "../../src/Models/ThemeConfig/Conversion/Validation"
 import { AllKnownLayouts } from "../../src/Customizations/AllKnownLayouts"
 
 class LuaSnippets {
-    /**
+
-     * The main piece of code that calls `process_poi`
+    public static helpers = [
-     */
+        "function countTbl(tbl)\n" +
-    static tail = [
+        "  local c = 0\n" +
-        "function osm2pgsql.process_node(object)",
+        "  for n in pairs(tbl) do \n" +
-        "  process_poi(object, object:as_point())",
+        "    c = c + 1 \n" +
+        "  end\n" +
+        "  return c\n" +
         "end",
-        "",
+    ].join("\n")
-        "function osm2pgsql.process_way(object)",
-        "  if object.is_closed then",
-        "    process_poi(object, object:as_polygon():centroid())",
-        "  end",
-        "end",
-        ""].join("\n")
 
-    public static combine(calls: string[]): string {
+    public static toLuaFilter(tag: TagsFilter, useParens: boolean = false): string {
-        return [
+        if (tag instanceof Tag) {
-            `function process_poi(object, geom)`,
+            return `object.tags["${tag.key}"] == "${tag.value}"`
-            ...calls.map(c => "  " + c + "(object, geom)"),
-            `end`,
-        ].join("\n")
-    }
-}
-
-class GenerateLayerLua {
-    private readonly _id: string
-    private readonly _tags: TagsFilter
-    private readonly _foundInThemes: string[]
-
-    constructor(id: string, tags: TagsFilter, foundInThemes: string[] = []) {
-        this._tags = tags
-        this._id = id
-        this._foundInThemes = foundInThemes
-    }
-
-    public functionName() {
-        if (!this._tags) {
-            return undefined
         }
-        return `process_poi_${this._id}`
+        if (tag instanceof And) {
-    }
+            const expr = tag.and.map(t => this.toLuaFilter(t, true)).join(" and ")
-
+            if (useParens) {
-    public generateFunction(): string {
+                return "(" + expr + ")"
-        if (!this._tags) {
+            }
-            return undefined
+            return expr
         }
-        return [
+        if (tag instanceof Or) {
-            `local pois_${this._id} = osm2pgsql.define_table({`,
+            const expr = tag.or.map(t => this.toLuaFilter(t, true)).join(" or ")
-            this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "",
+            if (useParens) {
-            `  name = '${this._id}',`,
+                return "(" + expr + ")"
-            "  ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },",
+            }
-            "  columns = {",
+            return expr
-            "    { column = 'tags', type = 'jsonb' },",
+        }
-            "    { column = 'geom', type = 'point', projection = 4326, not_null = true },",
+        if (tag instanceof RegexTag) {
-            "  }" +
+            let expr = LuaSnippets.regexTagToLua(tag)
-            "})",
+            if (useParens) {
-            "",
+                expr = "(" + expr + ")"
-            "",
+            }
-            `function ${this.functionName()}(object, geom)`,
+            return expr
-            "  local matches_filter = " + this.toLuaFilter(this._tags),
+        }
-            "  if( not matches_filter) then",
+        let msg = "Could not handle" + tag.asHumanString(false, false, {})
-            "    return",
+        console.error(msg)
-            "  end",
+        throw msg
-            "  local a = {",
-            "    geom = geom,",
-            "    tags = object.tags",
-            "  }",
-            "  ",
-            `  pois_${this._id}:insert(a)`,
-            "end",
-            "",
-        ].join("\n")
     }
 
-    private regexTagToLua(tag: RegexTag) {
+    private static regexTagToLua(tag: RegexTag) {
         if (typeof tag.value === "string" && tag.invert) {
             return `object.tags["${tag.key}"] ~= "${tag.value}"`
         }
 
-        const v = (<RegExp> tag.value).source.replace(/\\\//g, "/")
+        const v = (<RegExp>tag.value).source.replace(/\\\//g, "/")
 
         if ("" + tag.value === "/.+/is" && !tag.invert) {
             return `object.tags["${tag.key}"] ~= nil`
@@ -115,35 +81,58 @@ class GenerateLayerLua {
         return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${v}"))`
     }
 
-    private toLuaFilter(tag: TagsFilter, useParens: boolean = false): string {
+}
-        if (tag instanceof Tag) {
+
-            return `object.tags["${tag.key}"] == "${tag.value}"`
+class GenerateLayerLua {
-        }
+    private readonly _id: string
-        if (tag instanceof And) {
+    private readonly _tags: TagsFilter
-            const expr = tag.and.map(t => this.toLuaFilter(t, true)).join(" and ")
+    private readonly _foundInThemes: string[]
-            if (useParens) {
+
-                return "(" + expr + ")"
+    constructor(id: string, tags: TagsFilter, foundInThemes: string[] = []) {
-            }
+        this._tags = tags
-            return expr
+        this._id = id
-        }
+        this._foundInThemes = foundInThemes
-        if (tag instanceof Or) {
-            const expr = tag.or.map(t => this.toLuaFilter(t, true)).join(" or ")
-            if (useParens) {
-                return "(" + expr + ")"
-            }
-            return expr
-        }
-        if (tag instanceof RegexTag) {
-            let expr = this.regexTagToLua(tag)
-            if (useParens) {
-                expr = "(" + expr + ")"
-            }
-            return expr
-        }
-        let msg = "Could not handle" + tag.asHumanString(false, false, {})
-        console.error(msg)
-        throw msg
     }
+
+    public generateTables(): string {
+        if (!this._tags) {
+            return undefined
+        }
+        return [
+            `db_tables.pois_${this._id} = osm2pgsql.define_table({`,
+            this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "",
+            `  name = 'pois_${this._id}',`,
+            "  ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },",
+            "  columns = {",
+            "    { column = 'tags', type = 'jsonb' },",
+            "    { column = 'geom', type = 'point', projection = 4326, not_null = true },",
+            "  }",
+            "})",
+            "",
+            `db_tables.lines_${this._id} = osm2pgsql.define_table({`,
+            this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "",
+            `  name = 'lines_${this._id}',`,
+            "  ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },",
+            "  columns = {",
+            "    { column = 'tags', type = 'jsonb' },",
+            "    { column = 'geom', type = 'linestring', projection = 4326, not_null = true },",
+            "  }",
+            "})",
+
+            `db_tables.polygons_${this._id} = osm2pgsql.define_table({`,
+            this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "",
+            `  name = 'polygons_${this._id}',`,
+            "  ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },",
+            "  columns = {",
+            "    { column = 'tags', type = 'jsonb' },",
+            "    { column = 'geom', type = 'polygon', projection = 4326, not_null = true },",
+            "  }",
+            "})",
+            "",
+        ].join("\n")
+    }
+
+
 }
 
 class GenerateBuildDbScript extends Script {
@@ -163,14 +152,93 @@ class GenerateBuildDbScript extends Script {
         })
 
         const script = [
-            ...generators.map(g => g.generateFunction()),
+            "local db_tables = {}",
-            LuaSnippets.combine(Utils.NoNull(generators.map(g => g.functionName()))),
+            LuaSnippets.helpers,
-            LuaSnippets.tail,
+            ...generators.map(g => g.generateTables()),
+            this.generateProcessPoi(allNeededLayers),
+            this.generateProcessWay(allNeededLayers),
         ].join("\n\n\n")
         const path = "build_db.lua"
         fs.writeFileSync(path, script, "utf-8")
         console.log("Written", path)
-        console.log(allNeededLayers.size+" layers will be created. Make sure to set 'max_connections' to at least  "+(10 + allNeededLayers.size) )
+        console.log(allNeededLayers.size + " layers will be created with 3 tables each. Make sure to set 'max_connections' to at least  " + (10 + 3 * allNeededLayers.size))
+    }
+
+    private earlyAbort() {
+        return ["  if countTbl(object.tags) == 0 then",
+            "    return",
+            "  end",
+            ""].join("\n")
+    }
+
+    private generateProcessPoi(allNeededLayers: Map<string, { tags: TagsFilter; foundInTheme: string[] }>) {
+        const body: string[] = []
+        allNeededLayers.forEach(({ tags }, layerId) => {
+            body.push(
+                this.insertInto(tags, layerId, "pois_").join("\n"),
+            )
+        })
+
+        return [
+            "function osm2pgsql.process_node(object)",
+            this.earlyAbort(),
+            "  local geom = object:as_point()",
+            "  local matches_filter = false",
+            body.join("\n"),
+            "end",
+        ].join("\n")
+    }
+
+    /**
+     * If matches_filter
+     * @param tags
+     * @param layerId
+     * @param tableprefix
+     * @private
+     */
+    private insertInto(tags: TagsFilter, layerId: string, tableprefix: "pois_" | "lines_" | "polygons_") {
+        const filter = LuaSnippets.toLuaFilter(tags)
+        return [
+            "  matches_filter = " + filter,
+            "  if matches_filter then",
+            "    db_tables." + tableprefix + layerId + ":insert({",
+            "      geom = geom,",
+            "      tags = object.tags",
+            "    })",
+            "  end",
+        ]
+    }
+
+    private generateProcessWay(allNeededLayers: Map<string, { tags: TagsFilter }>) {
+        const bodyLines: string[] = []
+        allNeededLayers.forEach(({ tags }, layerId) => {
+            bodyLines.push(this.insertInto(tags, layerId, "lines_").join("\n"))
+        })
+
+        const bodyPolygons: string[] = []
+        allNeededLayers.forEach(({ tags }, layerId) => {
+            bodyPolygons.push(this.insertInto(tags, layerId, "polygons_").join("\n"))
+        })
+
+        return [
+            "function process_polygon(object, geom)",
+            "  local matches_filter",
+            ...bodyPolygons,
+            "end",
+            "function process_linestring(object, geom)",
+            "  local matches_filter",
+            ...bodyLines,
+            "end",
+            "",
+            "function osm2pgsql.process_way(object)",
+            this.earlyAbort(),
+            "  if object.is_closed then",
+            "    process_polygon(object, object:as_polygon())",
+            "  else",
+            "    process_linestring(object, object:as_linestring())",
+            "  end",
+            "end",
+        ].join("\n")
     }
 }
 

src/Logic/FeatureSource/FeatureSource.ts

@@ -16,6 +16,12 @@ export interface FeatureSourceForLayer<T extends Feature = Feature> extends Feat
     readonly layer: FilteredLayer
 }
 
+export interface FeatureSourceForTile <T extends Feature = Feature> extends FeatureSource<T> {
+    readonly x: number
+    readonly y: number
+    readonly z: number
+
+}
 /**
  * A feature source which is aware of the indexes it contains
  */

src/Logic/FeatureSource/Sources/FeatureSourceMerger.ts

@@ -2,43 +2,49 @@ import { Store, UIEventSource } from "../../UIEventSource"
 import { FeatureSource, IndexedFeatureSource } from "../FeatureSource"
 import { Feature } from "geojson"
 import { Utils } from "../../../Utils"
+import DynamicTileSource from "../TiledFeatureSource/DynamicTileSource"
 
 /**
- *
+ * The featureSourceMerger receives complete geometries from various sources.
+ * If multiple sources contain the same object (as determined by 'id'), only one copy of them is retained
  */
-export default class FeatureSourceMerger implements IndexedFeatureSource {
+export default class FeatureSourceMerger<Src extends FeatureSource = FeatureSource> implements IndexedFeatureSource {
     public features: UIEventSource<Feature[]> = new UIEventSource([])
     public readonly featuresById: Store<Map<string, Feature>>
-    private readonly _featuresById: UIEventSource<Map<string, Feature>>
+    protected readonly _featuresById: UIEventSource<Map<string, Feature>>
-    private readonly _sources: FeatureSource[] = []
+    private readonly _sources: Src[] = []
     /**
      * Merges features from different featureSources.
      * In case that multiple features have the same id, the latest `_version_number` will be used. Otherwise, we will take the last one
      */
-    constructor(...sources: FeatureSource[]) {
+    constructor(...sources: Src[]) {
         this._featuresById = new UIEventSource<Map<string, Feature>>(new Map<string, Feature>())
         this.featuresById = this._featuresById
         const self = this
         sources = Utils.NoNull(sources)
         for (let source of sources) {
             source.features.addCallback(() => {
-                self.addData(sources.map((s) => s.features.data))
+                self.addDataFromSources(sources)
             })
         }
-        this.addData(sources.map((s) => s.features.data))
+        this.addDataFromSources(sources)
         this._sources = sources
     }
 
-    public addSource(source: FeatureSource) {
+    public addSource(source: Src) {
         if (!source) {
             return
         }
         this._sources.push(source)
         source.features.addCallbackAndRun(() => {
-            this.addData(this._sources.map((s) => s.features.data))
+            this.addDataFromSources(this._sources)
         })
     }
 
+    protected addDataFromSources(sources: Src[]){
+        this.addData(sources.map(s => s.features.data))
+    }
+
     protected addData(sources: Feature[][]) {
         sources = Utils.NoNull(sources)
         let somethingChanged = false
@@ -56,7 +62,7 @@ export default class FeatureSourceMerger implements IndexedFeatureSource {
                 const id = f.properties.id
                 unseen.delete(id)
                 if (!all.has(id)) {
-                    // This is a new feature
+                    // This is a new, previously unseen feature
                     somethingChanged = true
                     all.set(id, f)
                     continue
@@ -81,10 +87,8 @@ export default class FeatureSourceMerger implements IndexedFeatureSource {
             return
         }
 
-        const newList = []
+        const newList = Array.from(all.values())
-        all.forEach((value) => {
+
-            newList.push(value)
-        })
         this.features.setData(newList)
         this._featuresById.setData(all)
     }

src/Logic/FeatureSource/Sources/MvtSource.ts

@@ -1,6 +1,6 @@
 import { Feature, Geometry } from "geojson"
 import { Store, UIEventSource } from "../../UIEventSource"
-import { FeatureSource } from "../FeatureSource"
+import { FeatureSourceForTile } from "../FeatureSource"
 import Pbf from "pbf"
 import * as pbfCompile from "pbf/compile"
 import * as PbfSchema from "protocol-buffers-schema"
@@ -19,8 +19,67 @@ class MvtFeatureBuilder {
         this._y0 = extent * y
     }
 
-    public toGeoJson(geometry, typeIndex, properties): Feature {
+    private static signedArea(ring: Coords): number {
-        let coords: [number, number] | Coords | Coords[] = this.encodeGeometry(geometry)
+        let sum = 0
+        const len = ring.length
+        // J is basically (i - 1) % len
+        let j = len - 1
+        let p1
+        let p2
+        for (let i = 0; i < len; i++) {
+            p1 = ring[i]
+            p2 = ring[j]
+            sum += (p2.x - p1.x) * (p1.y + p2.y)
+            j = i
+        }
+        return sum
+    }
+
+    /**
+     *
+     * const rings = [   [     [       3.208361864089966,       51.186908820014736     ],     [       3.2084155082702637,       51.18689537073311     ],     [       3.208436965942383,       51.186888646090836     ],     [       3.2084155082702637,       51.18686174751187     ],     [       3.2084155082702637,       51.18685502286465     ],     [       3.2083725929260254,       51.18686847215807     ],     [       3.2083404064178467,       51.18687519680333     ],     [       3.208361864089966,       51.186908820014736     ]   ] ]
+     * MvtFeatureBuilder.classifyRings(rings) // => [rings]
+     */
+    private static classifyRings(rings: Coords[]): Coords[][] {
+        if (rings.length <= 0) {
+            throw "Now rings in polygon found"
+        }
+        if (rings.length == 1) {
+            return [rings]
+        }
+
+        const polygons: Coords[][] = []
+        let currentPolygon: Coords[]
+
+        for (let i = 0; i < rings.length; i++) {
+            let ring = rings[i]
+            const area = this.signedArea(ring)
+            if (area === 0) {
+                // Weird, degenerate ring
+                continue
+            }
+            const ccw = area < 0
+
+            if (ccw === (area < 0)) {
+                if (currentPolygon) {
+                    polygons.push(currentPolygon)
+                }
+                currentPolygon = [ring]
+
+            } else {
+                currentPolygon.push(ring)
+            }
+        }
+        if (currentPolygon) {
+            polygons.push(currentPolygon)
+        }
+
+        return polygons
+    }
+
+    public toGeoJson(geometry: number[], typeIndex: 1 | 2 | 3, properties: any): Feature {
+        let coords: Coords[] = this.encodeGeometry(geometry)
+        let classified = undefined
         switch (typeIndex) {
             case 1:
                 const points = []
@@ -38,9 +97,9 @@ class MvtFeatureBuilder {
                 break
 
             case 3:
-                let classified = this.classifyRings(coords)
+                classified = MvtFeatureBuilder.classifyRings(coords)
-                for (let i = 0; i < coords.length; i++) {
+                for (let i = 0; i < classified.length; i++) {
-                    for (let j = 0; j < coords[i].length; j++) {
+                    for (let j = 0; j < classified[i].length; j++) {
                         this.project(classified[i][j])
                     }
                 }
@@ -48,9 +107,11 @@ class MvtFeatureBuilder {
         }
 
         let type: string = MvtFeatureBuilder.geom_types[typeIndex]
+        let polygonCoords: Coords | Coords[] | Coords[][]
         if (coords.length === 1) {
-            coords = coords[0]
+            polygonCoords = (classified ?? coords)[0]
         } else {
+            polygonCoords = classified ?? coords
             type = "Multi" + type
         }
 
@@ -58,13 +119,22 @@ class MvtFeatureBuilder {
             type: "Feature",
             geometry: {
                 type: <any>type,
-                coordinates: <any>coords,
+                coordinates: <any>polygonCoords,
             },
             properties,
         }
     }
 
-    private encodeGeometry(geometry: number[]) {
+    /**
+     *
+     * const geometry = [9,233,8704,130,438,1455,270,653,248,423,368,493,362,381,330,267,408,301,406,221,402,157,1078,429,1002,449,1036,577,800,545,1586,1165,164,79,40]
+     * const builder = new MvtFeatureBuilder(4096, 66705, 43755, 17)
+     * const expected = [[3.2106759399175644,51.213658395282124],[3.2108227908611298,51.21396418776169],[3.2109133154153824,51.21410154168976],[3.210996463894844,51.214190590500664],[3.211119845509529,51.214294340548975],[3.211241215467453,51.2143745681588],[3.2113518565893173,51.21443085341426],[3.211488649249077,51.21449427925393],[3.2116247713565826,51.214540903490956],[3.211759552359581,51.21457408647774],[3.2121209800243378,51.214664394485254],[3.212456926703453,51.21475890267553],[3.2128042727708817,51.214880292910834],[3.213072493672371,51.214994962285544],[3.2136042416095734,51.21523984134939],[3.2136592268943787,51.21525664260963],[3.213672637939453,51.21525664260963]]
+     * builder.project(builder.encodeGeometry(geometry)[0]) // => expected
+     * @param geometry
+     * @private
+     */
+    private encodeGeometry(geometry: number[]): Coords[] {
         let cX = 0
         let cY = 0
         let coordss: Coords[] = []
@@ -86,7 +156,7 @@ class MvtFeatureBuilder {
                     currentRing = []
                 }
             }
-            if (commandId === 1 || commandId === 2){
+            if (commandId === 1 || commandId === 2) {
                 for (let j = 0; j < commandCount; j++) {
                     const dx = geometry[i + j * 2 + 1]
                     cX += ((dx >> 1) ^ (-(dx & 1)))
@@ -94,10 +164,11 @@ class MvtFeatureBuilder {
                     cY += ((dy >> 1) ^ (-(dy & 1)))
                     currentRing.push([cX, cY])
                 }
-                i = commandCount * 2
+                i += commandCount * 2
             }
-            if(commandId === 7){
+            if (commandId === 7) {
                 currentRing.push([...currentRing[0]])
+                i++
             }
 
         }
@@ -107,62 +178,12 @@ class MvtFeatureBuilder {
         return coordss
     }
 
-    private signedArea(ring: Coords): number {
-        let sum = 0
-        const len = ring.length
-        // J is basically (i - 1) % len
-        let j = len - 1
-        let p1
-        let p2
-        for (let i = 0; i < len; i++) {
-            p1 = ring[i]
-            p2 = ring[j]
-            sum += (p2.x - p1.x) * (p1.y + p2.y)
-            j = i
-        }
-        return sum
-    }
-
-    private classifyRings(rings: Coords[]): Coords[][] {
-        const len = rings.length
-
-        if (len <= 1) return [rings]
-
-        const polygons = []
-        let polygon
-        // CounterClockWise
-        let ccw: boolean
-
-        for (let i = 0; i < len; i++) {
-            const area = this.signedArea(rings[i])
-            if (area === 0) continue
-
-            if (ccw === undefined) {
-                ccw = area < 0
-            }
-            if (ccw === (area < 0)) {
-                if (polygon) {
-                    polygons.push(polygon)
-                }
-                polygon = [rings[i]]
-
-            } else {
-                polygon.push(rings[i])
-            }
-        }
-        if (polygon) {
-            polygons.push(polygon)
-        }
-
-        return polygons
-    }
-
     /**
      * Inline replacement of the location by projecting
-     * @param line
+     * @param line the line which will be rewritten inline
-     * @private
+     * @return line
      */
-    private project(line: [number, number][]) {
+    private project(line: Coords) {
         const y0 = this._y0
         const x0 = this._x0
         const size = this._size
@@ -174,12 +195,13 @@ class MvtFeatureBuilder {
                 360 / Math.PI * Math.atan(Math.exp(y2 * Math.PI / 180)) - 90,
             ]
         }
+        return line
     }
 }
 
-export default class MvtSource implements FeatureSource {
+export default class MvtSource implements FeatureSourceForTile {
 
-    private static readonly schemaSpec = `
+    private static readonly schemaSpec21 = `
     package vector_tile;
 
 option optimize_for = LITE_RUNTIME;
@@ -259,26 +281,30 @@ message Tile {
         extensions 16 to 8191;
 }
 `
-    private static readonly tile_schema = pbfCompile(PbfSchema.parse(MvtSource.schemaSpec)).Tile
+    private static readonly tile_schema = (pbfCompile.default ?? pbfCompile)(PbfSchema.parse(MvtSource.schemaSpec21)).Tile
-
+    public readonly features: Store<Feature<Geometry, { [name: string]: any }>[]>
-
     private readonly _url: string
     private readonly _layerName: string
     private readonly _features: UIEventSource<Feature<Geometry, {
         [name: string]: any
     }>[]> = new UIEventSource<Feature<Geometry, { [p: string]: any }>[]>([])
-    public readonly features: Store<Feature<Geometry, { [name: string]: any }>[]> = this._features
+    public readonly x: number
-    private readonly x: number
+    public readonly y: number
-    private readonly y: number
+    public readonly z: number
-    private readonly z: number
 
-    constructor(url: string, x: number, y: number, z: number, layerName?: string) {
+    constructor(url: string, x: number, y: number, z: number, layerName?: string, isActive?: Store<boolean>) {
         this._url = url
         this._layerName = layerName
         this.x = x
         this.y = y
         this.z = z
         this.downloadSync()
+        this.features = this._features.map(fs => {
+            if (fs === undefined || isActive?.data === false) {
+                return []
+            }
+            return fs
+        }, [isActive])
     }
 
     private getValue(v: {
@@ -316,16 +342,23 @@ message Tile {
 
     }
 
-    private downloadSync(){
+    private downloadSync() {
         this.download().then(d => {
-            if(d.length === 0){
+            if (d.length === 0) {
                 return
             }
             return this._features.setData(d)
-        }).catch(e => {console.error(e)})
+        }).catch(e => {
+            console.error(e)
+        })
     }
+
     private async download(): Promise<Feature[]> {
         const result = await fetch(this._url)
+        if (result.status !== 200) {
+            console.error("Could not download tile " + this._url)
+            return []
+        }
         const buffer = await result.arrayBuffer()
         const data = MvtSource.tile_schema.read(new Pbf(buffer))
         const layers = data.layers
@@ -336,7 +369,7 @@ message Tile {
             }
             layer = layers.find(l => l.name === this._layerName)
         }
-        if(!layer){
+        if (!layer) {
             return []
         }
         const builder = new MvtFeatureBuilder(layer.extent, this.x, this.y, this.z)

src/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts

@@ -1,4 +1,4 @@
-import { Store } from "../../UIEventSource"
+import { ImmutableStore, Store } from "../../UIEventSource"
 import DynamicTileSource from "./DynamicTileSource"
 import { Utils } from "../../../Utils"
 import GeoJsonSource from "../Sources/GeoJsonSource"
@@ -65,7 +65,7 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
 
         const blackList = new Set<string>()
         super(
-            source.geojsonZoomLevel,
+           new ImmutableStore(source.geojsonZoomLevel),
             layer.minzoom,
             (zxy) => {
                 if (whitelist !== undefined) {

src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts

@@ -1,13 +1,45 @@
 import { Store } from "../../UIEventSource"
-import DynamicTileSource from "./DynamicTileSource"
+import DynamicTileSource, { PolygonSourceMerger } from "./DynamicTileSource"
 import { Utils } from "../../../Utils"
 import { BBox } from "../../BBox"
 import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
 import MvtSource from "../Sources/MvtSource"
 import { Tiles } from "../../../Models/TileRange"
 import Constants from "../../../Models/Constants"
+import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
 
-export default class DynamicMvtileSource extends DynamicTileSource {
+
+class PolygonMvtSource extends PolygonSourceMerger{
+    constructor( layer: LayerConfig,
+                 mapProperties: {
+                     zoom: Store<number>
+                     bounds: Store<BBox>
+                 },
+                 options?: {
+                     isActive?: Store<boolean>
+                 }) {
+        const roundedZoom = mapProperties.zoom.mapD(z => Math.min(Math.floor(z/2)*2, 14))
+        super(
+            roundedZoom,
+            layer.minzoom,
+            (zxy) => {
+                const [z, x, y] = Tiles.tile_from_index(zxy)
+                const url = Utils.SubstituteKeys(Constants.VectorTileServer,
+                    {
+                        z, x, y, layer: layer.id,
+                        type: "polygons",
+                    })
+                return new MvtSource(url, x, y, z)
+            },
+            mapProperties,
+            {
+                isActive: options?.isActive,
+            })
+    }
+}
+
+
+class PointMvtSource extends DynamicTileSource {
 
     constructor(
         layer: LayerConfig,
@@ -19,14 +51,16 @@ export default class DynamicMvtileSource extends DynamicTileSource {
             isActive?: Store<boolean>
         },
     ) {
+        const roundedZoom = mapProperties.zoom.mapD(z => Math.min(Math.floor(z/2)*2, 14))
         super(
-            mapProperties.zoom,
+            roundedZoom,
             layer.minzoom,
             (zxy) => {
                 const [z, x, y] = Tiles.tile_from_index(zxy)
                 const url = Utils.SubstituteKeys(Constants.VectorTileServer,
                     {
                         z, x, y, layer: layer.id,
+                        type: "pois",
                     })
                 return new MvtSource(url, x, y, z)
             },
@@ -37,3 +71,24 @@ export default class DynamicMvtileSource extends DynamicTileSource {
         )
     }
 }
+
+export default class DynamicMvtileSource extends FeatureSourceMerger {
+
+    constructor(
+        layer: LayerConfig,
+        mapProperties: {
+            zoom: Store<number>
+            bounds: Store<BBox>
+        },
+        options?: {
+            isActive?: Store<boolean>
+        },
+    ) {
+        const roundedZoom = mapProperties.zoom.mapD(z => Math.floor(z))
+        super(
+            new PointMvtSource(layer, mapProperties, options),
+            new PolygonMvtSource(layer, mapProperties, options)
+
+        )
+    }
+}

src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts

@@ -1,25 +1,37 @@
 import { Store, Stores } from "../../UIEventSource"
 import { Tiles } from "../../../Models/TileRange"
 import { BBox } from "../../BBox"
-import { FeatureSource } from "../FeatureSource"
+import { FeatureSource, FeatureSourceForTile } from "../FeatureSource"
 import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
+import { Feature } from "geojson"
+import { Utils } from "../../../Utils"
+import { GeoOperations } from "../../GeoOperations"
+
 
 /***
  * A tiled source which dynamically loads the required tiles at a fixed zoom level.
  * A single featureSource will be initialized for every tile in view; which will later be merged into this featureSource
  */
-export default class DynamicTileSource extends FeatureSourceMerger {
+export default class DynamicTileSource<Src extends FeatureSource = FeatureSource> extends FeatureSourceMerger<Src> {
+    /**
+     *
+     * @param zoomlevel If {z} is specified in the source, the 'zoomlevel' will be used as zoomlevel to download from
+     * @param minzoom Only activate this feature source if zoomed in further then this
+     * @param constructSource
+     * @param mapProperties
+     * @param options
+     */
     constructor(
         zoomlevel: Store<number>,
         minzoom: number,
-        constructSource: (tileIndex: number) => FeatureSource,
+        constructSource: (tileIndex: number) => Src,
         mapProperties: {
             bounds: Store<BBox>
             zoom: Store<number>
         },
         options?: {
             isActive?: Store<boolean>
-        }
+        },
     ) {
         super()
         const loadedTiles = new Set<number>()
@@ -34,32 +46,32 @@ export default class DynamicTileSource extends FeatureSourceMerger {
                         if (mapProperties.zoom.data < minzoom) {
                             return undefined
                         }
-                        const z = Math.round(zoomlevel.data)
+                        const z = Math.floor(zoomlevel.data)
                         const tileRange = Tiles.TileRangeBetween(
                             z,
                             bounds.getNorth(),
                             bounds.getEast(),
                             bounds.getSouth(),
-                            bounds.getWest()
+                            bounds.getWest(),
                         )
                         if (tileRange.total > 500) {
                             console.warn(
-                                "Got a really big tilerange, bounds and location might be out of sync"
+                                "Got a really big tilerange, bounds and location might be out of sync",
                             )
                             return undefined
                         }
 
                         const needed = Tiles.MapRange(tileRange, (x, y) =>
-                            Tiles.tile_index(z, x, y)
+                            Tiles.tile_index(z, x, y),
                         ).filter((i) => !loadedTiles.has(i))
                         if (needed.length === 0) {
                             return undefined
                         }
                         return needed
                     },
-                    [options?.isActive, mapProperties.zoom]
+                    [options?.isActive, mapProperties.zoom],
                 )
-                .stabilized(250)
+                .stabilized(250),
         )
 
         neededTiles.addCallbackAndRunD((neededIndexes) => {
@@ -70,3 +82,70 @@ export default class DynamicTileSource extends FeatureSourceMerger {
         })
     }
 }
+
+
+/**
+ * The PolygonSourceMerger receives various small pieces of bigger polygons and stitches them together.
+ * This is used to reconstruct polygons of vector tiles
+ */
+export class PolygonSourceMerger extends DynamicTileSource<FeatureSourceForTile> {
+    constructor(
+        zoomlevel: Store<number>,
+        minzoom: number,
+        constructSource: (tileIndex: number) => FeatureSourceForTile,
+        mapProperties: {
+            bounds: Store<BBox>
+            zoom: Store<number>
+        },
+        options?: {
+            isActive?: Store<boolean>
+        },
+    ) {
+        super(zoomlevel, minzoom, constructSource, mapProperties, options)
+    }
+
+    protected addDataFromSources(sources: FeatureSourceForTile[]) {
+        sources = Utils.NoNull(sources)
+        const all: Map<string, Feature> = new Map()
+        const zooms: Map<string, number> = new Map()
+
+        for (const source of sources) {
+            let z = source.z
+            for (const f of source.features.data) {
+                const id = f.properties.id
+                if(id.endsWith("146616907")){
+                    console.log("Horeca totaal")
+                }
+                if (!all.has(id)) {
+                    // No other parts of this polygon have been seen before, simply add it
+                    all.set(id, f)
+                    zooms.set(id, z)
+                    continue
+                }
+
+                // A part of this object has been seen before, eventually from a different zoom level
+                const oldV = all.get(id)
+                const oldZ = zooms.get(id)
+                if (oldZ > z) {
+                    // The store contains more detailed information, so we ignore this part which has a lower accuraccy
+                    continue
+                }
+                if (oldZ < z) {
+                    // The old value has worse accuracy then what we receive now, we throw it away
+                    all.set(id, f)
+                    zooms.set(id, z)
+                    continue
+                }
+                const merged = GeoOperations.union(f, oldV)
+                merged.properties = oldV.properties
+                all.set(id, merged)
+                zooms.set(id, z)
+            }
+        }
+
+        const newList = Array.from(all.values())
+        this.features.setData(newList)
+        this._featuresById.setData(all)
+    }
+
+}

src/UI/Map/ShowDataLayer.ts

@@ -303,7 +303,6 @@ class LineRenderingLayer {
                         type: "FeatureCollection",
                         features,
                     },
-                    cluster: true,
                     promoteId: "id",
                 })
                 const linelayer = this._layername + "_line"

src/UI/Test.svelte

@@ -4,20 +4,18 @@
     import MaplibreMap from "./Map/MaplibreMap.svelte"
     import { Map as MlMap } from "maplibre-gl"
     import { MapLibreAdaptor } from "./Map/MapLibreAdaptor"
-    import Constants from "../Models/Constants"
+    import shops from "../assets/generated/layers/shops.json"
-    import toilet from "../assets/generated/layers/toilet.json"
     import LayerConfig from "../Models/ThemeConfig/LayerConfig"
     import DynamicMvtileSource from "../Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource"
     import ShowDataLayer from "./Map/ShowDataLayer"
 
-    const tl = new LayerConfig(<any>toilet)
+    const tl = new LayerConfig(<any>shops)
 
 
     let map: UIEventSource<MlMap> = new UIEventSource<MlMap>(undefined)
     let adaptor = new MapLibreAdaptor(map)
 
     const src = new DynamicMvtileSource(tl, adaptor)
-    src.features.addCallbackAndRun(f => console.log(">>> Features are", f))
     new ShowDataLayer(map, {
         layer: tl,
         features: src
@@ -27,94 +25,7 @@
         lat: 51.2095, lon: 3.2260,
     })
     adaptor.zoom.setData(13)
-    const loadedIcons = new Set<string>()
 
-    async function loadImage(map: MlMap, url: string, name: string): Promise<void> {
-        return new Promise((resolve, reject) => {
-            if (loadedIcons.has(name)) {
-                return new Promise<void>((resolve, reject) => resolve())
-            }
-            loadedIcons.add(name)
-            if (Constants.defaultPinIcons.indexOf(url) >= 0) {
-                url = "./assets/svg/" + url + ".svg"
-            }
-            map.loadImage(
-                url,
-                (error, image) => {
-                    if (error) {
-                        reject(error)
-                    }
-                    map.addImage(name, image)
-                    resolve()
-                })
-        })
-    }
-
-    map.addCallbackAndRunD(map => {
-        map.on("load", async () => {
-            console.log("Onload")
-            await loadImage(map, "https://upload.wikimedia.org/wikipedia/commons/7/7c/201408_cat.png", "cat")
-
-            /*
-            map.addSource("drinking_water", {
-                "type": "vector",
-                "tiles": ["http://127.0.0.2:7800/public.drinking_water/{z}/{x}/{y}.pbf"], // http://127.0.0.2:7800/public.drinking_water.json",
-            })
-
-            map.addLayer(
-                {
-                    "id": "drinking_water_layer",
-                    "type": "circle",
-                    "source": "drinking_water",
-                    "source-layer": "public.drinking_water",
-                    "paint": {
-                        "circle-radius": 5,
-                        "circle-color": "#ff00ff",
-                        "circle-stroke-width": 2,
-                        "circle-stroke-color": "#000000",
-                    },
-                },
-            )*/
-            /*
-                        map.addSource("toilet", {
-                            "type": "vector",
-                            "tiles": ["http://127.0.0.2:7800/public.toilet/{z}/{x}/{y}.pbf"], // http://127.0.0.2:7800/public.drinking_water.json",
-                        })
-            
-                        map.addLayer(
-                            {
-                                "id": "toilet_layer",
-                                "type": "circle",
-                                "source": "toilet",
-                                "source-layer": "public.toilet",
-                                "paint": {
-                                    "circle-radius": 5,
-                                    "circle-color": "#0000ff",
-                                    "circle-stroke-width": 2,
-                                    "circle-stroke-color": "#000000",
-                                },
-                            },
-                        )
-                        map.addLayer({
-                            "id": "points",
-                            "type": "symbol",
-                            "source": "toilet",
-                            "source-layer": "public.toilet",
-                            "layout": {
-                                "icon-overlap": "always",
-                                "icon-image": "cat",
-                                "icon-size": 0.05,
-                            },
-                        })*/
-
-
-            map.on("click", "drinking_water_layer", (e) => {
-// Copy coordinates array.
-                console.log(e)
-                console.warn(">>>", e.features[0])
-            })
-        })
-    })
 </script>
 
 <div class="h-screen w-screen">