More work on clustering, more or less finished

InitUiElements.ts

@@ -39,7 +39,8 @@ import Combine from "./UI/Base/Combine";
 import {SubtleButton} from "./UI/Base/SubtleButton";
 import ShowTileInfo from "./UI/ShowDataLayer/ShowTileInfo";
 import {Tiles} from "./Models/TileRange";
-import PerTileCountAggregator from "./UI/ShowDataLayer/PerTileCountAggregator";
+import {TileHierarchyAggregator} from "./UI/ShowDataLayer/PerTileCountAggregator";
+import {BBox} from "./Logic/GeoOperations";
 
 export class InitUiElements {
     static InitAll(
@@ -430,48 +431,67 @@ export class InitUiElements {
             return flayers;
         });
 
-        const clusterCounter = new PerTileCountAggregator(State.state.locationControl.map(l => {
+        const layers = State.state.layoutToUse.data.layers
-            const z = l.zoom + 1
+        const clusterShow = Math.min(...layers.map(layer => layer.minzoom))
-            if(z < 7){
+        
-                return 7
+        
-            }
+        const clusterCounter = TileHierarchyAggregator.createHierarchy()
-            return z
-        }))
-        const clusterShow = Math.min(...State.state.layoutToUse.data.layers.map(layer => layer.minzoomVisible ?? layer.minzoom))
         new ShowDataLayer({
-            features: clusterCounter,
+            features: clusterCounter.getCountsForZoom(State.state.locationControl, State.state.layoutToUse.data.clustering.minNeededElements),
             leafletMap: State.state.leafletMap,
             layerToShow: ShowTileInfo.styling,
-            doShowLayer: State.state.locationControl.map(l => l.zoom < clusterShow)
+            doShowLayer: layers.length === 1 ? undefined : State.state.locationControl.map(l => l.zoom < clusterShow)
         })
+
         State.state.featurePipeline = new FeaturePipeline(
             source => {
+
+                clusterCounter.addTile(source)
+                
                 const clustering = State.state.layoutToUse.data.clustering
                 const doShowFeatures = source.features.map(
                     f => {
                         const z = State.state.locationControl.data.zoom
+
+                        if (z < source.layer.layerDef.minzoom) {
+                            // Layer is always hidden for this zoom level
+                            return false;
+                        }
+
                         if (z >= clustering.maxZoom) {
                             return true
                         }
-                        if(z < source.layer.layerDef.minzoom){
+
-                            return false;
-                        }
                         if (f.length > clustering.minNeededElements) {
-                            console.log("Activating clustering for tile ", Tiles.tile_from_index(source.tileIndex)," as it has ", f.length, "features (clustering starts at)", clustering.minNeededElements)
+                            // This tile alone has too much features
+                            return false
+                        }
+
+                        let [tileZ, tileX, tileY] = Tiles.tile_from_index(source.tileIndex);
+                        if (tileZ >= z) {
+
+                            while (tileZ > z) {
+                                tileZ--
+                                tileX = Math.floor(tileX / 2)
+                                tileY = Math.floor(tileY / 2)
+                            }
+
+                            if (clusterCounter.getTile(Tiles.tile_index(tileZ, tileX, tileY))?.totalValue > clustering.minNeededElements) {
+                                return false
+                            }
+                        }
+
+
+                        const bounds = State.state.currentBounds.data
+                        const tilebbox = BBox.fromTileIndex(source.tileIndex)
+                        if (!tilebbox.overlapsWith(bounds)) {
                             return false
                         }
 
                         return true
-                    }, [State.state.locationControl]
+                    }, [State.state.locationControl, State.state.currentBounds]
                 )
-                clusterCounter.addTile(source, doShowFeatures.map(b => !b))
                
-                /*
-                new ShowTileInfo({source: source, 
-                    leafletMap: State.state.leafletMap, 
-                    layer: source.layer.layerDef,
-                    doShowLayer: doShowFeatures.map(b => !b)
-                })*/
                 new ShowDataLayer(
                     {
                         features: source,

Logic/FeatureSource/FeaturePipeline.ts

@@ -134,6 +134,8 @@ export default class FeaturePipeline implements FeatureSourceState {
                 layer: source.layer,
                 minZoomLevel: 14,
                 dontEnforceMinZoom: true,
+                maxFeatureCount: state.layoutToUse.data.clustering.minNeededElements,
+                maxZoomLevel: state.layoutToUse.data.clustering.maxZoom,
                 registerTile: (tile) => {
                     // We save the tile data for the given layer to local storage
                     new SaveTileToLocalStorageActor(tile, tile.tileIndex)
@@ -171,7 +173,9 @@ export default class FeaturePipeline implements FeatureSourceState {
     
     private applyMetaTags(src: FeatureSourceForLayer){
         const self = this
-        console.log("Applying metatagging onto ", src.name)
+        console.debug("Applying metatagging onto ", src.name)
+        window.setTimeout(
+            () => {
                 MetaTagging.addMetatags(
                     src.features.data,
                     {
@@ -185,6 +189,10 @@ export default class FeaturePipeline implements FeatureSourceState {
                         includeNonDates: !src.layer.layerDef.source.isOsmCacheLayer
                     }
                 )
+            },
+            15
+        )
+       
     }
 
     private updateAllMetaTagging() {

Models/ThemeConfig/Json/LayoutConfigJson.ts

@@ -228,8 +228,8 @@ export interface LayoutConfigJson {
          */
         maxZoom?: number,
         /**
-         * The number of elements that should be showed (in total) before clustering starts to happen.
+         * The number of elements per tile needed to start clustering 
-         * If clustering is defined, defaults to 0
+         * If clustering is defined, defaults to 25
          */
         minNeededElements?: number
     },

Models/ThemeConfig/LayoutConfig.ts

@@ -129,17 +129,12 @@ export default class LayoutConfig {
 
         this.clustering = {
             maxZoom: 16,
-            minNeededElements: 500
+            minNeededElements: 25
         };
         if (json.clustering) {
             this.clustering = {
                 maxZoom: json.clustering.maxZoom ?? 18,
-                minNeededElements: json.clustering.minNeededElements ?? 1
+                minNeededElements: json.clustering.minNeededElements ?? 25
-            }
-            for (const layer of this.layers) {
-                if (layer.wayHandling !== LayerConfig.WAYHANDLING_CENTER_ONLY) {
-                    console.debug("WARNING: In order to allow clustering, every layer must be set to CENTER_ONLY. Layer", layer.id, "does not respect this for layout", this.id);
-                }
             }
         }
 

UI/ShowDataLayer/PerTileCountAggregator.ts

@@ -3,49 +3,54 @@ import {BBox} from "../../Logic/GeoOperations";
 import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
 import {UIEventSource} from "../../Logic/UIEventSource";
 import {Tiles} from "../../Models/TileRange";
+import StaticFeatureSource from "../../Logic/FeatureSource/Sources/StaticFeatureSource";
 
+export class TileHierarchyAggregator implements FeatureSource {
+    private _parent: TileHierarchyAggregator;
+    private _root: TileHierarchyAggregator;
+    private _z: number;
+    private _x: number;
+    private _y: number;
+    private _tileIndex: number
+    private _counter: SingleTileCounter
 
-/**
+    private _subtiles: [TileHierarchyAggregator, TileHierarchyAggregator, TileHierarchyAggregator, TileHierarchyAggregator] = [undefined, undefined, undefined, undefined]
- * A feature source containing meta features.
+    public totalValue: number = 0
- * It will contain exactly one point for every tile of the specified (dynamic) zoom level
- */
-export default class PerTileCountAggregator implements FeatureSource {
-    public readonly features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]);
-    public readonly name: string = "PerTileCountAggregator"
 
-    private readonly perTile: Map<number, SingleTileCounter> = new Map<number, SingleTileCounter>()
+    private static readonly empty = []
-    private readonly _requestedZoomLevel: UIEventSource<number>;
+    public readonly features = new UIEventSource<{ feature: any, freshness: Date }[]>(TileHierarchyAggregator.empty)
+    public readonly name;
 
-    constructor(requestedZoomLevel: UIEventSource<number>) {
+    private readonly featuresStatic = []
-        this._requestedZoomLevel = requestedZoomLevel;
+    private readonly featureProperties: { count: number, tileId: number };
-        const self = this;
+
-        this._requestedZoomLevel.addCallbackAndRun(_ => self.update())
+    private constructor(parent: TileHierarchyAggregator, z: number, x: number, y: number) {
+        this._parent = parent;
+        this._root = parent?._root ?? this
+        this._z = z;
+        this._x = x;
+        this._y = y;
+        this._tileIndex = Tiles.tile_index(z, x, y)
+        this.name = "Count(" + this._tileIndex + ")"
+
+        const totals = {
+            tileId: this._tileIndex,
+            count: 0
         }
+        this.featureProperties = totals
 
-    private update() {
         const now = new Date()
-        const allCountsAsFeatures : {feature: any, freshness: Date}[] = []
-        const aggregate = this.calculatePerTileCount()
-        aggregate.forEach((totalsPerLayer, tileIndex) => {
-            const totals = {}
-            let totalCount = 0
-            totalsPerLayer.forEach((total, layerId) => {
-                totals[layerId] = total
-                totalCount += total
-            })
-            totals["tileId"] = tileIndex
-            totals["count"] = totalCount
         const feature = {
             "type": "Feature",
             "properties": totals,
             "geometry": {
                 "type": "Point",
-                    "coordinates": Tiles.centerPointOf(...Tiles.tile_from_index(tileIndex))
+                "coordinates": Tiles.centerPointOf(z, x, y)
             }
         }
-            allCountsAsFeatures.push({feature: feature, freshness: now})
+        this.featuresStatic.push({feature: feature, freshness: now})
 
-            const bbox=  BBox.fromTileIndex(tileIndex)
+        const bbox = BBox.fromTile(z, x, y)
         const box = {
             "type": "Feature",
             "properties": totals,
@@ -62,61 +67,114 @@ export default class PerTileCountAggregator implements FeatureSource {
                 ]
             }
         }
-            allCountsAsFeatures.push({feature:box, freshness: now})
+        this.featuresStatic.push({feature: box, freshness: now})
-        })
-        this.features.setData(allCountsAsFeatures)
     }
 
-    /**
+    public getTile(tileIndex): TileHierarchyAggregator {
-     * Calculates an aggregate count per tile and per subtile
+        if (tileIndex === this._tileIndex) {
-     * @private
+            return this;
-     */
-    private calculatePerTileCount() {
-        const perTileCount = new Map<number, Map<string, number>>()
-        const targetZoom = this._requestedZoomLevel.data;
-        // We only search for tiles of the same zoomlevel or a higher zoomlevel, which is embedded
-        for (const singleTileCounter of Array.from(this.perTile.values())) {
-
-            let tileZ = singleTileCounter.z
-            let tileX = singleTileCounter.x
-            let tileY = singleTileCounter.y
-            if (tileZ < targetZoom) {
-                continue;
         }
-
+        let [tileZ, tileX, tileY] = Tiles.tile_from_index(tileIndex)
-            while (tileZ > targetZoom) {
+        while (tileZ - 1 > this._z) {
             tileX = Math.floor(tileX / 2)
             tileY = Math.floor(tileY / 2)
             tileZ--
         }
-            const tileI = Tiles.tile_index(tileZ, tileX, tileY)
+        const xDiff = tileX - (2 * this._x)
-            let counts = perTileCount.get(tileI)
+        const yDiff = tileY - (2 * this._y)
-            if (counts === undefined) {
+        const subtileIndex = yDiff * 2 + xDiff;
-                counts = new Map<string, number>()
+        return this._subtiles[subtileIndex]?.getTile(tileIndex)
-                perTileCount.set(tileI, counts)
-            }
-            singleTileCounter.countsPerLayer.data.forEach((count, layerId) => {
-                if (counts.has(layerId)) {
-                    counts.set(layerId, count + counts.get(layerId))
-                } else {
-                    counts.set(layerId, count)
     }
+
+    private update() {
+        const newMap = new Map<string, number>()
+        let total = 0
+        this?._counter?.countsPerLayer?.data?.forEach((count, layerId) => {
+            newMap.set(layerId, count)
+            total += count
         })
+
+        for (const tile of this._subtiles) {
+            if (tile === undefined) {
+                continue;
+            }
+            total += tile.totalValue
+        }
+        this.totalValue = total
+        this._parent?.update()
+        
+        if (total === 0) {
+            this.features.setData(TileHierarchyAggregator.empty)
+        } else {
+            this.featureProperties.count = total;
+            this.features.data = this.featuresStatic
+            this.features.ping()
         }
-        return perTileCount;
     }
 
-    public addTile(tile: FeatureSourceForLayer & Tiled, shouldBeCounted: UIEventSource<boolean>) {
+    public addTile(source: FeatureSourceForLayer & Tiled) {
-        let counter = this.perTile.get(tile.tileIndex)
+        const self = this;
-        if (counter === undefined) {
+        if (source.tileIndex === this._tileIndex) {
-            counter = new SingleTileCounter(tile.tileIndex)
+            if (this._counter === undefined) {
-            this.perTile.set(tile.tileIndex, counter)
+                this._counter = new SingleTileCounter(this._tileIndex)
-            // We do **NOT** add a callback on the perTile index, even though we could! It'll update just fine without it
+                this._counter.countsPerLayer.addCallbackAndRun(_ => self.update())
             }
-        counter.addTileCount(tile, shouldBeCounted)
+            this._counter.addTileCount(source)
+        } else {
+
+            // We have to give it to one of the subtiles
+            let [tileZ, tileX, tileY] = Tiles.tile_from_index(source.tileIndex)
+            while (tileZ - 1 > this._z) {
+                tileX = Math.floor(tileX / 2)
+                tileY = Math.floor(tileY / 2)
+                tileZ--
+            }
+            const xDiff = tileX - (2 * this._x)
+            const yDiff = tileY - (2 * this._y)
+
+            const subtileIndex = yDiff * 2 + xDiff;
+            if (this._subtiles[subtileIndex] === undefined) {
+                this._subtiles[subtileIndex] = new TileHierarchyAggregator(this, tileZ, tileX, tileY)
+            }
+            this._subtiles[subtileIndex].addTile(source)
+        }
+
+    }
+
+    public static createHierarchy() {
+        return new TileHierarchyAggregator(undefined, 0, 0, 0)
     }
 
 
+    private visitSubTiles(f : (aggr: TileHierarchyAggregator) => boolean){
+        const visitFurther = f(this)
+        if(visitFurther){
+            this._subtiles.forEach(tile => tile?.visitSubTiles(f))
+        }
+    }
+    
+    getCountsForZoom(locationControl: UIEventSource<{ zoom : number }>, cutoff: number) : FeatureSource{
+        const self = this
+        return new StaticFeatureSource(
+            locationControl.map(loc => {
+                const features = []
+                const targetZoom = loc.zoom
+                self.visitSubTiles(aggr => {
+                    if(aggr.totalValue < cutoff) {
+                        return false
+                    }
+                    if(aggr._z === targetZoom){
+                        features.push(...aggr.features.data)
+                        return false
+                    }
+                    return aggr._z <= targetZoom;
+                    
+                })
+                
+                return features
+            })
+            , true);
+    }
 }
 
 /**
@@ -131,6 +189,7 @@ class SingleTileCounter implements Tiled {
     public readonly x: number
     public readonly y: number
 
+
     constructor(tileIndex: number) {
         this.tileIndex = tileIndex
         this.bbox = BBox.fromTileIndex(tileIndex)
@@ -140,17 +199,16 @@ class SingleTileCounter implements Tiled {
         this.y = y
     }
 
-    public addTileCount(source: FeatureSourceForLayer, shouldBeCounted: UIEventSource<boolean>) {
+    public addTileCount(source: FeatureSourceForLayer) {
         const layer = source.layer.layerDef
         this.registeredLayers.set(layer.id, layer)
         const self = this
+
         source.features.map(f => {
-            /*if (!shouldBeCounted.data) {
-                return;
-            }*/
             self.countsPerLayer.data.set(layer.id, f.length)
             self.countsPerLayer.ping()
-        }, [shouldBeCounted])
+        })
+
     }
 
 }

UI/ShowDataLayer/ShowDataLayer.ts

@@ -17,6 +17,7 @@ export default class ShowDataLayer {
     // Used to generate a fresh ID when needed
     private _cleanCount = 0;
     private geoLayer = undefined;
+    private isDirty = false;
 
     /**
      * If the selected element triggers, this is used to lookup the correct layer and to open the popup
@@ -37,9 +38,30 @@ export default class ShowDataLayer {
         this._layerToShow = options.layerToShow;
         const self = this;
 
+        options.leafletMap.addCallbackAndRunD(_ => {
+                self.update(options)
+            }
+        );
+
         features.addCallback(_ => self.update(options));
-        options.leafletMap.addCallback(_ => self.update(options));
+        options.doShowLayer?.addCallbackAndRun(doShow => {
-        this.update(options);
+            const mp = options.leafletMap.data;
+            if (mp == undefined) {
+                return;
+            }
+            if (doShow) {
+                if (self.isDirty) {
+                    self.update(options)
+                } else {
+                    mp.addLayer(this.geoLayer)
+                }
+            } else {
+                if(this.geoLayer !== undefined){
+                    mp.removeLayer(this.geoLayer)
+                }
+            }
+
+        })
 
         State.state.selectedElement.addCallbackAndRunD(selected => {
             if (self._leafletMap.data === undefined) {
@@ -68,24 +90,14 @@ export default class ShowDataLayer {
             }
         })
 
-        options.doShowLayer?.addCallbackAndRun(doShow => {
+    }
-            const mp = options.leafletMap.data;
+
-            if (this.geoLayer == undefined || mp == undefined) {
+    private update(options: ShowDataLayerOptions) {
+        if (this._features.data === undefined) {
             return;
         }
-            if (doShow) {
+        this.isDirty = true;
-                mp.addLayer(this.geoLayer)
+        if (options?.doShowLayer?.data === false) {
-            } else {
-                mp.removeLayer(this.geoLayer)
-            }
-
-
-        })
-
-    }
-
-    private update(options) {
-        if (this._features.data === undefined) {
             return;
         }
         const mp = options.leafletMap.data;
@@ -99,7 +111,18 @@ export default class ShowDataLayer {
             mp.removeLayer(this.geoLayer);
         }
 
-        this.geoLayer = this.CreateGeojsonLayer()
+        const self = this;
+        const data = {
+            type: "FeatureCollection",
+            features: []
+        }
+        // @ts-ignore
+        this.geoLayer = L.geoJSON(data, {
+            style: feature => self.createStyleFor(feature),
+            pointToLayer: (feature, latLng) => self.pointToLayer(feature, latLng),
+            onEachFeature: (feature, leafletLayer) => self.postProcessFeature(feature, leafletLayer)
+        });
+
         const allFeats = this._features.data;
         for (const feat of allFeats) {
             if (feat === undefined) {
@@ -123,6 +146,7 @@ export default class ShowDataLayer {
         if (options.doShowLayer?.data ?? true) {
             mp.addLayer(this.geoLayer)
         }
+        this.isDirty = false;
     }
 
 
@@ -218,19 +242,4 @@ export default class ShowDataLayer {
 
     }
 
-    private CreateGeojsonLayer(): L.Layer {
-        const self = this;
-        const data = {
-            type: "FeatureCollection",
-            features: []
-        }
-        // @ts-ignore
-        return L.geoJSON(data, {
-            style: feature => self.createStyleFor(feature),
-            pointToLayer: (feature, latLng) => self.pointToLayer(feature, latLng),
-            onEachFeature: (feature, leafletLayer) => self.postProcessFeature(feature, leafletLayer)
-        });
-
-    }
-
 }

assets/layers/tree_node/tree_node.json

@@ -7,7 +7,7 @@
         "ru": "Дерево",
         "fr": "Arbre"
     },
-    "minzoom": 14,
+    "minzoom": 16,
     "source": {
         "osmTags": {
             "and": [

assets/themes/trees/trees.json

@@ -45,10 +45,11 @@
   "startLat": 50.642,
   "startLon": 4.482,
   "startZoom": 8,
-  "widenFactor": 1.5,
+  "widenFactor": 1.01,
   "socialImage": "./assets/themes/trees/logo.svg",
   "clustering": {
-    "maxZoom": 18
+    "maxZoom": 19,
+    "minNeededElements": 25
   },
   "layers": [
     "tree_node"