Optimizing trees theme and clustering

Logic/FeatureSource/FeaturePipeline.ts

@@ -58,7 +58,7 @@ export default class FeaturePipeline {
     private readonly freshnesses = new Map<string, TileFreshnessCalculator>();
 
     private readonly oldestAllowedDate: Date = new Date(new Date().getTime() - 60 * 60 * 24 * 30 * 1000);
-    private readonly osmSourceZoomLevel = 15
+    private readonly osmSourceZoomLevel
 
     constructor(
         handleFeatureSource: (source: FeatureSourceForLayer & Tiled) => void,
@@ -74,10 +74,12 @@ export default class FeaturePipeline {
             readonly overpassMaxZoom: UIEventSource<number>;
             readonly osmConnection: OsmConnection
             readonly currentBounds: UIEventSource<BBox>,
+            readonly osmApiTileSize: UIEventSource<number>
         }) {
         this.state = state;
 
         const self = this
+        this.osmSourceZoomLevel = state.osmApiTileSize.data;
         // milliseconds
         const useOsmApi = state.locationControl.map(l => l.zoom > (state.overpassMaxZoom.data ?? 12))
         this.relationTracker = new RelationsTracker()
@@ -275,18 +277,19 @@ export default class FeaturePipeline {
     private getNeededTilesFromOsm(isSufficientlyZoomed: UIEventSource<boolean>): UIEventSource<number[]> {
         const self = this
         return this.state.currentBounds.map(bbox => {
+            console.log("Current bbox is", bbox)
             if (bbox === undefined) {
-                return undefined
+                return []
             }
             if (!isSufficientlyZoomed.data) {
-                return undefined;
+                return [];
             }
             const osmSourceZoomLevel = self.osmSourceZoomLevel
             const range = bbox.containingTileRange(osmSourceZoomLevel)
             const tileIndexes = []
             if (range.total >= 100) {
                 // Too much tiles!
-                return []
+                return undefined
             }
             Tiles.MapRange(range, (x, y) => {
                 const i = Tiles.tile_index(osmSourceZoomLevel, x, y);
@@ -299,7 +302,7 @@ export default class FeaturePipeline {
                 tileIndexes.push(i)
             })
             return tileIndexes
-        })
+        }, [isSufficientlyZoomed])
     }
 
     private initOverpassUpdater(state: {

Logic/State/FeaturePipelineState.ts

@@ -8,10 +8,6 @@ import {UIEventSource} from "../UIEventSource";
 import MapState from "./MapState";
 import SelectedFeatureHandler from "../Actors/SelectedFeatureHandler";
 import Hash from "../Web/Hash";
-import ScrollableFullScreen from "../../UI/Base/ScrollableFullScreen";
-import Translations from "../../UI/i18n/Translations";
-import SimpleAddUI from "../../UI/BigComponents/SimpleAddUI";
-import StrayClickHandler from "../Actors/StrayClickHandler";
 
 export default class FeaturePipelineState extends MapState {
 
@@ -25,7 +21,7 @@ export default class FeaturePipelineState extends MapState {
         super(layoutToUse);
 
         const clustering = layoutToUse.clustering
-        this.featureAggregator = TileHierarchyAggregator.createHierarchy();
+        this.featureAggregator = TileHierarchyAggregator.createHierarchy(this);
         const clusterCounter = this.featureAggregator
         const self = this;
         this.featurePipeline = new FeaturePipeline(
@@ -117,12 +113,9 @@ export default class FeaturePipelineState extends MapState {
             features: this.featureAggregator.getCountsForZoom(clustering, this.locationControl, clustering.minNeededElements),
             leafletMap: leafletMap,
             layerToShow: ShowTileInfo.styling,
-            enablePopups: false,
+            enablePopups: this.featureSwitchIsDebugging.data,
         })
     }
 
-  
-
-
 
 }

Logic/State/UserRelatedState.ts

@@ -83,6 +83,7 @@ export default class UserRelatedState extends ElementsState {
 
 
         this.InitializeLanguage();
+        this.initHomeLocation()
         new SelectedElementTagsUpdater(this)
 
     }

Models/ThemeConfig/LayoutConfig.ts

@@ -90,7 +90,7 @@ export default class LayoutConfig {
         this.startZoom = json.startZoom;
         this.startLat = json.startLat;
         this.startLon = json.startLon;
-        if(json.widenFactor < 1){
+        if(json.widenFactor < 0.02){
             if(official){
                 throw "Widenfactor too small"
             }else{

UI/ShowDataLayer/ShowDataLayer.ts

@@ -38,7 +38,8 @@ export default class ShowDataLayer {
         ShowDataLayer.dataLayerIds++
         this._enablePopups = options.enablePopups ?? true;
         if (options.features === undefined) {
-            throw "Invalid ShowDataLayer invocation"
+            console.error("Invalid ShowDataLayer invocation: options.features is undefed")
+            throw "Invalid ShowDataLayer invocation: options.features is undefed"
         }
         const features = options.features.features.map(featFreshes => featFreshes.map(ff => ff.feature));
         this._features = features;
@@ -166,7 +167,7 @@ export default class ShowDataLayer {
 
 
     private createStyleFor(feature) {
-        const tagsSource = this.allElements?.addOrGetElement(feature) ?? new UIEventSource<any>(feature.properties.id);
+        const tagsSource = this.allElements?.addOrGetElement(feature) ?? new UIEventSource<any>(feature.properties);
         // Every object is tied to exactly one layer
         const layer = this._layerToShow
         return layer?.GenerateLeafletStyle(tagsSource, true);

UI/ShowDataLayer/TileHierarchyAggregator.ts

@@ -4,7 +4,11 @@ import {UIEventSource} from "../../Logic/UIEventSource";
 import {Tiles} from "../../Models/TileRange";
 import StaticFeatureSource from "../../Logic/FeatureSource/Sources/StaticFeatureSource";
 import {BBox} from "../../Logic/BBox";
+import FilteredLayer from "../../Models/FilteredLayer";
 
+/**
+ * A feature source containing but a single feature, which keeps stats about a tile
+ */
 export class TileHierarchyAggregator implements FeatureSource {
     private _parent: TileHierarchyAggregator;
     private _root: TileHierarchyAggregator;
@@ -16,28 +20,39 @@ export class TileHierarchyAggregator implements FeatureSource {
 
     private _subtiles: [TileHierarchyAggregator, TileHierarchyAggregator, TileHierarchyAggregator, TileHierarchyAggregator] = [undefined, undefined, undefined, undefined]
     public totalValue: number = 0
+    public showCount: number = 0
+    public hiddenCount: number = 0
 
     private static readonly empty = []
     public readonly features = new UIEventSource<{ feature: any, freshness: Date }[]>(TileHierarchyAggregator.empty)
     public readonly name;
 
     private readonly featuresStatic = []
-    private readonly featureProperties: { count: string, kilocount: string, tileId: string, id: string };
+    private readonly featureProperties: { count: string, kilocount: string, tileId: string, id: string, showCount: string, totalCount: string };
-
+    private readonly _state: { filteredLayers: UIEventSource<FilteredLayer[]> };
-    private constructor(parent: TileHierarchyAggregator, z: number, x: number, y: number) {
+    private readonly updateSignal = new UIEventSource<any>(undefined)
+    
+    private constructor(parent: TileHierarchyAggregator,
+                        state: {
+                            filteredLayers: UIEventSource<FilteredLayer[]>
+                        },
+                        z: number, x: number, y: number) {
         this._parent = parent;
+        this._state = state;
         this._root = parent?._root ?? this
         this._z = z;
         this._x = x;
         this._y = y;
         this._tileIndex = Tiles.tile_index(z, x, y)
         this.name = "Count(" + this._tileIndex + ")"
-
+        
         const totals = {
-            id: ""+this._tileIndex,
+            id: "" + this._tileIndex,
-            tileId: ""+this._tileIndex,
+            tileId: "" + this._tileIndex,
             count: `0`,
-            kilocount: "0"
+            kilocount: "0",
+            showCount: "0",
+            totalCount: "0"
         }
         this.featureProperties = totals
 
@@ -91,25 +106,57 @@ export class TileHierarchyAggregator implements FeatureSource {
     private update() {
         const newMap = new Map<string, number>()
         let total = 0
+        let hiddenCount = 0
+        let showCount = 0
+        let isShown: Map<string, FilteredLayer> = new Map<string, FilteredLayer>()
+        for (const filteredLayer of this._state.filteredLayers.data) {
+            isShown.set(filteredLayer.layerDef.id, filteredLayer)
+        }
         this?._counter?.countsPerLayer?.data?.forEach((count, layerId) => {
-            newMap.set(layerId, count)
+            newMap.set("layer:" + layerId, count)
             total += count
+            this.featureProperties["direct_layer:" + layerId] = count
+            const flayer = isShown.get(layerId)
+            if (flayer.isDisplayed.data && this._z >= flayer.layerDef.minzoom) {
+                showCount += count
+            } else {
+                hiddenCount += count;
+            }
         })
 
+
         for (const tile of this._subtiles) {
             if (tile === undefined) {
                 continue;
             }
             total += tile.totalValue
+
+            showCount += tile.showCount
+            hiddenCount += tile.hiddenCount
+
+            for (const key in tile.featureProperties) {
+                if (key.startsWith("layer:")) {
+                    newMap.set(key, (newMap.get(key) ?? 0) + Number(tile.featureProperties[key] ?? 0))
+                }
+            }
         }
+
         this.totalValue = total
+        this.showCount = showCount
+        this.hiddenCount = hiddenCount
         this._parent?.update()
-        
+
         if (total === 0) {
             this.features.setData(TileHierarchyAggregator.empty)
         } else {
             this.featureProperties.count = "" + total;
-            this.featureProperties.kilocount = "" +Math.floor(total/1000);
+            this.featureProperties.kilocount = "" + Math.floor(total / 1000);
+            this.featureProperties.showCount = "" + showCount
+            this.featureProperties.totalCount = "" + total
+            newMap.forEach((value, key) => {
+                this.featureProperties[key] = "" + value
+            })
+
             this.features.data = this.featuresStatic
             this.features.ping()
         }
@@ -137,49 +184,48 @@ export class TileHierarchyAggregator implements FeatureSource {
 
             const subtileIndex = yDiff * 2 + xDiff;
             if (this._subtiles[subtileIndex] === undefined) {
-                this._subtiles[subtileIndex] = new TileHierarchyAggregator(this, tileZ, tileX, tileY)
+                this._subtiles[subtileIndex] = new TileHierarchyAggregator(this, this._state, tileZ, tileX, tileY)
             }
             this._subtiles[subtileIndex].addTile(source)
         }
-
+        this.updateSignal.setData(source)
     }
 
-    public static createHierarchy() {
+    public static createHierarchy(state: { filteredLayers: UIEventSource<FilteredLayer[]> }) {
-        return new TileHierarchyAggregator(undefined, 0, 0, 0)
+        return new TileHierarchyAggregator(undefined, state, 0, 0, 0)
     }
 
-    private visitSubTiles(f : (aggr: TileHierarchyAggregator) => boolean){
+    private visitSubTiles(f: (aggr: TileHierarchyAggregator) => boolean) {
         const visitFurther = f(this)
-        if(visitFurther){
+        if (visitFurther) {
             this._subtiles.forEach(tile => tile?.visitSubTiles(f))
         }
     }
-    
+
-    getCountsForZoom(clusteringConfig: {maxZoom: number}, locationControl: UIEventSource<{ zoom : number }>, cutoff: number = 0) : FeatureSource{
+    getCountsForZoom(clusteringConfig: { maxZoom: number }, locationControl: UIEventSource<{ zoom: number }>, cutoff: number = 0): FeatureSource {
         const self = this
         const empty = []
-        return new StaticFeatureSource(
+        const features = locationControl.map(loc => loc.zoom).map(targetZoom => {
-            locationControl.map(loc => loc.zoom).map(targetZoom => {
+            if (targetZoom - 1 > clusteringConfig.maxZoom) {
-                if(targetZoom-1 > clusteringConfig.maxZoom){
+                return empty
-                    return empty
+            }
+
+            const features = []
+            self.visitSubTiles(aggr => {
+                if (aggr.showCount < cutoff) {
+                    return false
                 }
-                
+                if (aggr._z === targetZoom) {
-                const features = []
+                    features.push(...aggr.features.data)
-                self.visitSubTiles(aggr => {
+                    return false
-                    if(aggr.totalValue < cutoff) {
+                }
-                        return false
+                return aggr._z <= targetZoom;
-                    }
-                    if(aggr._z === targetZoom){
-                        features.push(...aggr.features.data)
-                        return false
-                    }
-                    return aggr._z <= targetZoom;
-                    
-                })
-                
-                return features
             })
-            , true);
+
+            return features
+        }, [this.updateSignal.stabilized(500)])
+
+        return new StaticFeatureSource(features, true);
     }
 }
 
@@ -209,14 +255,11 @@ class SingleTileCounter implements Tiled {
         const layer = source.layer.layerDef
         this.registeredLayers.set(layer.id, layer)
         const self = this
-
         source.features.map(f => {
             const isDisplayed = source.layer.isDisplayed.data
             self.countsPerLayer.data.set(layer.id, isDisplayed ? f.length : 0)
             self.countsPerLayer.ping()
         }, [source.layer.isDisplayed])
-        
-
     }
 
 }

Utils.ts

@@ -168,7 +168,12 @@ export class Utils {
             if (!tags.hasOwnProperty(key)) {
                 continue
             }
+            try{
             txt = txt.replace(new RegExp("{" + key + "}", "g"), tags[key] ?? "")
+            }catch(e){
+                console.error("WEIRD" , e)
+                throw e
+            }
         }
         txt = txt.replace(new RegExp('{.*}', "g"), "")
         return txt;

assets/layers/cluster_style/cluster_style.json

@@ -1,22 +1,26 @@
 {
     "id": "cluster_style",
-    "description": "The style for the clustering in all themes.",
+    "description": "The style for the clustering in all themes. Enable `debug=true` to peak into clustered tiles",
     "source": {
         "osmTags": "tileId~*"
     },
+    "title": "Clustered data",
+    "tagRenderings": [
+        "all_tags"
+    ],
     "color": {
         "render": "#3c3",
         "mappings": [
             {
-                "if": "count>200",
+                "if": "showCount>200",
                 "then": "#f33"
             },
             {
-                "if": "count>100",
+                "if": "showCount>100",
                 "then": "#c93"
             },
             {
-                "if": "count>50",
+                "if": "showCount>50",
                 "then": "#cc3"
             }
         ]
@@ -25,10 +29,10 @@
         "render": "1"
     },
     "label": {
-        "render": "<div class='rounded-full text-xl font-bold' style='width: 2rem; height: 2rem; background: white'>{count}</div>",
+        "render": "<div class='rounded-full text-xl font-bold' style='width: 2rem; height: 2rem; background: white'>{showCount}</div>",
         "mappings": [
             {
-                "if": "count>1000",
+                "if": "showCount>1000",
                 "then": "<div class='rounded-full text-xl font-bold flex flex-col' style='width: 2.5rem; height: 2.5rem; background: white'>{kilocount}K</div>"
             }
         ]

assets/layers/tree_node/tree_node.json

@@ -563,5 +563,12 @@
                 "ru": "Если вы не уверены в том, лиственное это дерево или хвойное."
             }
         }
-    ]
+    ],
+    "allowMove": {
+        "enableRelocation": false,
+        "enableImproveAccuraccy": true
+    },
+    "deletion": {
+        "minNeededChangesets": 5
+    }
 }

assets/themes/trees/trees.json

@@ -45,12 +45,13 @@
   "startLat": 50.642,
   "startLon": 4.482,
   "startZoom": 8,
-  "widenFactor": 1.01,
+  "widenFactor": 0.2,
   "socialImage": "./assets/themes/trees/logo.svg",
   "clustering": {
     "maxZoom": 19,
     "minNeededElements": 25
   },
+  "osmApiTileSize": 18,
   "layers": [
     "tree_node"
   ],

index.ts

@@ -106,7 +106,7 @@ DetermineLayout.GetLayout().then(value => {
     console.log("Got ", value)
     Init.Init(value[0], value[1])
 }).catch(err => {
-    console.error(err)
+    console.error("Error while initializing: ", err, err.stack)
 })