Fix bounds and loading with overpass

Logic/Actors/OverpassFeatureSource.ts

@@ -38,8 +38,7 @@ export default class OverpassFeatureSource implements FeatureSource {
         readonly currentBounds: UIEventSource<BBox>
     }
     private readonly _isActive: UIEventSource<boolean>;
-    private readonly onBboxLoaded: (bbox: BBox, date: Date, layers: LayerConfig[]) => void;
+    private readonly onBboxLoaded: (bbox: BBox, date: Date, layers: LayerConfig[], zoomlevel: number) => void;
-    private readonly padToTiles : number
     constructor(
         state: {
             readonly locationControl: UIEventSource<Loc>,
@@ -50,20 +49,19 @@ export default class OverpassFeatureSource implements FeatureSource {
             readonly currentBounds: UIEventSource<BBox>
         },
         options: {
-            padToTiles: number,
+            padToTiles: UIEventSource<number>,
             isActive?: UIEventSource<boolean>,
             relationTracker: RelationsTracker,
-            onBboxLoaded?: (bbox: BBox, date: Date, layers: LayerConfig[]) => void
+            onBboxLoaded?: (bbox: BBox, date: Date, layers: LayerConfig[], zoomlevel: number) => void
         }) {
 
         this.state = state
         this._isActive = options.isActive;
-        this.padToTiles = options.padToTiles;
         this.onBboxLoaded = options.onBboxLoaded
         this.relationsTracker = options.relationTracker
         const self = this;
         state.currentBounds.addCallback(_ => {
-            self.update()
+            self.update(options.padToTiles.data)
         })
 
     }
@@ -86,21 +84,21 @@ export default class OverpassFeatureSource implements FeatureSource {
         return new Overpass(new Or(filters), extraScripts, interpreterUrl, this.state.overpassTimeout, this.relationsTracker);
     }
 
-    private update() {
+    private update(paddedZoomLevel: number) {
         if (!this._isActive.data) {
             return;
         }
         const self = this;
-        this.updateAsync().then(bboxDate => {
+        this.updateAsync(paddedZoomLevel).then(bboxDate => {
             if(bboxDate === undefined || self.onBboxLoaded === undefined){
                 return;
             }
             const [bbox, date, layers] = bboxDate
-            self.onBboxLoaded(bbox, date, layers)
+            self.onBboxLoaded(bbox, date, layers, paddedZoomLevel)
         })
     }
 
-    private async updateAsync(): Promise<[BBox, Date, LayerConfig[]]> {
+    private async updateAsync(padToZoomLevel: number): Promise<[BBox, Date, LayerConfig[]]> {
         if (this.runningQuery.data) {
             console.log("Still running a query, not updating");
             return undefined;
@@ -111,14 +109,11 @@ export default class OverpassFeatureSource implements FeatureSource {
             return undefined;
         }
 
-        const bounds = this.state.currentBounds.data?.pad(this.state.layoutToUse.widenFactor)?.expandToTileBounds(this.padToTiles);
+        const bounds = this.state.currentBounds.data?.pad(this.state.layoutToUse.widenFactor)?.expandToTileBounds(padToZoomLevel);
 
         if (bounds === undefined) {
             return undefined;
         }
-        console.log("Current bounds are", this.state.currentBounds.data," padded with",this.state.layoutToUse.widenFactor+":", 
-            this.state.currentBounds.data.pad(this.state.layoutToUse.widenFactor),
-            "Tileexpanded: ",this.state.currentBounds.data?.pad(this.state.layoutToUse.widenFactor)?.expandToTileBounds(this.padToTiles) )
         const self = this;
         
         

Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts

@@ -7,7 +7,7 @@ import {FeatureSourceForLayer} from "../FeatureSource";
 
 export default class SaveTileToLocalStorageActor {
     public static readonly storageKey: string = "cached-features";
-    public static readonly formatVersion: string = "1"
+    public static readonly formatVersion: string = "2"
 
     constructor(source: FeatureSourceForLayer, tileIndex: number) {
         source.features.addCallbackAndRunD(features => {
@@ -31,6 +31,6 @@ export default class SaveTileToLocalStorageActor {
         const key = `${SaveTileToLocalStorageActor.storageKey}-${layerId}-${tileId}`
         localStorage.setItem(key + "-time", JSON.stringify(freshness.getTime()))
         localStorage.setItem(key + "-format", SaveTileToLocalStorageActor.formatVersion)
-
+        console.log("Marked ", key, "as visited")
     }
 }

Logic/FeatureSource/FeaturePipeline.ts

@@ -155,7 +155,7 @@ export default class FeaturePipeline {
                             tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
                         }
                     })
-                }else{
+                } else {
                     new RegisteringAllFromFeatureSourceActor(src)
                     perLayerHierarchy.get(id).registerTile(src)
                     src.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(src))
@@ -200,7 +200,7 @@ export default class FeaturePipeline {
         new PerLayerFeatureSourceSplitter(state.filteredLayers,
             (source) => TiledFeatureSource.createHierarchy(source, {
                 layer: source.layer,
-                minZoomLevel: this.osmSourceZoomLevel,
+                minZoomLevel: source.layer.layerDef.minzoom,
                 dontEnforceMinZoom: true,
                 maxFeatureCount: state.layoutToUse.clustering.minNeededElements,
                 maxZoomLevel: state.layoutToUse.clustering.maxZoom,
@@ -233,7 +233,7 @@ export default class FeaturePipeline {
 
 
         // Whenever fresh data comes in, we need to update the metatagging
-        self.newDataLoadedSignal.stabilized(1000).addCallback(src => {
+        self.newDataLoadedSignal.stabilized(1000).addCallback(_ => {
             self.updateAllMetaTagging()
         })
 
@@ -270,15 +270,15 @@ export default class FeaturePipeline {
         const self = this
         return this.state.currentBounds.map(bbox => {
             if (bbox === undefined) {
-                return
+                return undefined
             }
             if (!isSufficientlyZoomed.data) {
-                return;
+                return undefined;
             }
             const osmSourceZoomLevel = self.osmSourceZoomLevel
             const range = bbox.containingTileRange(osmSourceZoomLevel)
             const tileIndexes = []
-            if (range.total > 100) {
+            if (range.total >= 100) {
                 // Too much tiles!
                 return []
             }
@@ -288,7 +288,7 @@ export default class FeaturePipeline {
                 if (oldestDate !== undefined && oldestDate > this.oldestAllowedDate) {
                     console.debug("Skipping tile", osmSourceZoomLevel, x, y, "as a decently fresh one is available")
                     // The cached tiles contain decently fresh data
-                    return;
+                    return undefined;
                 }
                 tileIndexes.push(i)
             })
@@ -321,29 +321,30 @@ export default class FeaturePipeline {
             }
 
             const range = bbox.containingTileRange(zoom)
-            if (range.total > 100) {
+            if (range.total >= 5000) {
                 return false
             }
             const self = this;
             const allFreshnesses = Tiles.MapRange(range, (x, y) => self.freshnessForVisibleLayers(zoom, x, y))
             return allFreshnesses.some(freshness => freshness === undefined || freshness < this.oldestAllowedDate)
-
         }, [state.locationControl])
 
         const self = this;
         const updater = new OverpassFeatureSource(state,
             {
-                padToTiles: this.osmSourceZoomLevel,
+                padToTiles: state.locationControl.map(l => Math.min(15, l.zoom + 1)),
                 relationTracker: this.relationTracker,
                 isActive: useOsmApi.map(b => !b && overpassIsActive.data, [overpassIsActive]),
-                onBboxLoaded: ((bbox, date, downloadedLayers) => {
+                onBboxLoaded: (bbox, date, downloadedLayers, paddedToZoomLevel) => {
-                    Tiles.MapRange(bbox.containingTileRange(self.osmSourceZoomLevel), (x, y) => {
+                    Tiles.MapRange(bbox.containingTileRange(paddedToZoomLevel), (x, y) => {
+                       const tileIndex =  Tiles.tile_index(paddedToZoomLevel, x, y)
                         downloadedLayers.forEach(layer => {
-                            SaveTileToLocalStorageActor.MarkVisited(layer.id, Tiles.tile_index(this.osmSourceZoomLevel, x, y), date)
+                            self.freshnesses.get(layer.id).addTileLoad(tileIndex, date)
+                            SaveTileToLocalStorageActor.MarkVisited(layer.id, tileIndex, date)
                         })
                     })
 
-                })
+                }
             });
 
 
@@ -379,7 +380,6 @@ export default class FeaturePipeline {
 
     private updateAllMetaTagging() {
         const self = this;
-        console.log("Reupdating all metatagging")
         this.perLayerHierarchy.forEach(hierarchy => {
             hierarchy.loadedTiles.forEach(src => {
                 self.applyMetaTags(src)

Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts

@@ -1,14 +1,16 @@
 import FilteredLayer from "../../../Models/FilteredLayer";
 import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
 import {UIEventSource} from "../../UIEventSource";
-import Loc from "../../../Models/Loc";
 import TileHierarchy from "./TileHierarchy";
 import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor";
 import {Tiles} from "../../../Models/TileRange";
 import {BBox} from "../../BBox";
 
 export default class TiledFromLocalStorageSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
-    public loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
+    public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
+    private readonly layer: FilteredLayer;
+    private readonly handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void;
+    private readonly undefinedTiles: Set<number>;
 
     public static GetFreshnesses(layerId: string): Map<number, Date> {
         const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layerId + "-"
@@ -29,14 +31,15 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
     constructor(layer: FilteredLayer,
                 handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void,
                 state: {
-                    locationControl: UIEventSource<Loc>
+                    currentBounds: UIEventSource<BBox>
-                    leafletMap: any
                 }) {
+        this.layer = layer;
+        this.handleFeatureSource = handleFeatureSource;
 
-        const undefinedTiles = new Set<number>()
+        
+        this.undefinedTiles = new Set<number>()
         const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-"
-        // @ts-ignore
+        const knownTiles: number[] = Object.keys(localStorage)
-        const indexes: number[] = Object.keys(localStorage)
             .filter(key => {
                 return key.startsWith(prefix) && !key.endsWith("-time") && !key.endsWith("-format");
             })
@@ -45,8 +48,8 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
             })
             .filter(i => !isNaN(i))
 
-        console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", indexes.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
+        console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", knownTiles.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
-        for (const index of indexes) {
+        for (const index of knownTiles) {
 
             const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + index;
             const version = localStorage.getItem(prefix + "-format")
@@ -55,78 +58,54 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
                 localStorage.removeItem(prefix)
                 localStorage.removeItem(prefix+"-time")
                 localStorage.removeItem(prefix+"-format")
-                undefinedTiles.add(index)
+              this.  undefinedTiles.add(index)
                 console.log("Dropped old format tile", prefix)
             }
         }
 
-        const zLevels = indexes.map(i => i % 100)
+        const self = this
-        const indexesSet = new Set(indexes)
+        state.currentBounds.map(bounds => {
-        const maxZoom = Math.max(...zLevels)
-        const minZoom = Math.min(...zLevels)
-        const self = this;
 
-        const neededTiles = state.locationControl.map(
+            if(bounds === undefined){
-            location => {
+                return;
-                if (!layer.isDisplayed.data) {
-                    // No need to download! - the layer is disabled
-                    return undefined;
-                }
-
-                if (location.zoom < layer.layerDef.minzoom) {
-                    // No need to download! - the layer is disabled
-                    return undefined;
-                }
-
-                // Yup, this is cheating to just get the bounds here
-                const bounds = state.leafletMap.data?.getBounds()
-                if (bounds === undefined) {
-                    // We'll retry later
-                    return undefined
-                }
-
-                const needed = []
-                for (let z = minZoom; z <= maxZoom; z++) {
-
-                    const tileRange = Tiles.TileRangeBetween(z, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
-                    const neededZ = Tiles.MapRange(tileRange, (x, y) => Tiles.tile_index(z, x, y))
-                        .filter(i => !self.loadedTiles.has(i) && !undefinedTiles.has(i) && indexesSet.has(i))
-                    needed.push(...neededZ)
-                }
-
-                if (needed.length === 0) {
-                    return undefined
-                }
-                return needed
             }
-            , [layer.isDisplayed, state.leafletMap]).stabilized(50);
+            for (const knownTile of knownTiles) {
-
+                
-        neededTiles.addCallbackAndRunD(neededIndexes => {
+                if(this.loadedTiles.has(knownTile)){
-            for (const neededIndex of neededIndexes) {
+                    continue;
-                // We load the features from localStorage
-                try {
-                    const key = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + neededIndex
-                    const data = localStorage.getItem(key)
-                    const features = JSON.parse(data)
-                    const src = {
-                        layer: layer,
-                        features: new UIEventSource<{ feature: any; freshness: Date }[]>(features),
-                        name: "FromLocalStorage(" + key + ")",
-                        tileIndex: neededIndex,
-                        bbox: BBox.fromTileIndex(neededIndex)
-                    }
-                    handleFeatureSource(src, neededIndex)
-                    self.loadedTiles.set(neededIndex, src)
-                } catch (e) {
-                    console.error("Could not load data tile from local storage due to", e)
-                    undefinedTiles.add(neededIndex)
                 }
+                if(this.undefinedTiles.has(knownTile)){
+                    continue;
+                }
+                
+                if(!bounds.overlapsWith(BBox.fromTileIndex(knownTile))){
+                    continue;
+                }
+                self.loadTile(knownTile)
             }
-
-
         })
 
     }
+    
+    private loadTile( neededIndex: number){
+        try {
+            const key = SaveTileToLocalStorageActor.storageKey + "-" + this.layer.layerDef.id + "-" + neededIndex
+            const data = localStorage.getItem(key)
+            const features = JSON.parse(data)
+            const src = {
+                layer: this.layer,
+                features: new UIEventSource<{ feature: any; freshness: Date }[]>(features),
+                name: "FromLocalStorage(" + key + ")",
+                tileIndex: neededIndex,
+                bbox: BBox.fromTileIndex(neededIndex)
+            }
+            this.handleFeatureSource(src, neededIndex)
+            this.loadedTiles.set(neededIndex, src)
+        } catch (e) {
+            console.error("Could not load data tile from local storage due to", e)
+            this.undefinedTiles.add(neededIndex)
+        }
+    }
 
 
 }

Models/Constants.ts

@@ -2,7 +2,7 @@ import {Utils} from "../Utils";
 
 export default class Constants {
 
-    public static vNumber = "0.10.4";
+    public static vNumber = "0.10.5";
     public static ImgurApiKey = '7070e7167f0a25a'
     public static readonly mapillary_client_token_v3 = 'TXhLaWthQ1d4RUg0czVxaTVoRjFJZzowNDczNjUzNmIyNTQyYzI2'
     public static readonly mapillary_client_token_v4 = "MLY|4441509239301885|b40ad2d3ea105435bd40c7e76993ae85"

Models/TileRange.ts

@@ -15,7 +15,7 @@ export class Tiles {
     public static MapRange<T>(tileRange: TileRange, f: (x: number, y: number) => T): T[] {
         const result: T[] = []
         const total = tileRange.total
-        if(total > 5000){
+        if(total > 100000){
             throw "Tilerange too big"
         }
         for (let x = tileRange.xstart; x <= tileRange.xend; x++) {

assets/themes/toerisme_vlaanderen/toerisme_vlaanderen.json

@@ -37,6 +37,7 @@
     {
       "builtin": [
         "bench",
+        "picnic_table",
         "waste_basket"
       ],
       "override": {