From 178cefceb41c854196356bad856d70eb728de2d7 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Mon, 11 Oct 2021 22:30:22 +0200 Subject: [PATCH] Fix bounds and loading with overpass --- Logic/Actors/OverpassFeatureSource.ts | 23 ++-- .../Actors/SaveTileToLocalStorageActor.ts | 4 +- Logic/FeatureSource/FeaturePipeline.ts | 30 ++--- .../TiledFromLocalStorageSource.ts | 119 ++++++++---------- Models/Constants.ts | 2 +- Models/TileRange.ts | 2 +- .../toerisme_vlaanderen.json | 1 + 7 files changed, 78 insertions(+), 103 deletions(-) diff --git a/Logic/Actors/OverpassFeatureSource.ts b/Logic/Actors/OverpassFeatureSource.ts index e27e0665d9..62cf799988 100644 --- a/Logic/Actors/OverpassFeatureSource.ts +++ b/Logic/Actors/OverpassFeatureSource.ts @@ -38,8 +38,7 @@ export default class OverpassFeatureSource implements FeatureSource { readonly currentBounds: UIEventSource } private readonly _isActive: UIEventSource; - private readonly onBboxLoaded: (bbox: BBox, date: Date, layers: LayerConfig[]) => void; - private readonly padToTiles : number + private readonly onBboxLoaded: (bbox: BBox, date: Date, layers: LayerConfig[], zoomlevel: number) => void; constructor( state: { readonly locationControl: UIEventSource, @@ -50,20 +49,19 @@ export default class OverpassFeatureSource implements FeatureSource { readonly currentBounds: UIEventSource }, options: { - padToTiles: number, + padToTiles: UIEventSource, isActive?: UIEventSource, relationTracker: RelationsTracker, - onBboxLoaded?: (bbox: BBox, date: Date, layers: LayerConfig[]) => void + onBboxLoaded?: (bbox: BBox, date: Date, layers: LayerConfig[], zoomlevel: number) => void }) { this.state = state this._isActive = options.isActive; - this.padToTiles = options.padToTiles; this.onBboxLoaded = options.onBboxLoaded this.relationsTracker = options.relationTracker const self = this; state.currentBounds.addCallback(_ => { - self.update() + self.update(options.padToTiles.data) }) } @@ -86,21 +84,21 @@ export default class OverpassFeatureSource implements FeatureSource { return new Overpass(new Or(filters), extraScripts, interpreterUrl, this.state.overpassTimeout, this.relationsTracker); } - private update() { + private update(paddedZoomLevel: number) { if (!this._isActive.data) { return; } const self = this; - this.updateAsync().then(bboxDate => { + this.updateAsync(paddedZoomLevel).then(bboxDate => { if(bboxDate === undefined || self.onBboxLoaded === undefined){ return; } const [bbox, date, layers] = bboxDate - self.onBboxLoaded(bbox, date, layers) + self.onBboxLoaded(bbox, date, layers, paddedZoomLevel) }) } - private async updateAsync(): Promise<[BBox, Date, LayerConfig[]]> { + private async updateAsync(padToZoomLevel: number): Promise<[BBox, Date, LayerConfig[]]> { if (this.runningQuery.data) { console.log("Still running a query, not updating"); return undefined; @@ -111,14 +109,11 @@ export default class OverpassFeatureSource implements FeatureSource { return undefined; } - const bounds = this.state.currentBounds.data?.pad(this.state.layoutToUse.widenFactor)?.expandToTileBounds(this.padToTiles); + const bounds = this.state.currentBounds.data?.pad(this.state.layoutToUse.widenFactor)?.expandToTileBounds(padToZoomLevel); if (bounds === undefined) { return undefined; } - console.log("Current bounds are", this.state.currentBounds.data," padded with",this.state.layoutToUse.widenFactor+":", - this.state.currentBounds.data.pad(this.state.layoutToUse.widenFactor), - "Tileexpanded: ",this.state.currentBounds.data?.pad(this.state.layoutToUse.widenFactor)?.expandToTileBounds(this.padToTiles) ) const self = this; diff --git a/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts b/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts index 2894d56b82..96e9938aac 100644 --- a/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts +++ b/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts @@ -7,7 +7,7 @@ import {FeatureSourceForLayer} from "../FeatureSource"; export default class SaveTileToLocalStorageActor { public static readonly storageKey: string = "cached-features"; - public static readonly formatVersion: string = "1" + public static readonly formatVersion: string = "2" constructor(source: FeatureSourceForLayer, tileIndex: number) { source.features.addCallbackAndRunD(features => { @@ -31,6 +31,6 @@ export default class SaveTileToLocalStorageActor { const key = `${SaveTileToLocalStorageActor.storageKey}-${layerId}-${tileId}` localStorage.setItem(key + "-time", JSON.stringify(freshness.getTime())) localStorage.setItem(key + "-format", SaveTileToLocalStorageActor.formatVersion) - + console.log("Marked ", key, "as visited") } } \ No newline at end of file diff --git a/Logic/FeatureSource/FeaturePipeline.ts b/Logic/FeatureSource/FeaturePipeline.ts index ea33827f60..6df7b87f4f 100644 --- a/Logic/FeatureSource/FeaturePipeline.ts +++ b/Logic/FeatureSource/FeaturePipeline.ts @@ -155,7 +155,7 @@ export default class FeaturePipeline { tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile)) } }) - }else{ + } else { new RegisteringAllFromFeatureSourceActor(src) perLayerHierarchy.get(id).registerTile(src) src.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(src)) @@ -200,7 +200,7 @@ export default class FeaturePipeline { new PerLayerFeatureSourceSplitter(state.filteredLayers, (source) => TiledFeatureSource.createHierarchy(source, { layer: source.layer, - minZoomLevel: this.osmSourceZoomLevel, + minZoomLevel: source.layer.layerDef.minzoom, dontEnforceMinZoom: true, maxFeatureCount: state.layoutToUse.clustering.minNeededElements, maxZoomLevel: state.layoutToUse.clustering.maxZoom, @@ -233,7 +233,7 @@ export default class FeaturePipeline { // Whenever fresh data comes in, we need to update the metatagging - self.newDataLoadedSignal.stabilized(1000).addCallback(src => { + self.newDataLoadedSignal.stabilized(1000).addCallback(_ => { self.updateAllMetaTagging() }) @@ -270,15 +270,15 @@ export default class FeaturePipeline { const self = this return this.state.currentBounds.map(bbox => { if (bbox === undefined) { - return + return undefined } if (!isSufficientlyZoomed.data) { - return; + return undefined; } const osmSourceZoomLevel = self.osmSourceZoomLevel const range = bbox.containingTileRange(osmSourceZoomLevel) const tileIndexes = [] - if (range.total > 100) { + if (range.total >= 100) { // Too much tiles! return [] } @@ -288,7 +288,7 @@ export default class FeaturePipeline { if (oldestDate !== undefined && oldestDate > this.oldestAllowedDate) { console.debug("Skipping tile", osmSourceZoomLevel, x, y, "as a decently fresh one is available") // The cached tiles contain decently fresh data - return; + return undefined; } tileIndexes.push(i) }) @@ -321,29 +321,30 @@ export default class FeaturePipeline { } const range = bbox.containingTileRange(zoom) - if (range.total > 100) { + if (range.total >= 5000) { return false } const self = this; const allFreshnesses = Tiles.MapRange(range, (x, y) => self.freshnessForVisibleLayers(zoom, x, y)) return allFreshnesses.some(freshness => freshness === undefined || freshness < this.oldestAllowedDate) - }, [state.locationControl]) const self = this; const updater = new OverpassFeatureSource(state, { - padToTiles: this.osmSourceZoomLevel, + padToTiles: state.locationControl.map(l => Math.min(15, l.zoom + 1)), relationTracker: this.relationTracker, isActive: useOsmApi.map(b => !b && overpassIsActive.data, [overpassIsActive]), - onBboxLoaded: ((bbox, date, downloadedLayers) => { - Tiles.MapRange(bbox.containingTileRange(self.osmSourceZoomLevel), (x, y) => { + onBboxLoaded: (bbox, date, downloadedLayers, paddedToZoomLevel) => { + Tiles.MapRange(bbox.containingTileRange(paddedToZoomLevel), (x, y) => { + const tileIndex = Tiles.tile_index(paddedToZoomLevel, x, y) downloadedLayers.forEach(layer => { - SaveTileToLocalStorageActor.MarkVisited(layer.id, Tiles.tile_index(this.osmSourceZoomLevel, x, y), date) + self.freshnesses.get(layer.id).addTileLoad(tileIndex, date) + SaveTileToLocalStorageActor.MarkVisited(layer.id, tileIndex, date) }) }) - }) + } }); @@ -379,7 +380,6 @@ export default class FeaturePipeline { private updateAllMetaTagging() { const self = this; - console.log("Reupdating all metatagging") this.perLayerHierarchy.forEach(hierarchy => { hierarchy.loadedTiles.forEach(src => { self.applyMetaTags(src) diff --git a/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts b/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts index 6f815f48d5..a2cd50be08 100644 --- a/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts +++ b/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts @@ -1,14 +1,16 @@ import FilteredLayer from "../../../Models/FilteredLayer"; import {FeatureSourceForLayer, Tiled} from "../FeatureSource"; import {UIEventSource} from "../../UIEventSource"; -import Loc from "../../../Models/Loc"; import TileHierarchy from "./TileHierarchy"; import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor"; import {Tiles} from "../../../Models/TileRange"; import {BBox} from "../../BBox"; export default class TiledFromLocalStorageSource implements TileHierarchy { - public loadedTiles: Map = new Map(); + public readonly loadedTiles: Map = new Map(); + private readonly layer: FilteredLayer; + private readonly handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void; + private readonly undefinedTiles: Set; public static GetFreshnesses(layerId: string): Map { const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layerId + "-" @@ -29,14 +31,15 @@ export default class TiledFromLocalStorageSource implements TileHierarchy void, state: { - locationControl: UIEventSource - leafletMap: any + currentBounds: UIEventSource }) { + this.layer = layer; + this.handleFeatureSource = handleFeatureSource; - const undefinedTiles = new Set() + + this.undefinedTiles = new Set() const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" - // @ts-ignore - const indexes: number[] = Object.keys(localStorage) + const knownTiles: number[] = Object.keys(localStorage) .filter(key => { return key.startsWith(prefix) && !key.endsWith("-time") && !key.endsWith("-format"); }) @@ -45,8 +48,8 @@ export default class TiledFromLocalStorageSource implements TileHierarchy !isNaN(i)) - console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", indexes.map(i => Tiles.tile_from_index(i).join("/")).join(", ")) - for (const index of indexes) { + console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", knownTiles.map(i => Tiles.tile_from_index(i).join("/")).join(", ")) + for (const index of knownTiles) { const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + index; const version = localStorage.getItem(prefix + "-format") @@ -55,78 +58,54 @@ export default class TiledFromLocalStorageSource implements TileHierarchy i % 100) - const indexesSet = new Set(indexes) - const maxZoom = Math.max(...zLevels) - const minZoom = Math.min(...zLevels) - const self = this; + const self = this + state.currentBounds.map(bounds => { - const neededTiles = state.locationControl.map( - location => { - if (!layer.isDisplayed.data) { - // No need to download! - the layer is disabled - return undefined; - } - - if (location.zoom < layer.layerDef.minzoom) { - // No need to download! - the layer is disabled - return undefined; - } - - // Yup, this is cheating to just get the bounds here - const bounds = state.leafletMap.data?.getBounds() - if (bounds === undefined) { - // We'll retry later - return undefined - } - - const needed = [] - for (let z = minZoom; z <= maxZoom; z++) { - - const tileRange = Tiles.TileRangeBetween(z, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest()) - const neededZ = Tiles.MapRange(tileRange, (x, y) => Tiles.tile_index(z, x, y)) - .filter(i => !self.loadedTiles.has(i) && !undefinedTiles.has(i) && indexesSet.has(i)) - needed.push(...neededZ) - } - - if (needed.length === 0) { - return undefined - } - return needed + if(bounds === undefined){ + return; } - , [layer.isDisplayed, state.leafletMap]).stabilized(50); - - neededTiles.addCallbackAndRunD(neededIndexes => { - for (const neededIndex of neededIndexes) { - // We load the features from localStorage - try { - const key = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + neededIndex - const data = localStorage.getItem(key) - const features = JSON.parse(data) - const src = { - layer: layer, - features: new UIEventSource<{ feature: any; freshness: Date }[]>(features), - name: "FromLocalStorage(" + key + ")", - tileIndex: neededIndex, - bbox: BBox.fromTileIndex(neededIndex) - } - handleFeatureSource(src, neededIndex) - self.loadedTiles.set(neededIndex, src) - } catch (e) { - console.error("Could not load data tile from local storage due to", e) - undefinedTiles.add(neededIndex) + for (const knownTile of knownTiles) { + + if(this.loadedTiles.has(knownTile)){ + continue; } + if(this.undefinedTiles.has(knownTile)){ + continue; + } + + if(!bounds.overlapsWith(BBox.fromTileIndex(knownTile))){ + continue; + } + self.loadTile(knownTile) } - - }) } + + private loadTile( neededIndex: number){ + try { + const key = SaveTileToLocalStorageActor.storageKey + "-" + this.layer.layerDef.id + "-" + neededIndex + const data = localStorage.getItem(key) + const features = JSON.parse(data) + const src = { + layer: this.layer, + features: new UIEventSource<{ feature: any; freshness: Date }[]>(features), + name: "FromLocalStorage(" + key + ")", + tileIndex: neededIndex, + bbox: BBox.fromTileIndex(neededIndex) + } + this.handleFeatureSource(src, neededIndex) + this.loadedTiles.set(neededIndex, src) + } catch (e) { + console.error("Could not load data tile from local storage due to", e) + this.undefinedTiles.add(neededIndex) + } + } } \ No newline at end of file diff --git a/Models/Constants.ts b/Models/Constants.ts index 2b266f4ad5..c79773971a 100644 --- a/Models/Constants.ts +++ b/Models/Constants.ts @@ -2,7 +2,7 @@ import {Utils} from "../Utils"; export default class Constants { - public static vNumber = "0.10.4"; + public static vNumber = "0.10.5"; public static ImgurApiKey = '7070e7167f0a25a' public static readonly mapillary_client_token_v3 = 'TXhLaWthQ1d4RUg0czVxaTVoRjFJZzowNDczNjUzNmIyNTQyYzI2' public static readonly mapillary_client_token_v4 = "MLY|4441509239301885|b40ad2d3ea105435bd40c7e76993ae85" diff --git a/Models/TileRange.ts b/Models/TileRange.ts index 7a542a2135..cccfe19643 100644 --- a/Models/TileRange.ts +++ b/Models/TileRange.ts @@ -15,7 +15,7 @@ export class Tiles { public static MapRange(tileRange: TileRange, f: (x: number, y: number) => T): T[] { const result: T[] = [] const total = tileRange.total - if(total > 5000){ + if(total > 100000){ throw "Tilerange too big" } for (let x = tileRange.xstart; x <= tileRange.xend; x++) { diff --git a/assets/themes/toerisme_vlaanderen/toerisme_vlaanderen.json b/assets/themes/toerisme_vlaanderen/toerisme_vlaanderen.json index 7a401b0ab8..489c36ed7d 100644 --- a/assets/themes/toerisme_vlaanderen/toerisme_vlaanderen.json +++ b/assets/themes/toerisme_vlaanderen/toerisme_vlaanderen.json @@ -37,6 +37,7 @@ { "builtin": [ "bench", + "picnic_table", "waste_basket" ], "override": {