From 4f456e8a7f68ac1ffd8241636733409c82671fab Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Thu, 30 Sep 2021 04:13:23 +0200 Subject: [PATCH] Better tracking of cached data, only load data if needed --- Logic/Actors/GeoLocationHandler.ts | 3 + Logic/Actors/OverpassFeatureSource.ts | 110 +++------ .../Actors/SaveTileToLocalStorageActor.ts | 26 +- Logic/FeatureSource/FeaturePipeline.ts | 225 ++++++++++++------ .../FeatureSource/TileFreshnessCalculator.ts | 72 ++++++ .../TiledFeatureSource/OsmFeatureSource.ts | 12 +- .../TiledFeatureSource/TileHierarchyMerger.ts | 1 - .../TiledFromLocalStorageSource.ts | 38 +-- Logic/SimpleMetaTagger.ts | 9 +- Models/Constants.ts | 2 +- Models/TileRange.ts | 1 + test/TestAll.ts | 4 +- test/TileFreshnessCalculator.spec.ts | 31 +++ 13 files changed, 349 insertions(+), 185 deletions(-) create mode 100644 Logic/FeatureSource/TileFreshnessCalculator.ts create mode 100644 test/TileFreshnessCalculator.spec.ts diff --git a/Logic/Actors/GeoLocationHandler.ts b/Logic/Actors/GeoLocationHandler.ts index 3abf8e8e2..43ce441d7 100644 --- a/Logic/Actors/GeoLocationHandler.ts +++ b/Logic/Actors/GeoLocationHandler.ts @@ -207,6 +207,9 @@ export default class GeoLocationHandler extends VariableUiElement { }); const map = self._leafletMap.data; + if(map === undefined){ + return; + } const newMarker = L.marker(location.latlng, {icon: icon}); newMarker.addTo(map); diff --git a/Logic/Actors/OverpassFeatureSource.ts b/Logic/Actors/OverpassFeatureSource.ts index fa5de3eed..4f4f7a77f 100644 --- a/Logic/Actors/OverpassFeatureSource.ts +++ b/Logic/Actors/OverpassFeatureSource.ts @@ -1,5 +1,4 @@ import {UIEventSource} from "../UIEventSource"; -import Loc from "../../Models/Loc"; import {Or} from "../Tags/Or"; import {Overpass} from "../Osm/Overpass"; import FeatureSource from "../FeatureSource/FeatureSource"; @@ -9,6 +8,8 @@ import SimpleMetaTagger from "../SimpleMetaTagger"; import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig"; import RelationsTracker from "../Osm/RelationsTracker"; import {BBox} from "../BBox"; +import Loc from "../../Models/Loc"; +import LayerConfig from "../../Models/ThemeConfig/LayerConfig"; export default class OverpassFeatureSource implements FeatureSource { @@ -28,14 +29,7 @@ export default class OverpassFeatureSource implements FeatureSource { private readonly retries: UIEventSource = new UIEventSource(0); - /** - * The previous bounds for which the query has been run at the given zoom level - * - * Note that some layers only activate on a certain zoom level. - * If the map location changes, we check for each layer if it is loaded: - * we start checking the bounds at the first zoom level the layer might operate. If in bounds - no reload needed, otherwise we continue walking down - */ - private readonly _previousBounds: Map = new Map(); + private readonly state: { readonly locationControl: UIEventSource, readonly layoutToUse: LayoutConfig, @@ -44,11 +38,8 @@ export default class OverpassFeatureSource implements FeatureSource { readonly currentBounds: UIEventSource } private readonly _isActive: UIEventSource; - private _onUpdated?: (bbox: BBox, dataFreshness: Date) => void; + private readonly onBboxLoaded: (bbox: BBox, date: Date, layers: LayerConfig[]) => void; - /** - * The most important layer should go first, as that one gets first pick for the questions - */ constructor( state: { readonly locationControl: UIEventSource, @@ -60,68 +51,25 @@ export default class OverpassFeatureSource implements FeatureSource { }, options?: { isActive?: UIEventSource, - onUpdated?: (bbox: BBox, freshness: Date) => void, - relationTracker: RelationsTracker + relationTracker: RelationsTracker, + onBboxLoaded?: (bbox: BBox, date: Date, layers: LayerConfig[]) => void }) { this.state = state this._isActive = options.isActive; - this._onUpdated = options.onUpdated; + this.onBboxLoaded = options.onBboxLoaded this.relationsTracker = options.relationTracker - const location = state.locationControl const self = this; - - for (let i = 0; i < 25; i++) { - // This update removes all data on all layers -> erase the map on lower levels too - this._previousBounds.set(i, []); - } - - location.addCallback(() => { - self.update() - }); - state.currentBounds.addCallback(_ => { self.update() }) } - private GetFilter(interpreterUrl: string): Overpass { + private GetFilter(interpreterUrl: string, layersToDownload: LayerConfig[]): Overpass { let filters: TagsFilter[] = []; let extraScripts: string[] = []; - for (const layer of this.state.layoutToUse.layers) { - if (typeof (layer) === "string") { - throw "A layer was not expanded!" - } - if (this.state.locationControl.data.zoom < layer.minzoom) { - continue; - } - if (layer.doNotDownload) { - continue; - } - if (layer.source.geojsonSource !== undefined) { - // Not our responsibility to download this layer! - continue; - } - - - // Check if data for this layer has already been loaded - let previouslyLoaded = false; - for (let z = layer.minzoom; z < 25 && !previouslyLoaded; z++) { - const previousLoadedBounds = this._previousBounds.get(z); - if (previousLoadedBounds === undefined) { - continue; - } - for (const previousLoadedBound of previousLoadedBounds) { - previouslyLoaded = previouslyLoaded || this.state.currentBounds.data.isContainedIn(previousLoadedBound); - if (previouslyLoaded) { - break; - } - } - } - if (previouslyLoaded) { - continue; - } + for (const layer of layersToDownload) { if (layer.source.overpassScript !== undefined) { extraScripts.push(layer.source.overpassScript) } else { @@ -140,17 +88,17 @@ export default class OverpassFeatureSource implements FeatureSource { if (!this._isActive.data) { return; } - const self = this - this.updateAsync().then(bboxAndDate => { - if (bboxAndDate === undefined || self._onUpdated === undefined) { + const self = this; + this.updateAsync().then(bboxDate => { + if(bboxDate === undefined || self.onBboxLoaded === undefined){ return; } - const [bbox, date] = bboxAndDate - self._onUpdated(bbox, date); + const [bbox, date, layers] = bboxDate + self.onBboxLoaded(bbox, date, layers) }) } - private async updateAsync(): Promise<[BBox, Date]> { + private async updateAsync(): Promise<[BBox, Date, LayerConfig[]]> { if (this.runningQuery.data) { console.log("Still running a query, not updating"); return undefined; @@ -167,6 +115,26 @@ export default class OverpassFeatureSource implements FeatureSource { return undefined; } const self = this; + + + const layersToDownload = [] + for (const layer of this.state.layoutToUse.layers) { + + if (typeof (layer) === "string") { + throw "A layer was not expanded!" + } + if(this.state.locationControl.data.zoom < layer.minzoom){ + continue; + } + if (layer.doNotDownload) { + continue; + } + if (layer.source.geojsonSource !== undefined) { + // Not our responsibility to download this layer! + continue; + } + layersToDownload.push(layer) + } let data: any = undefined let date: Date = undefined @@ -175,7 +143,8 @@ export default class OverpassFeatureSource implements FeatureSource { do { try { - const overpass = this.GetFilter(overpassUrls[lastUsed]); + + const overpass = this.GetFilter(overpassUrls[lastUsed], layersToDownload); if (overpass === undefined) { return undefined; @@ -208,14 +177,11 @@ export default class OverpassFeatureSource implements FeatureSource { } } while (data === undefined); - const z = Math.floor(this.state.locationControl.data.zoom ?? 0); - self._previousBounds.get(z).push(bounds); self.retries.setData(0); - try { data.features.forEach(feature => SimpleMetaTagger.objectMetaInfo.applyMetaTagsOnFeature(feature, date)); self.features.setData(data.features.map(f => ({feature: f, freshness: date}))); - return [bounds, date]; + return [bounds, date, layersToDownload]; } catch (e) { console.error("Got the overpass response, but could not process it: ", e, e.stack) } finally { diff --git a/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts b/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts index 22e6a7ba8..2894d56b8 100644 --- a/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts +++ b/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts @@ -7,28 +7,30 @@ import {FeatureSourceForLayer} from "../FeatureSource"; export default class SaveTileToLocalStorageActor { public static readonly storageKey: string = "cached-features"; - public static readonly formatVersion : string = "1" - + public static readonly formatVersion: string = "1" + constructor(source: FeatureSourceForLayer, tileIndex: number) { source.features.addCallbackAndRunD(features => { const key = `${SaveTileToLocalStorageActor.storageKey}-${source.layer.layerDef.id}-${tileIndex}` - const now = new Date().getTime() - - if (features.length == 0) { - return; - } + const now = new Date() try { - localStorage.setItem(key, JSON.stringify(features)); - localStorage.setItem(key + "-time", JSON.stringify(now)) - localStorage.setItem(key+"-format", SaveTileToLocalStorageActor.formatVersion) + if (features.length > 0) { + localStorage.setItem(key, JSON.stringify(features)); + } + // We _still_ write the time to know that this tile is empty! + SaveTileToLocalStorageActor.MarkVisited(source.layer.layerDef.id, tileIndex, now) } catch (e) { console.warn("Could not save the features to local storage:", e) } }) - - } + public static MarkVisited(layerId: string, tileId: number, freshness: Date){ + const key = `${SaveTileToLocalStorageActor.storageKey}-${layerId}-${tileId}` + localStorage.setItem(key + "-time", JSON.stringify(freshness.getTime())) + localStorage.setItem(key + "-format", SaveTileToLocalStorageActor.formatVersion) + + } } \ No newline at end of file diff --git a/Logic/FeatureSource/FeaturePipeline.ts b/Logic/FeatureSource/FeaturePipeline.ts index 3db3a039d..5d5b3e166 100644 --- a/Logic/FeatureSource/FeaturePipeline.ts +++ b/Logic/FeatureSource/FeaturePipeline.ts @@ -25,6 +25,7 @@ import {BBox} from "../BBox"; import OsmFeatureSource from "./TiledFeatureSource/OsmFeatureSource"; import {OsmConnection} from "../Osm/OsmConnection"; import {Tiles} from "../../Models/TileRange"; +import TileFreshnessCalculator from "./TileFreshnessCalculator"; export default class FeaturePipeline { @@ -38,9 +39,27 @@ export default class FeaturePipeline { public readonly newDataLoadedSignal: UIEventSource = new UIEventSource(undefined) private readonly overpassUpdater: OverpassFeatureSource + private state: { + readonly filteredLayers: UIEventSource, + readonly locationControl: UIEventSource, + readonly selectedElement: UIEventSource, + readonly changes: Changes, + readonly layoutToUse: LayoutConfig, + readonly leafletMap: any, + readonly overpassUrl: UIEventSource; + readonly overpassTimeout: UIEventSource; + readonly overpassMaxZoom: UIEventSource; + readonly osmConnection: OsmConnection + readonly currentBounds: UIEventSource + }; private readonly relationTracker: RelationsTracker private readonly perLayerHierarchy: Map; + private readonly freshnesses = new Map(); + + private readonly oldestAllowedDate: Date = new Date(new Date().getTime() - 60 * 60 * 24 * 30 * 1000); + private readonly osmSourceZoomLevel = 14 + constructor( handleFeatureSource: (source: FeatureSourceForLayer & Tiled) => void, state: { @@ -48,7 +67,7 @@ export default class FeaturePipeline { readonly locationControl: UIEventSource, readonly selectedElement: UIEventSource, readonly changes: Changes, - readonly layoutToUse: LayoutConfig, + readonly layoutToUse: LayoutConfig, readonly leafletMap: any, readonly overpassUrl: UIEventSource; readonly overpassTimeout: UIEventSource; @@ -56,59 +75,15 @@ export default class FeaturePipeline { readonly osmConnection: OsmConnection readonly currentBounds: UIEventSource }) { + this.state = state; const self = this - - /** - * Maps tileid onto last download moment - */ - const tileFreshnesses = new UIEventSource>(new Map()) - const osmSourceZoomLevel = 14 + // milliseconds const useOsmApi = state.locationControl.map(l => l.zoom > (state.overpassMaxZoom.data ?? 12)) this.relationTracker = new RelationsTracker() + const neededTilesFromOsm = this.getNeededTilesFromOsm() - console.log("Tilefreshnesses are", tileFreshnesses.data) - const oldestAllowedDate = new Date(new Date().getTime() - (60 * 60 * 24 * 30 * 1000)); - const neededTilesFromOsm = state.currentBounds.map(bbox => { - if (bbox === undefined) { - return - } - const range = bbox.containingTileRange(osmSourceZoomLevel) - const tileIndexes = [] - if (range.total > 100) { - // Too much tiles! - return [] - } - Tiles.MapRange(range, (x, y) => { - const i = Tiles.tile_index(osmSourceZoomLevel, x, y); - if (tileFreshnesses.data.get(i) > oldestAllowedDate) { - console.debug("Skipping tile", osmSourceZoomLevel, x, y, "as a decently fresh one is available") - // The cached tiles contain decently fresh data - return; - } - tileIndexes.push(i) - }) - return tileIndexes - }, [tileFreshnesses]) - - const updater = new OverpassFeatureSource(state, - { - relationTracker: this.relationTracker, - isActive: useOsmApi.map(b => !b), - onUpdated: (bbox, freshness) => { - // This callback contains metadata of the overpass call - const range = bbox.containingTileRange(osmSourceZoomLevel) - Tiles.MapRange(range, (x, y) => { - tileFreshnesses.data.set(Tiles.tile_index(osmSourceZoomLevel, x, y), freshness) - }) - tileFreshnesses.ping(); - - } - }); - - this.overpassUpdater = updater; - this.timeout = updater.timeout this.sufficientlyZoomed = state.locationControl.map(location => { if (location?.zoom === undefined) { @@ -119,12 +94,6 @@ export default class FeaturePipeline { } ); - this.timeout = updater.timeout - - - // Register everything in the state' 'AllElements' - new RegisteringAllFromFeatureSourceActor(updater) - const perLayerHierarchy = new Map() this.perLayerHierarchy = perLayerHierarchy @@ -140,40 +109,32 @@ export default class FeaturePipeline { handleFeatureSource(srcFiltered) self.somethingLoaded.setData(true) + self.freshnesses.get(src.layer.layerDef.id).addTileLoad(src.tileIndex, new Date()) }; - function addToHierarchy(src: FeatureSource & Tiled, layerId: string) { - perLayerHierarchy.get(layerId).registerTile(src) - } - for (const filteredLayer of state.filteredLayers.data) { - const hierarchy = new TileHierarchyMerger(filteredLayer, (tile, _) => patchedHandleFeatureSource(tile)) const id = filteredLayer.layerDef.id - perLayerHierarchy.set(id, hierarchy) const source = filteredLayer.layerDef.source + const hierarchy = new TileHierarchyMerger(filteredLayer, (tile, _) => patchedHandleFeatureSource(tile)) + perLayerHierarchy.set(id, hierarchy) + + this.freshnesses.set(id, new TileFreshnessCalculator()) + if (source.geojsonSource === undefined) { // This is an OSM layer // We load the cached values and register them // Getting data from upstream happens a bit lower - const localStorage = new TiledFromLocalStorageSource(filteredLayer, + new TiledFromLocalStorageSource(filteredLayer, (src) => { new RegisteringAllFromFeatureSourceActor(src) hierarchy.registerTile(src); src.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(src)) }, state) - localStorage.tileFreshness.forEach((value, key) => { - if (tileFreshnesses.data.has(key)) { - const previous = tileFreshnesses.data.get(key) - if (value < previous) { - tileFreshnesses.data.set(key, value) - } - } else { - tileFreshnesses.data.set(key, value) - } - tileFreshnesses.ping() + TiledFromLocalStorageSource.GetFreshnesses(id).forEach((value, key) => { + self.freshnesses.get(id).addTileLoad(key, value) }) continue @@ -189,7 +150,7 @@ export default class FeaturePipeline { dontEnforceMinZoom: true, registerTile: (tile) => { new RegisteringAllFromFeatureSourceActor(tile) - addToHierarchy(tile, id) + perLayerHierarchy.get(id).registerTile(tile) tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile)) } }) @@ -198,7 +159,7 @@ export default class FeaturePipeline { filteredLayer, tile => { new RegisteringAllFromFeatureSourceActor(tile) - addToHierarchy(tile, id) + perLayerHierarchy.get(id).registerTile(tile) tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile)) }, state @@ -213,14 +174,22 @@ export default class FeaturePipeline { handleTile: tile => { new RegisteringAllFromFeatureSourceActor(tile) new SaveTileToLocalStorageActor(tile, tile.tileIndex) - addToHierarchy(tile, tile.layer.layerDef.id), - tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile)) + perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile) + tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile)) }, - state: state + state: state, + markTileVisited: (tileId) => + state.filteredLayers.data.forEach(flayer => { + SaveTileToLocalStorageActor.MarkVisited(flayer.layerDef.id, tileId, new Date()) + }) }) + const updater = this.initOverpassUpdater(state, useOsmApi) + this.overpassUpdater = updater; + this.timeout = updater.timeout + // Actually load data from the overpass source new PerLayerFeatureSourceSplitter(state.filteredLayers, (source) => TiledFeatureSource.createHierarchy(source, { @@ -232,7 +201,7 @@ export default class FeaturePipeline { registerTile: (tile) => { // We save the tile data for the given layer to local storage new SaveTileToLocalStorageActor(tile, tile.tileIndex) - addToHierarchy(new RememberingSource(tile), source.layer.layerDef.id); + perLayerHierarchy.get(source.layer.layerDef.id).registerTile(new RememberingSource(tile)) tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile)) } @@ -247,7 +216,7 @@ export default class FeaturePipeline { new PerLayerFeatureSourceSplitter(state.filteredLayers, (perLayer) => { // We don't bother to split them over tiles as it'll contain little features by default, so we simply add them like this - addToHierarchy(perLayer, perLayer.layer.layerDef.id) + perLayerHierarchy.get(perLayer.layer.layerDef.id).registerTile(perLayer) // AT last, we always apply the metatags whenever possible perLayer.features.addCallbackAndRunD(_ => self.applyMetaTags(perLayer)) perLayer.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(perLayer)) @@ -270,6 +239,106 @@ export default class FeaturePipeline { } + private freshnessForVisibleLayers(z: number, x: number, y: number): Date { + let oldestDate = undefined; + for (const flayer of this.state.filteredLayers.data) { + if (!flayer.isDisplayed.data) { + continue + } + if (this.state.locationControl.data.zoom < flayer.layerDef.minzoom) { + continue; + } + const freshness = this.freshnesses.get(flayer.layerDef.id).freshnessFor(z, x, y) + if (freshness === undefined) { + // SOmething is undefined --> we return undefined as we have to download + return undefined + } + if (oldestDate === undefined || oldestDate > freshness) { + oldestDate = freshness + } + } + return oldestDate + } + + private getNeededTilesFromOsm(): UIEventSource { + const self = this + return this.state.currentBounds.map(bbox => { + if (bbox === undefined) { + return + } + const osmSourceZoomLevel = self.osmSourceZoomLevel + const range = bbox.containingTileRange(osmSourceZoomLevel) + const tileIndexes = [] + if (range.total > 100) { + // Too much tiles! + return [] + } + Tiles.MapRange(range, (x, y) => { + const i = Tiles.tile_index(osmSourceZoomLevel, x, y); + const oldestDate = self.freshnessForVisibleLayers(osmSourceZoomLevel, x, y) + if (oldestDate !== undefined && oldestDate > this.oldestAllowedDate) { + console.debug("Skipping tile", osmSourceZoomLevel, x, y, "as a decently fresh one is available") + // The cached tiles contain decently fresh data + return; + } + tileIndexes.push(i) + }) + return tileIndexes + }) + } + + private initOverpassUpdater(state: { + layoutToUse: LayoutConfig, + currentBounds: UIEventSource, + locationControl: UIEventSource, + readonly overpassUrl: UIEventSource; + readonly overpassTimeout: UIEventSource; + readonly overpassMaxZoom: UIEventSource, + }, useOsmApi: UIEventSource): OverpassFeatureSource { + const minzoom = Math.min(...state.layoutToUse.layers.map(layer => layer.minzoom)) + const allUpToDateAndZoomSufficient = state.currentBounds.map(bbox => { + if (bbox === undefined) { + return true + } + let zoom = state.locationControl.data.zoom + if (zoom < minzoom) { + return true; + } + if (zoom > 16) { + zoom = 16 + } + if (zoom < 8) { + zoom = zoom + 2 + } + const range = bbox.containingTileRange(zoom) + const self = this; + const allFreshnesses = Tiles.MapRange(range, (x, y) => self.freshnessForVisibleLayers(zoom, x, y)) + return !allFreshnesses.some(freshness => freshness === undefined || freshness < this.oldestAllowedDate) + + }, [state.locationControl]) + + allUpToDateAndZoomSufficient.addCallbackAndRunD(allUpToDate => console.log("All up to data is: ", allUpToDate)) + const self = this; + const updater = new OverpassFeatureSource(state, + { + relationTracker: this.relationTracker, + isActive: useOsmApi.map(b => !b && !allUpToDateAndZoomSufficient.data, [allUpToDateAndZoomSufficient]), + onBboxLoaded: ((bbox, date, downloadedLayers) => { + Tiles.MapRange(bbox.containingTileRange(self.osmSourceZoomLevel), (x, y) => { + downloadedLayers.forEach(layer => { + SaveTileToLocalStorageActor.MarkVisited(layer.id, Tiles.tile_index(this.osmSourceZoomLevel, x, y), date) + }) + }) + + }) + }); + + + // Register everything in the state' 'AllElements' + new RegisteringAllFromFeatureSourceActor(updater) + return updater; + } + private applyMetaTags(src: FeatureSourceForLayer) { const self = this console.debug("Applying metatagging onto ", src.name) diff --git a/Logic/FeatureSource/TileFreshnessCalculator.ts b/Logic/FeatureSource/TileFreshnessCalculator.ts new file mode 100644 index 000000000..6b397f540 --- /dev/null +++ b/Logic/FeatureSource/TileFreshnessCalculator.ts @@ -0,0 +1,72 @@ +import {Tiles} from "../../Models/TileRange"; + +export default class TileFreshnessCalculator { + + /** + * All the freshnesses per tile index + * @private + */ + private readonly freshnesses = new Map(); + + /** + * Marks that some data got loaded for this layer + * @param tileId + * @param freshness + */ + public addTileLoad(tileId: number, freshness: Date){ + const existingFreshness = this.freshnessFor(...Tiles.tile_from_index(tileId)) + if(existingFreshness >= freshness){ + return; + } + this.freshnesses.set(tileId, freshness) + + + // Do we have freshness for the neighbouring tiles? If so, we can mark the tile above as loaded too! + let [z, x, y] = Tiles.tile_from_index(tileId) + if(z === 0){ + return; + } + x = x - (x % 2) // Make the tiles always even + y = y - (y % 2) + + const ul = this.freshnessFor(z, x, y)?.getTime() + if(ul === undefined){ + return + } + const ur = this.freshnessFor(z, x + 1, y)?.getTime() + if(ur === undefined){ + return + } + const ll = this.freshnessFor(z, x, y + 1)?.getTime() + if(ll === undefined){ + return + } + const lr = this.freshnessFor(z, x + 1, y + 1)?.getTime() + if(lr === undefined){ + return + } + + const leastFresh = Math.min(ul, ur, ll, lr) + const date = new Date() + date.setTime(leastFresh) + this.addTileLoad( + Tiles.tile_index(z - 1, Math.floor(x / 2), Math.floor(y / 2)), + date + ) + + } + + public freshnessFor(z: number, x: number, y:number): Date { + if(z < 0){ + return undefined + } + const tileId = Tiles.tile_index(z, x, y) + if(this.freshnesses.has(tileId)) { + return this.freshnesses.get(tileId) + } + // recurse up + return this.freshnessFor(z - 1, Math.floor(x /2), Math.floor(y / 2)) + + } + +} \ No newline at end of file diff --git a/Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts b/Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts index a7485facf..5641d5b98 100644 --- a/Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts +++ b/Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts @@ -22,7 +22,8 @@ export default class OsmFeatureSource { neededTiles: UIEventSource, state: { readonly osmConnection: OsmConnection; - }; + }, + markTileVisited?: (tileId: number) => void }; private readonly downloadedTiles = new Set() @@ -33,7 +34,8 @@ export default class OsmFeatureSource { state: { readonly filteredLayers: UIEventSource; readonly osmConnection: OsmConnection; - }; + }, + markTileVisited?: (tileId: number) => void }) { this.options = options; this._backend = options.state.osmConnection._oauth_config.url; @@ -84,13 +86,17 @@ export default class OsmFeatureSource { flatProperties: true }); console.log("Tile geojson:", z, x, y, "is", geojson) + const index = Tiles.tile_index(z, x, y); new PerLayerFeatureSourceSplitter(this.filteredLayers, this.handleTile, new StaticFeatureSource(geojson.features, false), { - tileIndex: Tiles.tile_index(z, x, y) + tileIndex:index } ); + if(this.options.markTileVisited){ + this.options.markTileVisited(index) + } } catch (e) { console.error("Weird error: ", e) } diff --git a/Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts b/Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts index 844754854..6fd3dae65 100644 --- a/Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts +++ b/Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts @@ -2,7 +2,6 @@ import TileHierarchy from "./TileHierarchy"; import {UIEventSource} from "../../UIEventSource"; import FeatureSource, {FeatureSourceForLayer, IndexedFeatureSource, Tiled} from "../FeatureSource"; import FilteredLayer from "../../../Models/FilteredLayer"; -import {Utils} from "../../../Utils"; import FeatureSourceMerger from "../Sources/FeatureSourceMerger"; import {Tiles} from "../../../Models/TileRange"; import {BBox} from "../../BBox"; diff --git a/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts b/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts index 016537968..6f815f48d 100644 --- a/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts +++ b/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts @@ -3,15 +3,29 @@ import {FeatureSourceForLayer, Tiled} from "../FeatureSource"; import {UIEventSource} from "../../UIEventSource"; import Loc from "../../../Models/Loc"; import TileHierarchy from "./TileHierarchy"; -import {Utils} from "../../../Utils"; import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor"; import {Tiles} from "../../../Models/TileRange"; import {BBox} from "../../BBox"; export default class TiledFromLocalStorageSource implements TileHierarchy { public loadedTiles: Map = new Map(); -public tileFreshness : Map = new Map() - + + public static GetFreshnesses(layerId: string): Map { + const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layerId + "-" + const freshnesses = new Map() + for (const key of Object.keys(localStorage)) { + if(!(key.startsWith(prefix) && key.endsWith("-time"))){ + continue + } + const index = Number(key.substring(prefix.length, key.length - "-time".length)) + const time = Number(localStorage.getItem(key)) + const freshness = new Date() + freshness.setTime(time) + freshnesses.set(index, freshness) + } + return freshnesses + } + constructor(layer: FilteredLayer, handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void, state: { @@ -33,21 +47,17 @@ public tileFreshness : Map = new Map() console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", indexes.map(i => Tiles.tile_from_index(i).join("/")).join(", ")) for (const index of indexes) { - - const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" +index; - const version = localStorage.getItem(prefix+"-format") - if(version === undefined || version !== SaveTileToLocalStorageActor.formatVersion){ + + const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + index; + const version = localStorage.getItem(prefix + "-format") + if (version === undefined || version !== SaveTileToLocalStorageActor.formatVersion) { // Invalid version! Remove this tile from local storage localStorage.removeItem(prefix) + localStorage.removeItem(prefix+"-time") + localStorage.removeItem(prefix+"-format") undefinedTiles.add(index) console.log("Dropped old format tile", prefix) - continue } - - const data = Number(localStorage.getItem(prefix+"-time")) - const freshness = new Date() - freshness.setTime(data) - this.tileFreshness.set(index, freshness) } const zLevels = indexes.map(i => i % 100) @@ -91,8 +101,6 @@ public tileFreshness : Map = new Map() } , [layer.isDisplayed, state.leafletMap]).stabilized(50); - neededTiles.addCallbackAndRun(t => console.debug("Tiles to load from localstorage:", t)) - neededTiles.addCallbackAndRunD(neededIndexes => { for (const neededIndex of neededIndexes) { // We load the features from localStorage diff --git a/Logic/SimpleMetaTagger.ts b/Logic/SimpleMetaTagger.ts index 4b047fc7d..360bb54e5 100644 --- a/Logic/SimpleMetaTagger.ts +++ b/Logic/SimpleMetaTagger.ts @@ -97,7 +97,7 @@ export default class SimpleMetaTagger { continue; } for (const unit of units) { - if(unit === undefined){ + if (unit === undefined) { continue } if (unit.appliesToKeys === undefined) { @@ -108,7 +108,12 @@ export default class SimpleMetaTagger { continue; } const value = feature.properties[key] - const [, denomination] = unit.findDenomination(value) + const denom = unit.findDenomination(value) + if (denom === undefined) { + // no valid value found + break; + } + const [, denomination] = denom; let canonical = denomination?.canonicalValue(value) ?? undefined; if (canonical === value) { break; diff --git a/Models/Constants.ts b/Models/Constants.ts index 4e588a057..855f9dd2b 100644 --- a/Models/Constants.ts +++ b/Models/Constants.ts @@ -2,7 +2,7 @@ import {Utils} from "../Utils"; export default class Constants { - public static vNumber = "0.10.0-rc1"; + public static vNumber = "0.10.0-rc2"; public static ImgurApiKey = '7070e7167f0a25a' public static readonly mapillary_client_token_v3 = 'TXhLaWthQ1d4RUg0czVxaTVoRjFJZzowNDczNjUzNmIyNTQyYzI2' public static readonly mapillary_client_token_v4 = "MLY|4441509239301885|b40ad2d3ea105435bd40c7e76993ae85" diff --git a/Models/TileRange.ts b/Models/TileRange.ts index b61f192a0..da30e498c 100644 --- a/Models/TileRange.ts +++ b/Models/TileRange.ts @@ -107,4 +107,5 @@ export class Tiles { } } + } \ No newline at end of file diff --git a/test/TestAll.ts b/test/TestAll.ts index 0efa98dd4..6123681fb 100644 --- a/test/TestAll.ts +++ b/test/TestAll.ts @@ -9,6 +9,7 @@ import UnitsSpec from "./Units.spec"; import RelationSplitHandlerSpec from "./RelationSplitHandler.spec"; import SplitActionSpec from "./SplitAction.spec"; import {Utils} from "../Utils"; +import TileFreshnessCalculatorSpec from "./TileFreshnessCalculator.spec"; ScriptUtils.fixUtils() @@ -21,7 +22,8 @@ const allTests = [ new UtilsSpec(), new UnitsSpec(), new RelationSplitHandlerSpec(), - new SplitActionSpec() + new SplitActionSpec(), + new TileFreshnessCalculatorSpec() ] Utils.externalDownloadFunction = async (url) => { diff --git a/test/TileFreshnessCalculator.spec.ts b/test/TileFreshnessCalculator.spec.ts new file mode 100644 index 000000000..6305b9a5d --- /dev/null +++ b/test/TileFreshnessCalculator.spec.ts @@ -0,0 +1,31 @@ +import T from "./TestHelper"; +import TileFreshnessCalculator from "../Logic/FeatureSource/TileFreshnessCalculator"; +import {Tiles} from "../Models/TileRange"; +import {equal} from "assert"; + +export default class TileFreshnessCalculatorSpec extends T { + + constructor() { + super("TileFreshnessCalculatorSpec", [ + [ + "TileFresnessTests", + () => { + const calc = new TileFreshnessCalculator(); + // 19/266407/175535 + const date = new Date() + date.setTime(42) + calc.addTileLoad(Tiles.tile_index(19, 266406, 175534), date) + equal(42, calc.freshnessFor(19, 266406, 175534).getTime()) + equal(42, calc.freshnessFor(20, 266406 * 2, 175534 * 2 + 1).getTime()) + equal(undefined, calc.freshnessFor(19, 266406, 175535)) + equal(undefined, calc.freshnessFor(18, 266406 / 2, 175534 / 2)) + calc.addTileLoad(Tiles.tile_index(19, 266406, 175534+1), date) + calc.addTileLoad(Tiles.tile_index(19, 266406+1, 175534), date) + calc.addTileLoad(Tiles.tile_index(19, 266406+1, 175534+1), date) + equal(42, calc.freshnessFor(18, 266406 / 2, 175534 / 2).getTime()) + } + ] + ]) + } + +} \ No newline at end of file