From 8fa7de661ea3d37724992c27910286be86288b07 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Mon, 15 Nov 2021 11:51:32 +0100 Subject: [PATCH 1/2] WIP: use indexedDB as datastore for geotiles --- .../Actors/SaveTileToLocalStorageActor.ts | 65 +++++++++---------- Logic/FeatureSource/FeaturePipeline.ts | 23 ++++--- .../TiledFromLocalStorageSource.ts | 17 ----- Logic/State/MapState.ts | 3 +- Logic/Web/IdbLocalStorage.ts | 25 +++++++ package-lock.json | 27 ++++++++ package.json | 1 + test.html | 2 + test.ts | 1 - 9 files changed, 99 insertions(+), 65 deletions(-) create mode 100644 Logic/Web/IdbLocalStorage.ts diff --git a/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts b/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts index 4bbba043f7..9b96f62e05 100644 --- a/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts +++ b/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts @@ -3,56 +3,49 @@ * * Technically, more an Actor then a featuresource, but it fits more neatly this ay */ -import {FeatureSourceForLayer} from "../FeatureSource"; +import FeatureSource, {Tiled} from "../FeatureSource"; import {Tiles} from "../../../Models/TileRange"; +import {IdbLocalStorage} from "../../Web/IdbLocalStorage"; +import {UIEventSource} from "../../UIEventSource"; export default class SaveTileToLocalStorageActor { - public static readonly storageKey: string = "cached-features"; - public static readonly formatVersion: string = "2" + private readonly visitedTiles: UIEventSource> + private readonly _layerId: string; + static storageKey: string = ""; - constructor(source: FeatureSourceForLayer, tileIndex: number) { + constructor(layerId: string) { + this._layerId = layerId; + this.visitedTiles = IdbLocalStorage.Get("visited_tiles_" + layerId, + {defaultValue: new Map(), }) + } - source.features.addCallbackAndRunD(features => { - const key = `${SaveTileToLocalStorageActor.storageKey}-${source.layer.layerDef.id}-${tileIndex}` + public loadAvailableTiles(){ + this.visitedTiles.addCallbackAndRunD() + } + + public addTile(tile: FeatureSource & Tiled){ + tile.features.addCallbackAndRunD(features => { const now = new Date() - try { - if (features.length > 0) { - localStorage.setItem(key, JSON.stringify(features)); - } - // We _still_ write the time to know that this tile is empty! - SaveTileToLocalStorageActor.MarkVisited(source.layer.layerDef.id, tileIndex, now) - } catch (e) { - console.warn("Could not save the features to local storage:", e) + if (features.length > 0) { + IdbLocalStorage.SetDirectly(this._layerId+"_"+tile.tileIndex, features) } + // We _still_ write the time to know that this tile is empty! + this.MarkVisited(tile.tileIndex, now) }) } - - - public static MarkVisited(layerId: string, tileId: number, freshness: Date) { - const key = `${SaveTileToLocalStorageActor.storageKey}-${layerId}-${tileId}` - try { - localStorage.setItem(key + "-time", JSON.stringify(freshness.getTime())) - localStorage.setItem(key + "-format", SaveTileToLocalStorageActor.formatVersion) - } catch (e) { - console.error("Could not mark tile ", key, "as visited") - } - } - - public static poison(layers: string[], lon: number, lat: number) { + + public poison(lon: number, lat: number) { for (let z = 0; z < 25; z++) { - const {x, y} = Tiles.embedded_tile(lat, lon, z) const tileId = Tiles.tile_index(z, x, y) - - for (const layerId of layers) { - - const key = `${SaveTileToLocalStorageActor.storageKey}-${layerId}-${tileId}` - localStorage.removeItem(key + "-time"); - localStorage.removeItem(key + "-format") - localStorage.removeItem(key) - } + this.visitedTiles.data.delete(tileId) } } + + public MarkVisited(tileId: number, freshness: Date) { + this.visitedTiles.data.set(tileId, freshness) + this.visitedTiles.ping() + } } \ No newline at end of file diff --git a/Logic/FeatureSource/FeaturePipeline.ts b/Logic/FeatureSource/FeaturePipeline.ts index 9d10ca94e2..f6149a1eef 100644 --- a/Logic/FeatureSource/FeaturePipeline.ts +++ b/Logic/FeatureSource/FeaturePipeline.ts @@ -56,6 +56,8 @@ export default class FeaturePipeline { private readonly oldestAllowedDate: Date; private readonly osmSourceZoomLevel + + private readonly localStorageSavers = new Map() constructor( handleFeatureSource: (source: FeatureSourceForLayer & Tiled) => void, @@ -77,7 +79,7 @@ export default class FeaturePipeline { .map(ch => ch.changes) .filter(coor => coor["lat"] !== undefined && coor["lon"] !== undefined) .forEach(coor => { - SaveTileToLocalStorageActor.poison(state.layoutToUse.layers.map(l => l.id), coor["lon"], coor["lat"]) + state.layoutToUse.layers.forEach(l => self.localStorageSavers.get(l.id).poison(coor["lon"], coor["lat"])) }) }) @@ -150,6 +152,8 @@ export default class FeaturePipeline { handlePriviligedFeatureSource(state.homeLocation) continue } + + this.localStorageSavers.set(filteredLayer.layerDef.id, new SaveTileToLocalStorageActor(filteredLayer.layerDef.id)) if (source.geojsonSource === undefined) { // This is an OSM layer @@ -210,7 +214,7 @@ export default class FeaturePipeline { handleTile: tile => { new RegisteringAllFromFeatureSourceActor(tile) if (tile.layer.layerDef.maxAgeOfCache > 0) { - new SaveTileToLocalStorageActor(tile, tile.tileIndex) + self.localStorageSavers.get(tile.layer.layerDef.id).addTile(tile) } perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile) tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile)) @@ -219,10 +223,11 @@ export default class FeaturePipeline { state: state, markTileVisited: (tileId) => state.filteredLayers.data.forEach(flayer => { - if (flayer.layerDef.maxAgeOfCache > 0) { - SaveTileToLocalStorageActor.MarkVisited(flayer.layerDef.id, tileId, new Date()) + const layer = flayer.layerDef + if (layer.maxAgeOfCache > 0) { + self.localStorageSavers.get(layer.id).MarkVisited(tileId, new Date()) } - self.freshnesses.get(flayer.layerDef.id).addTileLoad(tileId, new Date()) + self.freshnesses.get(layer.id).addTileLoad(tileId, new Date()) }) }) @@ -252,10 +257,8 @@ export default class FeaturePipeline { maxFeatureCount: state.layoutToUse.clustering.minNeededElements, maxZoomLevel: state.layoutToUse.clustering.maxZoom, registerTile: (tile) => { - // We save the tile data for the given layer to local storage - if (source.layer.layerDef.source.geojsonSource === undefined || source.layer.layerDef.source.isOsmCacheLayer == true) { - new SaveTileToLocalStorageActor(tile, tile.tileIndex) - } + // We save the tile data for the given layer to local storage - data sourced from overpass + self.localStorageSavers.get(tile.layer.layerDef.id).addTile(tile) perLayerHierarchy.get(source.layer.layerDef.id).registerTile(new RememberingSource(tile)) tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile)) @@ -417,7 +420,7 @@ export default class FeaturePipeline { const tileIndex = Tiles.tile_index(paddedToZoomLevel, x, y) downloadedLayers.forEach(layer => { self.freshnesses.get(layer.id).addTileLoad(tileIndex, date) - SaveTileToLocalStorageActor.MarkVisited(layer.id, tileIndex, date) + self.localStorageSavers.get(layer.id).MarkVisited(tileIndex, date) }) }) diff --git a/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts b/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts index c473491b26..3920eb8875 100644 --- a/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts +++ b/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts @@ -3,7 +3,6 @@ import {FeatureSourceForLayer, Tiled} from "../FeatureSource"; import {UIEventSource} from "../../UIEventSource"; import TileHierarchy from "./TileHierarchy"; import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor"; -import {Tiles} from "../../../Models/TileRange"; import {BBox} from "../../BBox"; import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"; @@ -33,21 +32,6 @@ export default class TiledFromLocalStorageSource implements TileHierarchy !isNaN(i)) - console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", knownTiles.map(i => Tiles.tile_from_index(i).join("/")).join(", ")) - for (const index of knownTiles) { - - const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + index; - const version = localStorage.getItem(prefix + "-format") - if (version === undefined || version !== SaveTileToLocalStorageActor.formatVersion) { - // Invalid version! Remove this tile from local storage - localStorage.removeItem(prefix) - localStorage.removeItem(prefix + "-time") - localStorage.removeItem(prefix + "-format") - this.undefinedTiles.add(index) - console.log("Dropped old format tile", prefix) - } - } - const self = this state.currentBounds.map(bounds => { @@ -91,7 +75,6 @@ export default class TiledFromLocalStorageSource implements TileHierarchy(key: string, options: { defaultValue?: T }): UIEventSource{ + const src = new UIEventSource(options.defaultValue, "idb-local-storage:"+key) + idb.get(key).then(v => { + src.setData(v ?? options.defaultValue) + }) + src.stabilized(1000).addCallback(v => { + idb.set(key, v) + }) + return src; + + } + + public static SetDirectly(key: string, value){ + idb.set(key, value) + } + +} diff --git a/package-lock.json b/package-lock.json index 11c576967e..cde01a2f79 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "email-validator": "^2.0.4", "escape-html": "^1.0.3", "i18next-client": "^1.11.4", + "idb-keyval": "^6.0.3", "jquery": "^3.6.0", "jspdf": "^2.3.1", "latlon2country": "^1.1.3", @@ -7435,6 +7436,14 @@ "resolved": "https://registry.npmjs.org/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz", "integrity": "sha1-Bupvg2ead0njhs/h/oEq5dsiPe0=" }, + "node_modules/idb-keyval": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.0.3.tgz", + "integrity": "sha512-yh8V7CnE6EQMu9YDwQXhRxwZh4nv+8xm/HV4ZqK4IiYFJBWYGjJuykADJbSP+F/GDXUBwCSSNn/14IpGL81TuA==", + "dependencies": { + "safari-14-idb-fix": "^3.0.0" + } + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -14366,6 +14375,11 @@ "integrity": "sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A==", "dev": true }, + "node_modules/safari-14-idb-fix": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/safari-14-idb-fix/-/safari-14-idb-fix-3.0.0.tgz", + "integrity": "sha512-eBNFLob4PMq8JA1dGyFn6G97q3/WzNtFK4RnzT1fnLq+9RyrGknzYiM/9B12MnKAxuj1IXr7UKYtTNtjyKMBog==" + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -24018,6 +24032,14 @@ "resolved": "https://registry.npmjs.org/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz", "integrity": "sha1-Bupvg2ead0njhs/h/oEq5dsiPe0=" }, + "idb-keyval": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.0.3.tgz", + "integrity": "sha512-yh8V7CnE6EQMu9YDwQXhRxwZh4nv+8xm/HV4ZqK4IiYFJBWYGjJuykADJbSP+F/GDXUBwCSSNn/14IpGL81TuA==", + "requires": { + "safari-14-idb-fix": "^3.0.0" + } + }, "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -29713,6 +29735,11 @@ } } }, + "safari-14-idb-fix": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/safari-14-idb-fix/-/safari-14-idb-fix-3.0.0.tgz", + "integrity": "sha512-eBNFLob4PMq8JA1dGyFn6G97q3/WzNtFK4RnzT1fnLq+9RyrGknzYiM/9B12MnKAxuj1IXr7UKYtTNtjyKMBog==" + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", diff --git a/package.json b/package.json index eae06c77e8..dc42d45f5a 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "email-validator": "^2.0.4", "escape-html": "^1.0.3", "i18next-client": "^1.11.4", + "idb-keyval": "^6.0.3", "jquery": "^3.6.0", "jspdf": "^2.3.1", "latlon2country": "^1.1.3", diff --git a/test.html b/test.html index 1ec543f931..76279a8a6f 100644 --- a/test.html +++ b/test.html @@ -23,6 +23,8 @@
'extradiv' not attached
+ + diff --git a/test.ts b/test.ts index d9d131fdf8..e69de29bb2 100644 --- a/test.ts +++ b/test.ts @@ -1 +0,0 @@ -console.log("Tests...") \ No newline at end of file From 9c848cfaee7fef01bdf27a36dedb1d85f2bb4c65 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Tue, 16 Nov 2021 02:57:26 +0100 Subject: [PATCH 2/2] Use IndexedDb to store cached geodata, fix #494. This should prevent crashes --- Logic/Actors/OverpassFeatureSource.ts | 4 + .../Actors/SaveTileToLocalStorageActor.ts | 82 +++++++++++-- Logic/FeatureSource/FeaturePipeline.ts | 26 ++-- .../TiledFromLocalStorageSource.ts | 116 ------------------ Logic/State/MapState.ts | 1 - Logic/UIEventSource.ts | 1 + Logic/Web/IdbLocalStorage.ts | 11 +- 7 files changed, 94 insertions(+), 147 deletions(-) delete mode 100644 Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts diff --git a/Logic/Actors/OverpassFeatureSource.ts b/Logic/Actors/OverpassFeatureSource.ts index feb09f9524..859b34efe5 100644 --- a/Logic/Actors/OverpassFeatureSource.ts +++ b/Logic/Actors/OverpassFeatureSource.ts @@ -10,6 +10,7 @@ import RelationsTracker from "../Osm/RelationsTracker"; import {BBox} from "../BBox"; import Loc from "../../Models/Loc"; import LayerConfig from "../../Models/ThemeConfig/LayerConfig"; +import AllKnownLayers from "../../Customizations/AllKnownLayers"; export default class OverpassFeatureSource implements FeatureSource { @@ -121,6 +122,9 @@ export default class OverpassFeatureSource implements FeatureSource { if (typeof (layer) === "string") { throw "A layer was not expanded!" } + if(AllKnownLayers.priviliged_layers.indexOf(layer.id) >= 0){ + continue + } if (this.state.locationControl.data.zoom < layer.minzoom) { continue; } diff --git a/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts b/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts index 9b96f62e05..283341a876 100644 --- a/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts +++ b/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts @@ -7,28 +7,92 @@ import FeatureSource, {Tiled} from "../FeatureSource"; import {Tiles} from "../../../Models/TileRange"; import {IdbLocalStorage} from "../../Web/IdbLocalStorage"; import {UIEventSource} from "../../UIEventSource"; +import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"; +import {BBox} from "../../BBox"; +import SimpleFeatureSource from "../Sources/SimpleFeatureSource"; +import FilteredLayer from "../../../Models/FilteredLayer"; export default class SaveTileToLocalStorageActor { private readonly visitedTiles: UIEventSource> - private readonly _layerId: string; - static storageKey: string = ""; + private readonly _layer: LayerConfig; + private readonly _flayer : FilteredLayer + private readonly initializeTime = new Date() - constructor(layerId: string) { - this._layerId = layerId; - this.visitedTiles = IdbLocalStorage.Get("visited_tiles_" + layerId, + constructor(layer: FilteredLayer) { + this._flayer = layer + this._layer = layer.layerDef + this.visitedTiles = IdbLocalStorage.Get("visited_tiles_" + this._layer.id, {defaultValue: new Map(), }) + this.visitedTiles.stabilized(100).addCallbackAndRunD(tiles => { + for (const key of Array.from(tiles.keys())) { + const tileFreshness = tiles.get(key) + + const toOld = (this.initializeTime.getTime() - tileFreshness.getTime()) > 1000 * this._layer.maxAgeOfCache + if(toOld){ + // Purge this tile + this.SetIdb(key, undefined) + console.debug("Purging tile",this._layer.id,key) + tiles.delete(key) + } + } + this.visitedTiles.ping() + return true; + }) + } + + public LoadTilesFromDisk(currentBounds: UIEventSource, + registerFreshness: (tileId: number, freshness: Date) => void, + registerTile: ((src: FeatureSource & Tiled ) => void)){ + const self = this; + this.visitedTiles.addCallbackD(tiles => { + if(tiles.size === 0){ + // We don't do anything yet as probably not yet loaded from disk + // We'll unregister later on + return; + } + for (const key of Array.from(tiles.keys())) { + const tileFreshness = tiles.get(key) + if(tileFreshness > self.initializeTime){ + // This tile is loaded by another source + continue + } + registerFreshness(key, tileFreshness) + + const tileBbox = BBox.fromTileIndex(key) + currentBounds.addCallbackAndRunD(bbox => { + if(bbox.overlapsWith(tileBbox)){ + // The current tile should be loaded from disk + this.GetIdb(key).then((features:{feature: any, freshness: Date}[] ) => { + console.log("Loaded tile "+self._layer.id+"_"+key+" from disk") + const src = new SimpleFeatureSource(self._flayer, key, new UIEventSource<{feature: any; freshness: Date}[]>(features)) + registerTile(src) + }) + return true; // only load once: unregister + } + }) + + } + + return true; // Remove the callback + + }) } - public loadAvailableTiles(){ - this.visitedTiles.addCallbackAndRunD() + private SetIdb(tileIndex, data){ + IdbLocalStorage.SetDirectly(this._layer.id+"_"+tileIndex, data) } + private GetIdb(tileIndex){ + return IdbLocalStorage.GetDirectly(this._layer.id+"_"+tileIndex) + } + public addTile(tile: FeatureSource & Tiled){ + const self = this tile.features.addCallbackAndRunD(features => { const now = new Date() if (features.length > 0) { - IdbLocalStorage.SetDirectly(this._layerId+"_"+tile.tileIndex, features) + self.SetIdb(tile.tileIndex, features) } // We _still_ write the time to know that this tile is empty! this.MarkVisited(tile.tileIndex, now) @@ -46,6 +110,6 @@ export default class SaveTileToLocalStorageActor { public MarkVisited(tileId: number, freshness: Date) { this.visitedTiles.data.set(tileId, freshness) - this.visitedTiles.ping() + this.visitedTiles.ping() } } \ No newline at end of file diff --git a/Logic/FeatureSource/FeaturePipeline.ts b/Logic/FeatureSource/FeaturePipeline.ts index f6149a1eef..56ed5cfd53 100644 --- a/Logic/FeatureSource/FeaturePipeline.ts +++ b/Logic/FeatureSource/FeaturePipeline.ts @@ -11,7 +11,6 @@ import OverpassFeatureSource from "../Actors/OverpassFeatureSource"; import GeoJsonSource from "./Sources/GeoJsonSource"; import Loc from "../../Models/Loc"; import RegisteringAllFromFeatureSourceActor from "./Actors/RegisteringAllFromFeatureSourceActor"; -import TiledFromLocalStorageSource from "./TiledFeatureSource/TiledFromLocalStorageSource"; import SaveTileToLocalStorageActor from "./Actors/SaveTileToLocalStorageActor"; import DynamicGeoJsonTileSource from "./TiledFeatureSource/DynamicGeoJsonTileSource"; import {TileHierarchyMerger} from "./TiledFeatureSource/TileHierarchyMerger"; @@ -66,9 +65,6 @@ export default class FeaturePipeline { const self = this const expiryInSeconds = Math.min(...state.layoutToUse.layers.map(l => l.maxAgeOfCache)) - for (const layer of state.layoutToUse.layers) { - TiledFromLocalStorageSource.cleanCacheForLayer(layer) - } this.oldestAllowedDate = new Date(new Date().getTime() - expiryInSeconds); this.osmSourceZoomLevel = state.osmApiTileSize.data; const useOsmApi = state.locationControl.map(l => l.zoom > (state.overpassMaxZoom.data ?? 12)) @@ -153,22 +149,22 @@ export default class FeaturePipeline { continue } - this.localStorageSavers.set(filteredLayer.layerDef.id, new SaveTileToLocalStorageActor(filteredLayer.layerDef.id)) + const localTileSaver = new SaveTileToLocalStorageActor(filteredLayer) + this.localStorageSavers.set(filteredLayer.layerDef.id, localTileSaver) if (source.geojsonSource === undefined) { // This is an OSM layer // We load the cached values and register them // Getting data from upstream happens a bit lower - new TiledFromLocalStorageSource(filteredLayer, - (src) => { - new RegisteringAllFromFeatureSourceActor(src) - hierarchy.registerTile(src); - src.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(src)) - }, state) - - TiledFromLocalStorageSource.GetFreshnesses(id).forEach((value, key) => { - self.freshnesses.get(id).addTileLoad(key, value) - }) + localTileSaver.LoadTilesFromDisk( + state.currentBounds, + (tileIndex, freshness) => self.freshnesses.get(id).addTileLoad(tileIndex, freshness), + (tile) => { + new RegisteringAllFromFeatureSourceActor(tile) + hierarchy.registerTile(tile); + tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile)) + } + ) continue } diff --git a/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts b/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts deleted file mode 100644 index 3920eb8875..0000000000 --- a/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts +++ /dev/null @@ -1,116 +0,0 @@ -import FilteredLayer from "../../../Models/FilteredLayer"; -import {FeatureSourceForLayer, Tiled} from "../FeatureSource"; -import {UIEventSource} from "../../UIEventSource"; -import TileHierarchy from "./TileHierarchy"; -import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor"; -import {BBox} from "../../BBox"; -import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"; - -export default class TiledFromLocalStorageSource implements TileHierarchy { - public readonly loadedTiles: Map = new Map(); - private readonly layer: FilteredLayer; - private readonly handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void; - private readonly undefinedTiles: Set; - - constructor(layer: FilteredLayer, - handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void, - state: { - currentBounds: UIEventSource - }) { - this.layer = layer; - this.handleFeatureSource = handleFeatureSource; - - - this.undefinedTiles = new Set() - const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" - const knownTiles: number[] = Object.keys(localStorage) - .filter(key => { - return key.startsWith(prefix) && !key.endsWith("-time") && !key.endsWith("-format"); - }) - .map(key => { - return Number(key.substring(prefix.length)); - }) - .filter(i => !isNaN(i)) - - const self = this - state.currentBounds.map(bounds => { - - if (bounds === undefined) { - return; - } - for (const knownTile of knownTiles) { - - if (this.loadedTiles.has(knownTile)) { - continue; - } - if (this.undefinedTiles.has(knownTile)) { - continue; - } - - if (!bounds.overlapsWith(BBox.fromTileIndex(knownTile))) { - continue; - } - self.loadTile(knownTile) - } - }) - - } - - public static GetFreshnesses(layerId: string): Map { - const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layerId + "-" - const freshnesses = new Map() - for (const key of Object.keys(localStorage)) { - if (!(key.startsWith(prefix) && key.endsWith("-time"))) { - continue - } - const index = Number(key.substring(prefix.length, key.length - "-time".length)) - const time = Number(localStorage.getItem(key)) - const freshness = new Date() - freshness.setTime(time) - freshnesses.set(index, freshness) - } - return freshnesses - } - - static cleanCacheForLayer(layer: LayerConfig) { - const now = new Date() - const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.id + "-" - for (const key of Object.keys(localStorage)) { - if (!(key.startsWith(prefix) && key.endsWith("-time"))) { - continue - } - const index = Number(key.substring(prefix.length, key.length - "-time".length)) - const time = Number(localStorage.getItem(key)) - const timeDiff = (now.getTime() - time) / 1000 - - if (timeDiff >= layer.maxAgeOfCache) { - const k = prefix + index; - localStorage.removeItem(k) - localStorage.removeItem(k + "-format") - localStorage.removeItem(k + "-time") - } - } - } - - private loadTile(neededIndex: number) { - try { - const key = SaveTileToLocalStorageActor.storageKey + "-" + this.layer.layerDef.id + "-" + neededIndex - const data = localStorage.getItem(key) - const features = JSON.parse(data) - const src = { - layer: this.layer, - features: new UIEventSource<{ feature: any; freshness: Date }[]>(features), - name: "FromLocalStorage(" + key + ")", - tileIndex: neededIndex, - bbox: BBox.fromTileIndex(neededIndex) - } - this.handleFeatureSource(src, neededIndex) - this.loadedTiles.set(neededIndex, src) - } catch (e) { - console.error("Could not load data tile from local storage due to", e) - this.undefinedTiles.add(neededIndex) - } - } - - -} \ No newline at end of file diff --git a/Logic/State/MapState.ts b/Logic/State/MapState.ts index 53b464b5c6..92a2cd1f3b 100644 --- a/Logic/State/MapState.ts +++ b/Logic/State/MapState.ts @@ -218,7 +218,6 @@ export default class MapState extends UserRelatedState { let timeDiff = Number.MAX_VALUE // in seconds const olderLocation = features.data[features.data.length - 2] if (olderLocation !== undefined) { - console.log("Previous location", previousLocation) timeDiff = (new Date(previousLocation.freshness).getTime() - new Date(olderLocation.freshness).getTime()) / 1000 } if (d < 20 && timeDiff < 60) { diff --git a/Logic/UIEventSource.ts b/Logic/UIEventSource.ts index bb862a94a8..acbf23f27b 100644 --- a/Logic/UIEventSource.ts +++ b/Logic/UIEventSource.ts @@ -223,6 +223,7 @@ export class UIEventSource { for (const callback of this._callbacks) { if (callback(this.data) === true) { // This callback wants to be deleted + // Note: it has to return precisely true in order to avoid accidental deletions if (toDelete === undefined) { toDelete = [callback] } else { diff --git a/Logic/Web/IdbLocalStorage.ts b/Logic/Web/IdbLocalStorage.ts index 6f50dea622..5b7d3db1d9 100644 --- a/Logic/Web/IdbLocalStorage.ts +++ b/Logic/Web/IdbLocalStorage.ts @@ -8,12 +8,8 @@ export class IdbLocalStorage { public static Get(key: string, options: { defaultValue?: T }): UIEventSource{ const src = new UIEventSource(options.defaultValue, "idb-local-storage:"+key) - idb.get(key).then(v => { - src.setData(v ?? options.defaultValue) - }) - src.stabilized(1000).addCallback(v => { - idb.set(key, v) - }) + idb.get(key).then(v => src.setData(v ?? options.defaultValue)) + src.addCallback(v => idb.set(key, v)) return src; } @@ -22,4 +18,7 @@ export class IdbLocalStorage { idb.set(key, value) } + static GetDirectly(key: string) { + return idb.get(key) + } }