From 8fa7de661ea3d37724992c27910286be86288b07 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Mon, 15 Nov 2021 11:51:32 +0100 Subject: [PATCH] WIP: use indexedDB as datastore for geotiles --- .../Actors/SaveTileToLocalStorageActor.ts | 65 +++++++++---------- Logic/FeatureSource/FeaturePipeline.ts | 23 ++++--- .../TiledFromLocalStorageSource.ts | 17 ----- Logic/State/MapState.ts | 3 +- Logic/Web/IdbLocalStorage.ts | 25 +++++++ package-lock.json | 27 ++++++++ package.json | 1 + test.html | 2 + test.ts | 1 - 9 files changed, 99 insertions(+), 65 deletions(-) create mode 100644 Logic/Web/IdbLocalStorage.ts diff --git a/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts b/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts index 4bbba043f..9b96f62e0 100644 --- a/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts +++ b/Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts @@ -3,56 +3,49 @@ * * Technically, more an Actor then a featuresource, but it fits more neatly this ay */ -import {FeatureSourceForLayer} from "../FeatureSource"; +import FeatureSource, {Tiled} from "../FeatureSource"; import {Tiles} from "../../../Models/TileRange"; +import {IdbLocalStorage} from "../../Web/IdbLocalStorage"; +import {UIEventSource} from "../../UIEventSource"; export default class SaveTileToLocalStorageActor { - public static readonly storageKey: string = "cached-features"; - public static readonly formatVersion: string = "2" + private readonly visitedTiles: UIEventSource> + private readonly _layerId: string; + static storageKey: string = ""; - constructor(source: FeatureSourceForLayer, tileIndex: number) { + constructor(layerId: string) { + this._layerId = layerId; + this.visitedTiles = IdbLocalStorage.Get("visited_tiles_" + layerId, + {defaultValue: new Map(), }) + } - source.features.addCallbackAndRunD(features => { - const key = `${SaveTileToLocalStorageActor.storageKey}-${source.layer.layerDef.id}-${tileIndex}` + public loadAvailableTiles(){ + this.visitedTiles.addCallbackAndRunD() + } + + public addTile(tile: FeatureSource & Tiled){ + tile.features.addCallbackAndRunD(features => { const now = new Date() - try { - if (features.length > 0) { - localStorage.setItem(key, JSON.stringify(features)); - } - // We _still_ write the time to know that this tile is empty! - SaveTileToLocalStorageActor.MarkVisited(source.layer.layerDef.id, tileIndex, now) - } catch (e) { - console.warn("Could not save the features to local storage:", e) + if (features.length > 0) { + IdbLocalStorage.SetDirectly(this._layerId+"_"+tile.tileIndex, features) } + // We _still_ write the time to know that this tile is empty! + this.MarkVisited(tile.tileIndex, now) }) } - - - public static MarkVisited(layerId: string, tileId: number, freshness: Date) { - const key = `${SaveTileToLocalStorageActor.storageKey}-${layerId}-${tileId}` - try { - localStorage.setItem(key + "-time", JSON.stringify(freshness.getTime())) - localStorage.setItem(key + "-format", SaveTileToLocalStorageActor.formatVersion) - } catch (e) { - console.error("Could not mark tile ", key, "as visited") - } - } - - public static poison(layers: string[], lon: number, lat: number) { + + public poison(lon: number, lat: number) { for (let z = 0; z < 25; z++) { - const {x, y} = Tiles.embedded_tile(lat, lon, z) const tileId = Tiles.tile_index(z, x, y) - - for (const layerId of layers) { - - const key = `${SaveTileToLocalStorageActor.storageKey}-${layerId}-${tileId}` - localStorage.removeItem(key + "-time"); - localStorage.removeItem(key + "-format") - localStorage.removeItem(key) - } + this.visitedTiles.data.delete(tileId) } } + + public MarkVisited(tileId: number, freshness: Date) { + this.visitedTiles.data.set(tileId, freshness) + this.visitedTiles.ping() + } } \ No newline at end of file diff --git a/Logic/FeatureSource/FeaturePipeline.ts b/Logic/FeatureSource/FeaturePipeline.ts index 9d10ca94e..f6149a1ee 100644 --- a/Logic/FeatureSource/FeaturePipeline.ts +++ b/Logic/FeatureSource/FeaturePipeline.ts @@ -56,6 +56,8 @@ export default class FeaturePipeline { private readonly oldestAllowedDate: Date; private readonly osmSourceZoomLevel + + private readonly localStorageSavers = new Map() constructor( handleFeatureSource: (source: FeatureSourceForLayer & Tiled) => void, @@ -77,7 +79,7 @@ export default class FeaturePipeline { .map(ch => ch.changes) .filter(coor => coor["lat"] !== undefined && coor["lon"] !== undefined) .forEach(coor => { - SaveTileToLocalStorageActor.poison(state.layoutToUse.layers.map(l => l.id), coor["lon"], coor["lat"]) + state.layoutToUse.layers.forEach(l => self.localStorageSavers.get(l.id).poison(coor["lon"], coor["lat"])) }) }) @@ -150,6 +152,8 @@ export default class FeaturePipeline { handlePriviligedFeatureSource(state.homeLocation) continue } + + this.localStorageSavers.set(filteredLayer.layerDef.id, new SaveTileToLocalStorageActor(filteredLayer.layerDef.id)) if (source.geojsonSource === undefined) { // This is an OSM layer @@ -210,7 +214,7 @@ export default class FeaturePipeline { handleTile: tile => { new RegisteringAllFromFeatureSourceActor(tile) if (tile.layer.layerDef.maxAgeOfCache > 0) { - new SaveTileToLocalStorageActor(tile, tile.tileIndex) + self.localStorageSavers.get(tile.layer.layerDef.id).addTile(tile) } perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile) tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile)) @@ -219,10 +223,11 @@ export default class FeaturePipeline { state: state, markTileVisited: (tileId) => state.filteredLayers.data.forEach(flayer => { - if (flayer.layerDef.maxAgeOfCache > 0) { - SaveTileToLocalStorageActor.MarkVisited(flayer.layerDef.id, tileId, new Date()) + const layer = flayer.layerDef + if (layer.maxAgeOfCache > 0) { + self.localStorageSavers.get(layer.id).MarkVisited(tileId, new Date()) } - self.freshnesses.get(flayer.layerDef.id).addTileLoad(tileId, new Date()) + self.freshnesses.get(layer.id).addTileLoad(tileId, new Date()) }) }) @@ -252,10 +257,8 @@ export default class FeaturePipeline { maxFeatureCount: state.layoutToUse.clustering.minNeededElements, maxZoomLevel: state.layoutToUse.clustering.maxZoom, registerTile: (tile) => { - // We save the tile data for the given layer to local storage - if (source.layer.layerDef.source.geojsonSource === undefined || source.layer.layerDef.source.isOsmCacheLayer == true) { - new SaveTileToLocalStorageActor(tile, tile.tileIndex) - } + // We save the tile data for the given layer to local storage - data sourced from overpass + self.localStorageSavers.get(tile.layer.layerDef.id).addTile(tile) perLayerHierarchy.get(source.layer.layerDef.id).registerTile(new RememberingSource(tile)) tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile)) @@ -417,7 +420,7 @@ export default class FeaturePipeline { const tileIndex = Tiles.tile_index(paddedToZoomLevel, x, y) downloadedLayers.forEach(layer => { self.freshnesses.get(layer.id).addTileLoad(tileIndex, date) - SaveTileToLocalStorageActor.MarkVisited(layer.id, tileIndex, date) + self.localStorageSavers.get(layer.id).MarkVisited(tileIndex, date) }) }) diff --git a/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts b/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts index c473491b2..3920eb887 100644 --- a/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts +++ b/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts @@ -3,7 +3,6 @@ import {FeatureSourceForLayer, Tiled} from "../FeatureSource"; import {UIEventSource} from "../../UIEventSource"; import TileHierarchy from "./TileHierarchy"; import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor"; -import {Tiles} from "../../../Models/TileRange"; import {BBox} from "../../BBox"; import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"; @@ -33,21 +32,6 @@ export default class TiledFromLocalStorageSource implements TileHierarchy !isNaN(i)) - console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", knownTiles.map(i => Tiles.tile_from_index(i).join("/")).join(", ")) - for (const index of knownTiles) { - - const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + index; - const version = localStorage.getItem(prefix + "-format") - if (version === undefined || version !== SaveTileToLocalStorageActor.formatVersion) { - // Invalid version! Remove this tile from local storage - localStorage.removeItem(prefix) - localStorage.removeItem(prefix + "-time") - localStorage.removeItem(prefix + "-format") - this.undefinedTiles.add(index) - console.log("Dropped old format tile", prefix) - } - } - const self = this state.currentBounds.map(bounds => { @@ -91,7 +75,6 @@ export default class TiledFromLocalStorageSource implements TileHierarchy(key: string, options: { defaultValue?: T }): UIEventSource{ + const src = new UIEventSource(options.defaultValue, "idb-local-storage:"+key) + idb.get(key).then(v => { + src.setData(v ?? options.defaultValue) + }) + src.stabilized(1000).addCallback(v => { + idb.set(key, v) + }) + return src; + + } + + public static SetDirectly(key: string, value){ + idb.set(key, value) + } + +} diff --git a/package-lock.json b/package-lock.json index 11c576967..cde01a2f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "email-validator": "^2.0.4", "escape-html": "^1.0.3", "i18next-client": "^1.11.4", + "idb-keyval": "^6.0.3", "jquery": "^3.6.0", "jspdf": "^2.3.1", "latlon2country": "^1.1.3", @@ -7435,6 +7436,14 @@ "resolved": "https://registry.npmjs.org/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz", "integrity": "sha1-Bupvg2ead0njhs/h/oEq5dsiPe0=" }, + "node_modules/idb-keyval": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.0.3.tgz", + "integrity": "sha512-yh8V7CnE6EQMu9YDwQXhRxwZh4nv+8xm/HV4ZqK4IiYFJBWYGjJuykADJbSP+F/GDXUBwCSSNn/14IpGL81TuA==", + "dependencies": { + "safari-14-idb-fix": "^3.0.0" + } + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -14366,6 +14375,11 @@ "integrity": "sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A==", "dev": true }, + "node_modules/safari-14-idb-fix": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/safari-14-idb-fix/-/safari-14-idb-fix-3.0.0.tgz", + "integrity": "sha512-eBNFLob4PMq8JA1dGyFn6G97q3/WzNtFK4RnzT1fnLq+9RyrGknzYiM/9B12MnKAxuj1IXr7UKYtTNtjyKMBog==" + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -24018,6 +24032,14 @@ "resolved": "https://registry.npmjs.org/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz", "integrity": "sha1-Bupvg2ead0njhs/h/oEq5dsiPe0=" }, + "idb-keyval": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.0.3.tgz", + "integrity": "sha512-yh8V7CnE6EQMu9YDwQXhRxwZh4nv+8xm/HV4ZqK4IiYFJBWYGjJuykADJbSP+F/GDXUBwCSSNn/14IpGL81TuA==", + "requires": { + "safari-14-idb-fix": "^3.0.0" + } + }, "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -29713,6 +29735,11 @@ } } }, + "safari-14-idb-fix": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/safari-14-idb-fix/-/safari-14-idb-fix-3.0.0.tgz", + "integrity": "sha512-eBNFLob4PMq8JA1dGyFn6G97q3/WzNtFK4RnzT1fnLq+9RyrGknzYiM/9B12MnKAxuj1IXr7UKYtTNtjyKMBog==" + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", diff --git a/package.json b/package.json index eae06c77e..dc42d45f5 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "email-validator": "^2.0.4", "escape-html": "^1.0.3", "i18next-client": "^1.11.4", + "idb-keyval": "^6.0.3", "jquery": "^3.6.0", "jspdf": "^2.3.1", "latlon2country": "^1.1.3", diff --git a/test.html b/test.html index 1ec543f93..76279a8a6 100644 --- a/test.html +++ b/test.html @@ -23,6 +23,8 @@
'extradiv' not attached
+ + diff --git a/test.ts b/test.ts index d9d131fdf..e69de29bb 100644 --- a/test.ts +++ b/test.ts @@ -1 +0,0 @@ -console.log("Tests...") \ No newline at end of file