From c99e15eed975a523eec46af1bf4cd0096f86b9a8 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Mon, 25 Oct 2021 20:38:57 +0200 Subject: [PATCH] Add cache timeout option on layerSource --- Logic/FeatureSource/FeaturePipeline.ts | 23 +++++---- .../TiledFromLocalStorageSource.ts | 50 ++++++++++++++----- Models/ThemeConfig/Json/LayerConfigJson.ts | 9 +++- Models/ThemeConfig/Json/LayoutConfigJson.ts | 20 -------- Models/ThemeConfig/LayerConfig.ts | 8 ++- Models/ThemeConfig/LayoutConfig.ts | 6 +-- assets/themes/grb.json | 2 +- 7 files changed, 67 insertions(+), 51 deletions(-) diff --git a/Logic/FeatureSource/FeaturePipeline.ts b/Logic/FeatureSource/FeaturePipeline.ts index e8b77bb77..a96f1e278 100644 --- a/Logic/FeatureSource/FeaturePipeline.ts +++ b/Logic/FeatureSource/FeaturePipeline.ts @@ -30,14 +30,14 @@ import TileFreshnessCalculator from "./TileFreshnessCalculator"; /** * The features pipeline ties together a myriad of various datasources: - * + * * - The Overpass-API * - The OSM-API * - Third-party geojson files, either sliced or directly. - * + * * In order to truly understand this class, please have a look at the following diagram: https://cdn-images-1.medium.com/fit/c/800/618/1*qTK1iCtyJUr4zOyw4IFD7A.jpeg - * - * + * + * */ export default class FeaturePipeline { @@ -68,7 +68,7 @@ export default class FeaturePipeline { private readonly freshnesses = new Map(); - private readonly oldestAllowedDate: Date = new Date(new Date().getTime() - 60 * 60 * 24 * 30 * 1000); + private readonly oldestAllowedDate: Date; private readonly osmSourceZoomLevel constructor( @@ -90,6 +90,11 @@ export default class FeaturePipeline { this.state = state; const self = this + const expiryInSeconds = Math.min(...state.layoutToUse.layers.map(l => l.maxAgeOfCache)) + this.oldestAllowedDate = new Date(new Date().getTime() - expiryInSeconds); + for (const layer of state.layoutToUse.layers) { + TiledFromLocalStorageSource.cleanCacheForLayer(layer) + } this.osmSourceZoomLevel = state.osmApiTileSize.data; // milliseconds const useOsmApi = state.locationControl.map(l => l.zoom > (state.overpassMaxZoom.data ?? 12)) @@ -220,7 +225,7 @@ export default class FeaturePipeline { maxZoomLevel: state.layoutToUse.clustering.maxZoom, registerTile: (tile) => { // We save the tile data for the given layer to local storage - if(source.layer.layerDef.source.geojsonSource === undefined || source.layer.layerDef.source.isOsmCacheLayer == true){ + if (source.layer.layerDef.source.geojsonSource === undefined || source.layer.layerDef.source.isOsmCacheLayer == true) { new SaveTileToLocalStorageActor(tile, tile.tileIndex) } perLayerHierarchy.get(source.layer.layerDef.id).registerTile(new RememberingSource(tile)) @@ -257,7 +262,7 @@ export default class FeaturePipeline { this.runningQuery = updater.runningQuery.map( overpass => { console.log("FeaturePipeline: runningQuery state changed. Overpass", overpass ? "is querying," : "is idle,", - "osmFeatureSource is", osmFeatureSource.isRunning ? "is running and needs "+neededTilesFromOsm.data?.length+" tiles (already got "+ osmFeatureSource.downloadedTiles.size +" tiles )" : "is idle") + "osmFeatureSource is", osmFeatureSource.isRunning ? "is running and needs " + neededTilesFromOsm.data?.length + " tiles (already got " + osmFeatureSource.downloadedTiles.size + " tiles )" : "is idle") return overpass || osmFeatureSource.isRunning.data; }, [osmFeatureSource.isRunning] ) @@ -353,7 +358,7 @@ export default class FeaturePipeline { isActive: useOsmApi.map(b => !b && overpassIsActive.data, [overpassIsActive]), onBboxLoaded: (bbox, date, downloadedLayers, paddedToZoomLevel) => { Tiles.MapRange(bbox.containingTileRange(paddedToZoomLevel), (x, y) => { - const tileIndex = Tiles.tile_index(paddedToZoomLevel, x, y) + const tileIndex = Tiles.tile_index(paddedToZoomLevel, x, y) downloadedLayers.forEach(layer => { self.freshnesses.get(layer.id).addTileLoad(tileIndex, date) SaveTileToLocalStorageActor.MarkVisited(layer.id, tileIndex, date) @@ -412,7 +417,7 @@ export default class FeaturePipeline { } public GetFeaturesWithin(layerId: string, bbox: BBox): any[][] { - if(layerId === "*"){ + if (layerId === "*") { return this.GetAllFeaturesWithin(bbox) } const requestedHierarchy = this.perLayerHierarchy.get(layerId) diff --git a/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts b/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts index a2cd50be0..7f69fedc4 100644 --- a/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts +++ b/Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts @@ -5,6 +5,7 @@ import TileHierarchy from "./TileHierarchy"; import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor"; import {Tiles} from "../../../Models/TileRange"; import {BBox} from "../../BBox"; +import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"; export default class TiledFromLocalStorageSource implements TileHierarchy { public readonly loadedTiles: Map = new Map(); @@ -16,7 +17,7 @@ export default class TiledFromLocalStorageSource implements TileHierarchy() for (const key of Object.keys(localStorage)) { - if(!(key.startsWith(prefix) && key.endsWith("-time"))){ + if (!(key.startsWith(prefix) && key.endsWith("-time"))) { continue } const index = Number(key.substring(prefix.length, key.length - "-time".length)) @@ -28,6 +29,29 @@ export default class TiledFromLocalStorageSource implements TileHierarchy= layer.maxAgeOfCache){ + const k = prefix+index; + localStorage.removeItem(k) + localStorage.removeItem(k+"-format") + localStorage.removeItem(k+"-time") + console.debug("Removed "+k+" from local storage: too old") + } + } + } + constructor(layer: FilteredLayer, handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void, state: { @@ -36,7 +60,7 @@ export default class TiledFromLocalStorageSource implements TileHierarchy() const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" const knownTiles: number[] = Object.keys(localStorage) @@ -56,9 +80,9 @@ export default class TiledFromLocalStorageSource implements TileHierarchy { - if(bounds === undefined){ + if (bounds === undefined) { return; } for (const knownTile of knownTiles) { - - if(this.loadedTiles.has(knownTile)){ + + if (this.loadedTiles.has(knownTile)) { continue; } - if(this.undefinedTiles.has(knownTile)){ + if (this.undefinedTiles.has(knownTile)) { continue; } - - if(!bounds.overlapsWith(BBox.fromTileIndex(knownTile))){ + + if (!bounds.overlapsWith(BBox.fromTileIndex(knownTile))) { continue; } self.loadTile(knownTile) @@ -86,8 +110,8 @@ export default class TiledFromLocalStorageSource implements TileHierarchy