Add cache timeout option on layerSource

Logic/FeatureSource/FeaturePipeline.ts

@@ -68,7 +68,7 @@ export default class FeaturePipeline {
 
     private readonly freshnesses = new Map<string, TileFreshnessCalculator>();
 
-    private readonly oldestAllowedDate: Date = new Date(new Date().getTime() - 60 * 60 * 24 * 30 * 1000);
+    private readonly oldestAllowedDate: Date;
     private readonly osmSourceZoomLevel
 
     constructor(
@@ -90,6 +90,11 @@ export default class FeaturePipeline {
         this.state = state;
 
         const self = this
+        const expiryInSeconds = Math.min(...state.layoutToUse.layers.map(l => l.maxAgeOfCache))
+        this.oldestAllowedDate = new Date(new Date().getTime() - expiryInSeconds);
+        for (const layer of state.layoutToUse.layers) {
+            TiledFromLocalStorageSource.cleanCacheForLayer(layer)
+        }
         this.osmSourceZoomLevel = state.osmApiTileSize.data;
         // milliseconds
         const useOsmApi = state.locationControl.map(l => l.zoom > (state.overpassMaxZoom.data ?? 12))
@@ -220,7 +225,7 @@ export default class FeaturePipeline {
                 maxZoomLevel: state.layoutToUse.clustering.maxZoom,
                 registerTile: (tile) => {
                     // We save the tile data for the given layer to local storage
-                    if(source.layer.layerDef.source.geojsonSource === undefined || source.layer.layerDef.source.isOsmCacheLayer == true){
+                    if (source.layer.layerDef.source.geojsonSource === undefined || source.layer.layerDef.source.isOsmCacheLayer == true) {
                         new SaveTileToLocalStorageActor(tile, tile.tileIndex)
                     }
                     perLayerHierarchy.get(source.layer.layerDef.id).registerTile(new RememberingSource(tile))
@@ -257,7 +262,7 @@ export default class FeaturePipeline {
         this.runningQuery = updater.runningQuery.map(
             overpass => {
                 console.log("FeaturePipeline: runningQuery state changed. Overpass", overpass ? "is querying," : "is idle,",
-                    "osmFeatureSource is", osmFeatureSource.isRunning ? "is running and needs "+neededTilesFromOsm.data?.length+" tiles (already got "+ osmFeatureSource.downloadedTiles.size  +" tiles )" : "is idle")
+                    "osmFeatureSource is", osmFeatureSource.isRunning ? "is running and needs " + neededTilesFromOsm.data?.length + " tiles (already got " + osmFeatureSource.downloadedTiles.size + " tiles )" : "is idle")
                 return overpass || osmFeatureSource.isRunning.data;
             }, [osmFeatureSource.isRunning]
         )
@@ -353,7 +358,7 @@ export default class FeaturePipeline {
                 isActive: useOsmApi.map(b => !b && overpassIsActive.data, [overpassIsActive]),
                 onBboxLoaded: (bbox, date, downloadedLayers, paddedToZoomLevel) => {
                     Tiles.MapRange(bbox.containingTileRange(paddedToZoomLevel), (x, y) => {
-                       const tileIndex =  Tiles.tile_index(paddedToZoomLevel, x, y)
+                        const tileIndex = Tiles.tile_index(paddedToZoomLevel, x, y)
                         downloadedLayers.forEach(layer => {
                             self.freshnesses.get(layer.id).addTileLoad(tileIndex, date)
                             SaveTileToLocalStorageActor.MarkVisited(layer.id, tileIndex, date)
@@ -412,7 +417,7 @@ export default class FeaturePipeline {
     }
 
     public GetFeaturesWithin(layerId: string, bbox: BBox): any[][] {
-        if(layerId === "*"){
+        if (layerId === "*") {
             return this.GetAllFeaturesWithin(bbox)
         }
         const requestedHierarchy = this.perLayerHierarchy.get(layerId)

Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts

@@ -5,6 +5,7 @@ import TileHierarchy from "./TileHierarchy";
 import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor";
 import {Tiles} from "../../../Models/TileRange";
 import {BBox} from "../../BBox";
+import LayerConfig from "../../../Models/ThemeConfig/LayerConfig";
 
 export default class TiledFromLocalStorageSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
     public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
@@ -16,7 +17,7 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
         const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layerId + "-"
         const freshnesses = new Map<number, Date>()
         for (const key of Object.keys(localStorage)) {
-            if(!(key.startsWith(prefix) && key.endsWith("-time"))){
+            if (!(key.startsWith(prefix) && key.endsWith("-time"))) {
                 continue
             }
             const index = Number(key.substring(prefix.length, key.length - "-time".length))
@@ -28,6 +29,29 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
         return freshnesses
     }
 
+
+    static cleanCacheForLayer(layer: LayerConfig) {
+        const now = new Date()
+        const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.id + "-"
+        console.log("Cleaning tiles of ", prefix, "with max age",layer.maxAgeOfCache)
+        for (const key of Object.keys(localStorage)) {
+            if (!(key.startsWith(prefix) && key.endsWith("-time"))) {
+                continue
+            }
+            const index = Number(key.substring(prefix.length, key.length - "-time".length))
+            const time = Number(localStorage.getItem(key))
+            const timeDiff = (now.getTime() - time) / 1000
+            
+            if(timeDiff >= layer.maxAgeOfCache){
+                const k = prefix+index;
+                localStorage.removeItem(k)
+                localStorage.removeItem(k+"-format")
+                localStorage.removeItem(k+"-time")
+                console.debug("Removed "+k+" from local storage: too old")
+            }
+        }
+    }
+
     constructor(layer: FilteredLayer,
                 handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void,
                 state: {
@@ -56,9 +80,9 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
             if (version === undefined || version !== SaveTileToLocalStorageActor.formatVersion) {
                 // Invalid version! Remove this tile from local storage
                 localStorage.removeItem(prefix)
-                localStorage.removeItem(prefix+"-time")
+                localStorage.removeItem(prefix + "-time")
-                localStorage.removeItem(prefix+"-format")
+                localStorage.removeItem(prefix + "-format")
-              this.  undefinedTiles.add(index)
+                this.undefinedTiles.add(index)
                 console.log("Dropped old format tile", prefix)
             }
         }
@@ -66,19 +90,19 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
         const self = this
         state.currentBounds.map(bounds => {
 
-            if(bounds === undefined){
+            if (bounds === undefined) {
                 return;
             }
             for (const knownTile of knownTiles) {
 
-                if(this.loadedTiles.has(knownTile)){
+                if (this.loadedTiles.has(knownTile)) {
                     continue;
                 }
-                if(this.undefinedTiles.has(knownTile)){
+                if (this.undefinedTiles.has(knownTile)) {
                     continue;
                 }
 
-                if(!bounds.overlapsWith(BBox.fromTileIndex(knownTile))){
+                if (!bounds.overlapsWith(BBox.fromTileIndex(knownTile))) {
                     continue;
                 }
                 self.loadTile(knownTile)
@@ -87,7 +111,7 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
 
     }
 
-    private loadTile( neededIndex: number){
+    private loadTile(neededIndex: number) {
         try {
             const key = SaveTileToLocalStorageActor.storageKey + "-" + this.layer.layerDef.id + "-" + neededIndex
             const data = localStorage.getItem(key)

Models/ThemeConfig/Json/LayerConfigJson.ts

@@ -60,8 +60,13 @@ export interface LayerConfigJson {
      * NOTE: the previous format was 'overpassTags: AndOrTagConfigJson | string', which is interpreted as a shorthand for source: {osmTags: "key=value"}
      *  While still supported, this is considered deprecated
      */
-    source: { osmTags: AndOrTagConfigJson | string, overpassScript?: string  } |
+    source: ({ osmTags: AndOrTagConfigJson | string, overpassScript?: string  } |
-        { osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number, isOsmCache?: boolean }
+        { osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number, isOsmCache?: boolean }) & ({
+        /**
+         * The maximum amount of seconds that a tile is allowed to linger in the cache
+         */
+        maxCacheAge?: number
+    })
     
     /**
      *

Models/ThemeConfig/Json/LayoutConfigJson.ts

@@ -174,26 +174,6 @@ export interface LayoutConfigJson {
      */
     tileLayerSources?: TilesourceConfigJson[]
 
-    /**
-     * The number of seconds that a feature is allowed to stay in the cache.
-     * The caching flow is as following:
-     *
-     * 1. The application is opened the first time
-     * 2. An overpass query is run
-     * 3. The result is saved to local storage
-     *
-     * On the next opening:
-     *
-     * 1. The application is opened
-     * 2. Data is loaded from cache and displayed
-     * 3. An overpass query is run
-     * 4. All data (both from overpass ánd local storage) are saved again to local storage (except when to old)
-     *
-     * Default value: 60 days
-     */
-    cacheTimout?: number;
-
-
     /**
      * The layers to display.
      *

Models/ThemeConfig/LayerConfig.ts

@@ -52,6 +52,10 @@ export default class LayerConfig {
     public readonly deletion: DeleteConfig | null;
     public readonly allowMove: MoveConfig | null
     public readonly allowSplit: boolean
+    /**
+     * In seconds
+     */
+    public readonly maxAgeOfCache: number
 
     presets: PresetConfig[];
 
@@ -87,7 +91,9 @@ export default class LayerConfig {
             // @ts-ignore
             legacy = TagUtils.Tag(json["overpassTags"], context + ".overpasstags");
         }
+
         if (json.source !== undefined) {
+            this.maxAgeOfCache = json.source.maxCacheAge ?? 24 * 60 * 60 * 30
             if (legacy !== undefined) {
                 throw (
                     context +

Models/ThemeConfig/LayoutConfig.ts

@@ -49,10 +49,7 @@ export default class LayoutConfig {
     public readonly enableIframePopout: boolean;
 
     public readonly customCss?: string;
-    /*
+
-    How long is the cache valid, in seconds?
-     */
-    public readonly cacheTimeout?: number;
     public readonly overpassUrl: string[];
     public readonly overpassTimeout: number;
     public readonly overpassMaxZoom: number
@@ -170,7 +167,6 @@ export default class LayoutConfig {
         this.enablePdfDownload = json.enablePdfDownload ?? false;
         this.enableIframePopout = json.enableIframePopout ?? true
         this.customCss = json.customCss;
-        this.cacheTimeout = json.cacheTimout ?? (60 * 24 * 60 * 60)
         this.overpassUrl = Constants.defaultOverpassUrls
         if(json.overpassUrl !== undefined){
             if(typeof json.overpassUrl === "string"){

assets/themes/grb.json

@@ -19,7 +19,6 @@
   "startLon": 3.231,
   "startZoom": 14,
   "widenFactor": 2,
-  "cacheTimeout": 3600,
   "socialImage": "",
   "layers": [
     {
@@ -29,6 +28,7 @@
       },
       "minzoom": 12,
       "source": {
+        "maxCacheAge": 0,
         "osmTags": {
           "and": [
             "fixme~*",