midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Add cache timeout option on layerSource

Browse source
This commit is contained in:
pietervdvn 2021-10-25 20:38:57 +02:00
parent 1bc7978f7c
commit c99e15eed9
7 changed files with 67 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -30,14 +30,14 @@ import TileFreshnessCalculator from "./TileFreshnessCalculator";
/**
* The features pipeline ties together a myriad of various datasources:
*
*
* - The Overpass-API
* - The OSM-API
* - Third-party geojson files, either sliced or directly.
*
*
* In order to truly understand this class, please have a look at the following diagram: https://cdn-images-1.medium.com/fit/c/800/618/1*qTK1iCtyJUr4zOyw4IFD7A.jpeg
*
*
*
*
*/
export default class FeaturePipeline {
@ -68,7 +68,7 @@ export default class FeaturePipeline {
private readonly freshnesses = new Map<string, TileFreshnessCalculator>();
private readonly oldestAllowedDate: Date = new Date(new Date().getTime() - 60 * 60 * 24 * 30 * 1000);
private readonly oldestAllowedDate: Date;
private readonly osmSourceZoomLevel
constructor(
@ -90,6 +90,11 @@ export default class FeaturePipeline {
this.state = state;
const self = this
const expiryInSeconds = Math.min(...state.layoutToUse.layers.map(l => l.maxAgeOfCache))
this.oldestAllowedDate = new Date(new Date().getTime() - expiryInSeconds);
for (const layer of state.layoutToUse.layers) {
TiledFromLocalStorageSource.cleanCacheForLayer(layer)
}
this.osmSourceZoomLevel = state.osmApiTileSize.data;
// milliseconds
const useOsmApi = state.locationControl.map(l => l.zoom > (state.overpassMaxZoom.data ?? 12))
@ -220,7 +225,7 @@ export default class FeaturePipeline {
maxZoomLevel: state.layoutToUse.clustering.maxZoom,
registerTile: (tile) => {
// We save the tile data for the given layer to local storage
if(source.layer.layerDef.source.geojsonSource === undefined || source.layer.layerDef.source.isOsmCacheLayer == true){
if (source.layer.layerDef.source.geojsonSource === undefined || source.layer.layerDef.source.isOsmCacheLayer == true) {
new SaveTileToLocalStorageActor(tile, tile.tileIndex)
}
perLayerHierarchy.get(source.layer.layerDef.id).registerTile(new RememberingSource(tile))
@ -257,7 +262,7 @@ export default class FeaturePipeline {
this.runningQuery = updater.runningQuery.map(
overpass => {
console.log("FeaturePipeline: runningQuery state changed. Overpass", overpass ? "is querying," : "is idle,",
"osmFeatureSource is", osmFeatureSource.isRunning ? "is running and needs "+neededTilesFromOsm.data?.length+" tiles (already got "+ osmFeatureSource.downloadedTiles.size +" tiles )" : "is idle")
"osmFeatureSource is", osmFeatureSource.isRunning ? "is running and needs " + neededTilesFromOsm.data?.length + " tiles (already got " + osmFeatureSource.downloadedTiles.size + " tiles )" : "is idle")
return overpass || osmFeatureSource.isRunning.data;
}, [osmFeatureSource.isRunning]
)
@ -353,7 +358,7 @@ export default class FeaturePipeline {
isActive: useOsmApi.map(b => !b && overpassIsActive.data, [overpassIsActive]),
onBboxLoaded: (bbox, date, downloadedLayers, paddedToZoomLevel) => {
Tiles.MapRange(bbox.containingTileRange(paddedToZoomLevel), (x, y) => {
const tileIndex = Tiles.tile_index(paddedToZoomLevel, x, y)
const tileIndex = Tiles.tile_index(paddedToZoomLevel, x, y)
downloadedLayers.forEach(layer => {
self.freshnesses.get(layer.id).addTileLoad(tileIndex, date)
SaveTileToLocalStorageActor.MarkVisited(layer.id, tileIndex, date)
@ -412,7 +417,7 @@ export default class FeaturePipeline {
}
public GetFeaturesWithin(layerId: string, bbox: BBox): any[][] {
if(layerId === "*"){
if (layerId === "*") {
return this.GetAllFeaturesWithin(bbox)
}
const requestedHierarchy = this.perLayerHierarchy.get(layerId)

50
Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts
View file

@ -5,6 +5,7 @@ import TileHierarchy from "./TileHierarchy";
import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig";
export default class TiledFromLocalStorageSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
@ -16,7 +17,7 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layerId + "-"
const freshnesses = new Map<number, Date>()
for (const key of Object.keys(localStorage)) {
if(!(key.startsWith(prefix) && key.endsWith("-time"))){
if (!(key.startsWith(prefix) && key.endsWith("-time"))) {
continue
}
const index = Number(key.substring(prefix.length, key.length - "-time".length))
@ -28,6 +29,29 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
return freshnesses
}
static cleanCacheForLayer(layer: LayerConfig) {
const now = new Date()
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.id + "-"
console.log("Cleaning tiles of ", prefix, "with max age",layer.maxAgeOfCache)
for (const key of Object.keys(localStorage)) {
if (!(key.startsWith(prefix) && key.endsWith("-time"))) {
continue
}
const index = Number(key.substring(prefix.length, key.length - "-time".length))
const time = Number(localStorage.getItem(key))
const timeDiff = (now.getTime() - time) / 1000
if(timeDiff >= layer.maxAgeOfCache){
const k = prefix+index;
localStorage.removeItem(k)
localStorage.removeItem(k+"-format")
localStorage.removeItem(k+"-time")
console.debug("Removed "+k+" from local storage: too old")
}
}
}
constructor(layer: FilteredLayer,
handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void,
state: {
@ -36,7 +60,7 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
this.layer = layer;
this.handleFeatureSource = handleFeatureSource;
this.undefinedTiles = new Set<number>()
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-"
const knownTiles: number[] = Object.keys(localStorage)
@ -56,9 +80,9 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
if (version === undefined || version !== SaveTileToLocalStorageActor.formatVersion) {
// Invalid version! Remove this tile from local storage
localStorage.removeItem(prefix)
localStorage.removeItem(prefix+"-time")
localStorage.removeItem(prefix+"-format")
this. undefinedTiles.add(index)
localStorage.removeItem(prefix + "-time")
localStorage.removeItem(prefix + "-format")
this.undefinedTiles.add(index)
console.log("Dropped old format tile", prefix)
}
}
@ -66,19 +90,19 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
const self = this
state.currentBounds.map(bounds => {
if(bounds === undefined){
if (bounds === undefined) {
return;
}
for (const knownTile of knownTiles) {
if(this.loadedTiles.has(knownTile)){
if (this.loadedTiles.has(knownTile)) {
continue;
}
if(this.undefinedTiles.has(knownTile)){
if (this.undefinedTiles.has(knownTile)) {
continue;
}
if(!bounds.overlapsWith(BBox.fromTileIndex(knownTile))){
if (!bounds.overlapsWith(BBox.fromTileIndex(knownTile))) {
continue;
}
self.loadTile(knownTile)
@ -86,8 +110,8 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
})
}
private loadTile( neededIndex: number){
private loadTile(neededIndex: number) {
try {
const key = SaveTileToLocalStorageActor.storageKey + "-" + this.layer.layerDef.id + "-" + neededIndex
const data = localStorage.getItem(key)

9
Models/ThemeConfig/Json/LayerConfigJson.ts
View file

@ -60,8 +60,13 @@ export interface LayerConfigJson {
* NOTE: the previous format was 'overpassTags: AndOrTagConfigJson | string', which is interpreted as a shorthand for source: {osmTags: "key=value"}
* While still supported, this is considered deprecated
*/
source: { osmTags: AndOrTagConfigJson | string, overpassScript?: string } |
{ osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number, isOsmCache?: boolean }
source: ({ osmTags: AndOrTagConfigJson | string, overpassScript?: string } |
{ osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number, isOsmCache?: boolean }) & ({
/**
* The maximum amount of seconds that a tile is allowed to linger in the cache
*/
maxCacheAge?: number
})
/**
*

20
Models/ThemeConfig/Json/LayoutConfigJson.ts
View file

@ -174,26 +174,6 @@ export interface LayoutConfigJson {
*/
tileLayerSources?: TilesourceConfigJson[]
/**
* The number of seconds that a feature is allowed to stay in the cache.
* The caching flow is as following:
*
* 1. The application is opened the first time
* 2. An overpass query is run
* 3. The result is saved to local storage
*
* On the next opening:
*
* 1. The application is opened
* 2. Data is loaded from cache and displayed
* 3. An overpass query is run
* 4. All data (both from overpass ánd local storage) are saved again to local storage (except when to old)
*
* Default value: 60 days
*/
cacheTimout?: number;
/**
* The layers to display.
*

8
Models/ThemeConfig/LayerConfig.ts
View file

@ -52,6 +52,10 @@ export default class LayerConfig {
public readonly deletion: DeleteConfig | null;
public readonly allowMove: MoveConfig | null
public readonly allowSplit: boolean
/**
* In seconds
*/
public readonly maxAgeOfCache: number
presets: PresetConfig[];
@ -87,7 +91,9 @@ export default class LayerConfig {
// @ts-ignore
legacy = TagUtils.Tag(json["overpassTags"], context + ".overpasstags");
}
if (json.source !== undefined) {
this.maxAgeOfCache = json.source.maxCacheAge ?? 24 * 60 * 60 * 30
if (legacy !== undefined) {
throw (
context +
@ -126,7 +132,7 @@ export default class LayerConfig {
osmTags: legacy,
});
}
this.calculatedTags = undefined;
if (json.calculatedTags !== undefined) {
if (!official) {

6
Models/ThemeConfig/LayoutConfig.ts
View file

@ -49,10 +49,7 @@ export default class LayoutConfig {
public readonly enableIframePopout: boolean;
public readonly customCss?: string;
/*
How long is the cache valid, in seconds?
*/
public readonly cacheTimeout?: number;
public readonly overpassUrl: string[];
public readonly overpassTimeout: number;
public readonly overpassMaxZoom: number
@ -170,7 +167,6 @@ export default class LayoutConfig {
this.enablePdfDownload = json.enablePdfDownload ?? false;
this.enableIframePopout = json.enableIframePopout ?? true
this.customCss = json.customCss;
this.cacheTimeout = json.cacheTimout ?? (60 * 24 * 60 * 60)
this.overpassUrl = Constants.defaultOverpassUrls
if(json.overpassUrl !== undefined){
if(typeof json.overpassUrl === "string"){

2
assets/themes/grb.json
View file

@ -19,7 +19,6 @@
"startLon": 3.231,
"startZoom": 14,
"widenFactor": 2,
"cacheTimeout": 3600,
"socialImage": "",
"layers": [
{
@ -29,6 +28,7 @@
},
"minzoom": 12,
"source": {
"maxCacheAge": 0,
"osmTags": {
"and": [
"fixme~*",