midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Better tracking of cached data, only load data if needed

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-09-30 04:13:23 +02:00
parent 32cbd6e2c1
commit 4f456e8a7f
13 changed files with 349 additions and 185 deletions
Download patch file Download diff file Expand all files Collapse all files

26
Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts
View file

@ -7,28 +7,30 @@ import {FeatureSourceForLayer} from "../FeatureSource";
export default class SaveTileToLocalStorageActor {
public static readonly storageKey: string = "cached-features";
public static readonly formatVersion : string = "1"
public static readonly formatVersion: string = "1"
constructor(source: FeatureSourceForLayer, tileIndex: number) {
source.features.addCallbackAndRunD(features => {
const key = `${SaveTileToLocalStorageActor.storageKey}-${source.layer.layerDef.id}-${tileIndex}`
const now = new Date().getTime()
if (features.length == 0) {
return;
}
const now = new Date()
try {
localStorage.setItem(key, JSON.stringify(features));
localStorage.setItem(key + "-time", JSON.stringify(now))
localStorage.setItem(key+"-format", SaveTileToLocalStorageActor.formatVersion)
if (features.length > 0) {
localStorage.setItem(key, JSON.stringify(features));
}
// We _still_ write the time to know that this tile is empty!
SaveTileToLocalStorageActor.MarkVisited(source.layer.layerDef.id, tileIndex, now)
} catch (e) {
console.warn("Could not save the features to local storage:", e)
}
})
}
public static MarkVisited(layerId: string, tileId: number, freshness: Date){
const key = `${SaveTileToLocalStorageActor.storageKey}-${layerId}-${tileId}`
localStorage.setItem(key + "-time", JSON.stringify(freshness.getTime()))
localStorage.setItem(key + "-format", SaveTileToLocalStorageActor.formatVersion)
}
}

225
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -25,6 +25,7 @@ import {BBox} from "../BBox";
import OsmFeatureSource from "./TiledFeatureSource/OsmFeatureSource";
import {OsmConnection} from "../Osm/OsmConnection";
import {Tiles} from "../../Models/TileRange";
import TileFreshnessCalculator from "./TileFreshnessCalculator";
export default class FeaturePipeline {
@ -38,9 +39,27 @@ export default class FeaturePipeline {
public readonly newDataLoadedSignal: UIEventSource<FeatureSource> = new UIEventSource<FeatureSource>(undefined)
private readonly overpassUpdater: OverpassFeatureSource
private state: {
readonly filteredLayers: UIEventSource<FilteredLayer[]>,
readonly locationControl: UIEventSource<Loc>,
readonly selectedElement: UIEventSource<any>,
readonly changes: Changes,
readonly layoutToUse: LayoutConfig,
readonly leafletMap: any,
readonly overpassUrl: UIEventSource<string[]>;
readonly overpassTimeout: UIEventSource<number>;
readonly overpassMaxZoom: UIEventSource<number>;
readonly osmConnection: OsmConnection
readonly currentBounds: UIEventSource<BBox>
};
private readonly relationTracker: RelationsTracker
private readonly perLayerHierarchy: Map<string, TileHierarchyMerger>;
private readonly freshnesses = new Map<string, TileFreshnessCalculator>();
private readonly oldestAllowedDate: Date = new Date(new Date().getTime() - 60 * 60 * 24 * 30 * 1000);
private readonly osmSourceZoomLevel = 14
constructor(
handleFeatureSource: (source: FeatureSourceForLayer & Tiled) => void,
state: {
@ -48,7 +67,7 @@ export default class FeaturePipeline {
readonly locationControl: UIEventSource<Loc>,
readonly selectedElement: UIEventSource<any>,
readonly changes: Changes,
readonly layoutToUse: LayoutConfig,
readonly layoutToUse: LayoutConfig,
readonly leafletMap: any,
readonly overpassUrl: UIEventSource<string[]>;
readonly overpassTimeout: UIEventSource<number>;
@ -56,59 +75,15 @@ export default class FeaturePipeline {
readonly osmConnection: OsmConnection
readonly currentBounds: UIEventSource<BBox>
}) {
this.state = state;
const self = this
/**
* Maps tileid onto last download moment
*/
const tileFreshnesses = new UIEventSource<Map<number, Date>>(new Map<number, Date>())
const osmSourceZoomLevel = 14
// milliseconds
const useOsmApi = state.locationControl.map(l => l.zoom > (state.overpassMaxZoom.data ?? 12))
this.relationTracker = new RelationsTracker()
const neededTilesFromOsm = this.getNeededTilesFromOsm()
console.log("Tilefreshnesses are", tileFreshnesses.data)
const oldestAllowedDate = new Date(new Date().getTime() - (60 * 60 * 24 * 30 * 1000));
const neededTilesFromOsm = state.currentBounds.map(bbox => {
if (bbox === undefined) {
return
}
const range = bbox.containingTileRange(osmSourceZoomLevel)
const tileIndexes = []
if (range.total > 100) {
// Too much tiles!
return []
}
Tiles.MapRange(range, (x, y) => {
const i = Tiles.tile_index(osmSourceZoomLevel, x, y);
if (tileFreshnesses.data.get(i) > oldestAllowedDate) {
console.debug("Skipping tile", osmSourceZoomLevel, x, y, "as a decently fresh one is available")
// The cached tiles contain decently fresh data
return;
}
tileIndexes.push(i)
})
return tileIndexes
}, [tileFreshnesses])
const updater = new OverpassFeatureSource(state,
{
relationTracker: this.relationTracker,
isActive: useOsmApi.map(b => !b),
onUpdated: (bbox, freshness) => {
// This callback contains metadata of the overpass call
const range = bbox.containingTileRange(osmSourceZoomLevel)
Tiles.MapRange(range, (x, y) => {
tileFreshnesses.data.set(Tiles.tile_index(osmSourceZoomLevel, x, y), freshness)
})
tileFreshnesses.ping();
}
});
this.overpassUpdater = updater;
this.timeout = updater.timeout
this.sufficientlyZoomed = state.locationControl.map(location => {
if (location?.zoom === undefined) {
@ -119,12 +94,6 @@ export default class FeaturePipeline {
}
);
this.timeout = updater.timeout
// Register everything in the state' 'AllElements'
new RegisteringAllFromFeatureSourceActor(updater)
const perLayerHierarchy = new Map<string, TileHierarchyMerger>()
this.perLayerHierarchy = perLayerHierarchy
@ -140,40 +109,32 @@ export default class FeaturePipeline {
handleFeatureSource(srcFiltered)
self.somethingLoaded.setData(true)
self.freshnesses.get(src.layer.layerDef.id).addTileLoad(src.tileIndex, new Date())
};
function addToHierarchy(src: FeatureSource & Tiled, layerId: string) {
perLayerHierarchy.get(layerId).registerTile(src)
}
for (const filteredLayer of state.filteredLayers.data) {
const hierarchy = new TileHierarchyMerger(filteredLayer, (tile, _) => patchedHandleFeatureSource(tile))
const id = filteredLayer.layerDef.id
perLayerHierarchy.set(id, hierarchy)
const source = filteredLayer.layerDef.source
const hierarchy = new TileHierarchyMerger(filteredLayer, (tile, _) => patchedHandleFeatureSource(tile))
perLayerHierarchy.set(id, hierarchy)
this.freshnesses.set(id, new TileFreshnessCalculator())
if (source.geojsonSource === undefined) {
// This is an OSM layer
// We load the cached values and register them
// Getting data from upstream happens a bit lower
const localStorage = new TiledFromLocalStorageSource(filteredLayer,
new TiledFromLocalStorageSource(filteredLayer,
(src) => {
new RegisteringAllFromFeatureSourceActor(src)
hierarchy.registerTile(src);
src.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(src))
}, state)
localStorage.tileFreshness.forEach((value, key) => {
if (tileFreshnesses.data.has(key)) {
const previous = tileFreshnesses.data.get(key)
if (value < previous) {
tileFreshnesses.data.set(key, value)
}
} else {
tileFreshnesses.data.set(key, value)
}
tileFreshnesses.ping()
TiledFromLocalStorageSource.GetFreshnesses(id).forEach((value, key) => {
self.freshnesses.get(id).addTileLoad(key, value)
})
continue
@ -189,7 +150,7 @@ export default class FeaturePipeline {
dontEnforceMinZoom: true,
registerTile: (tile) => {
new RegisteringAllFromFeatureSourceActor(tile)
addToHierarchy(tile, id)
perLayerHierarchy.get(id).registerTile(tile)
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
}
})
@ -198,7 +159,7 @@ export default class FeaturePipeline {
filteredLayer,
tile => {
new RegisteringAllFromFeatureSourceActor(tile)
addToHierarchy(tile, id)
perLayerHierarchy.get(id).registerTile(tile)
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
},
state
@ -213,14 +174,22 @@ export default class FeaturePipeline {
handleTile: tile => {
new RegisteringAllFromFeatureSourceActor(tile)
new SaveTileToLocalStorageActor(tile, tile.tileIndex)
addToHierarchy(tile, tile.layer.layerDef.id),
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile)
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
},
state: state
state: state,
markTileVisited: (tileId) =>
state.filteredLayers.data.forEach(flayer => {
SaveTileToLocalStorageActor.MarkVisited(flayer.layerDef.id, tileId, new Date())
})
})
const updater = this.initOverpassUpdater(state, useOsmApi)
this.overpassUpdater = updater;
this.timeout = updater.timeout
// Actually load data from the overpass source
new PerLayerFeatureSourceSplitter(state.filteredLayers,
(source) => TiledFeatureSource.createHierarchy(source, {
@ -232,7 +201,7 @@ export default class FeaturePipeline {
registerTile: (tile) => {
// We save the tile data for the given layer to local storage
new SaveTileToLocalStorageActor(tile, tile.tileIndex)
addToHierarchy(new RememberingSource(tile), source.layer.layerDef.id);
perLayerHierarchy.get(source.layer.layerDef.id).registerTile(new RememberingSource(tile))
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
}
@ -247,7 +216,7 @@ export default class FeaturePipeline {
new PerLayerFeatureSourceSplitter(state.filteredLayers,
(perLayer) => {
// We don't bother to split them over tiles as it'll contain little features by default, so we simply add them like this
addToHierarchy(perLayer, perLayer.layer.layerDef.id)
perLayerHierarchy.get(perLayer.layer.layerDef.id).registerTile(perLayer)
// AT last, we always apply the metatags whenever possible
perLayer.features.addCallbackAndRunD(_ => self.applyMetaTags(perLayer))
perLayer.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(perLayer))
@ -270,6 +239,106 @@ export default class FeaturePipeline {
}
private freshnessForVisibleLayers(z: number, x: number, y: number): Date {
let oldestDate = undefined;
for (const flayer of this.state.filteredLayers.data) {
if (!flayer.isDisplayed.data) {
continue
}
if (this.state.locationControl.data.zoom < flayer.layerDef.minzoom) {
continue;
}
const freshness = this.freshnesses.get(flayer.layerDef.id).freshnessFor(z, x, y)
if (freshness === undefined) {
// SOmething is undefined --> we return undefined as we have to download
return undefined
}
if (oldestDate === undefined || oldestDate > freshness) {
oldestDate = freshness
}
}
return oldestDate
}
private getNeededTilesFromOsm(): UIEventSource<number[]> {
const self = this
return this.state.currentBounds.map(bbox => {
if (bbox === undefined) {
return
}
const osmSourceZoomLevel = self.osmSourceZoomLevel
const range = bbox.containingTileRange(osmSourceZoomLevel)
const tileIndexes = []
if (range.total > 100) {
// Too much tiles!
return []
}
Tiles.MapRange(range, (x, y) => {
const i = Tiles.tile_index(osmSourceZoomLevel, x, y);
const oldestDate = self.freshnessForVisibleLayers(osmSourceZoomLevel, x, y)
if (oldestDate !== undefined && oldestDate > this.oldestAllowedDate) {
console.debug("Skipping tile", osmSourceZoomLevel, x, y, "as a decently fresh one is available")
// The cached tiles contain decently fresh data
return;
}
tileIndexes.push(i)
})
return tileIndexes
})
}
private initOverpassUpdater(state: {
layoutToUse: LayoutConfig,
currentBounds: UIEventSource<BBox>,
locationControl: UIEventSource<Loc>,
readonly overpassUrl: UIEventSource<string[]>;
readonly overpassTimeout: UIEventSource<number>;
readonly overpassMaxZoom: UIEventSource<number>,
}, useOsmApi: UIEventSource<boolean>): OverpassFeatureSource {
const minzoom = Math.min(...state.layoutToUse.layers.map(layer => layer.minzoom))
const allUpToDateAndZoomSufficient = state.currentBounds.map(bbox => {
if (bbox === undefined) {
return true
}
let zoom = state.locationControl.data.zoom
if (zoom < minzoom) {
return true;
}
if (zoom > 16) {
zoom = 16
}
if (zoom < 8) {
zoom = zoom + 2
}
const range = bbox.containingTileRange(zoom)
const self = this;
const allFreshnesses = Tiles.MapRange(range, (x, y) => self.freshnessForVisibleLayers(zoom, x, y))
return !allFreshnesses.some(freshness => freshness === undefined || freshness < this.oldestAllowedDate)
}, [state.locationControl])
allUpToDateAndZoomSufficient.addCallbackAndRunD(allUpToDate => console.log("All up to data is: ", allUpToDate))
const self = this;
const updater = new OverpassFeatureSource(state,
{
relationTracker: this.relationTracker,
isActive: useOsmApi.map(b => !b && !allUpToDateAndZoomSufficient.data, [allUpToDateAndZoomSufficient]),
onBboxLoaded: ((bbox, date, downloadedLayers) => {
Tiles.MapRange(bbox.containingTileRange(self.osmSourceZoomLevel), (x, y) => {
downloadedLayers.forEach(layer => {
SaveTileToLocalStorageActor.MarkVisited(layer.id, Tiles.tile_index(this.osmSourceZoomLevel, x, y), date)
})
})
})
});
// Register everything in the state' 'AllElements'
new RegisteringAllFromFeatureSourceActor(updater)
return updater;
}
private applyMetaTags(src: FeatureSourceForLayer) {
const self = this
console.debug("Applying metatagging onto ", src.name)

72
Logic/FeatureSource/TileFreshnessCalculator.ts Normal file
View file

@ -0,0 +1,72 @@
import {Tiles} from "../../Models/TileRange";
export default class TileFreshnessCalculator {
/**
* All the freshnesses per tile index
* @private
*/
private readonly freshnesses = new Map<number, Date>();
/**
* Marks that some data got loaded for this layer
* @param tileId
* @param freshness
*/
public addTileLoad(tileId: number, freshness: Date){
const existingFreshness = this.freshnessFor(...Tiles.tile_from_index(tileId))
if(existingFreshness >= freshness){
return;
}
this.freshnesses.set(tileId, freshness)
// Do we have freshness for the neighbouring tiles? If so, we can mark the tile above as loaded too!
let [z, x, y] = Tiles.tile_from_index(tileId)
if(z === 0){
return;
}
x = x - (x % 2) // Make the tiles always even
y = y - (y % 2)
const ul = this.freshnessFor(z, x, y)?.getTime()
if(ul === undefined){
return
}
const ur = this.freshnessFor(z, x + 1, y)?.getTime()
if(ur === undefined){
return
}
const ll = this.freshnessFor(z, x, y + 1)?.getTime()
if(ll === undefined){
return
}
const lr = this.freshnessFor(z, x + 1, y + 1)?.getTime()
if(lr === undefined){
return
}
const leastFresh = Math.min(ul, ur, ll, lr)
const date = new Date()
date.setTime(leastFresh)
this.addTileLoad(
Tiles.tile_index(z - 1, Math.floor(x / 2), Math.floor(y / 2)),
date
)
}
public freshnessFor(z: number, x: number, y:number): Date {
if(z < 0){
return undefined
}
const tileId = Tiles.tile_index(z, x, y)
if(this.freshnesses.has(tileId)) {
return this.freshnesses.get(tileId)
}
// recurse up
return this.freshnessFor(z - 1, Math.floor(x /2), Math.floor(y / 2))
}
}

12
Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts
View file

@ -22,7 +22,8 @@ export default class OsmFeatureSource {
neededTiles: UIEventSource<number[]>,
state: {
readonly osmConnection: OsmConnection;
};
},
markTileVisited?: (tileId: number) => void
};
private readonly downloadedTiles = new Set<number>()
@ -33,7 +34,8 @@ export default class OsmFeatureSource {
state: {
readonly filteredLayers: UIEventSource<FilteredLayer[]>;
readonly osmConnection: OsmConnection;
};
},
markTileVisited?: (tileId: number) => void
}) {
this.options = options;
this._backend = options.state.osmConnection._oauth_config.url;
@ -84,13 +86,17 @@ export default class OsmFeatureSource {
flatProperties: true
});
console.log("Tile geojson:", z, x, y, "is", geojson)
const index = Tiles.tile_index(z, x, y);
new PerLayerFeatureSourceSplitter(this.filteredLayers,
this.handleTile,
new StaticFeatureSource(geojson.features, false),
{
tileIndex: Tiles.tile_index(z, x, y)
tileIndex:index
}
);
if(this.options.markTileVisited){
this.options.markTileVisited(index)
}
} catch (e) {
console.error("Weird error: ", e)
}

1
Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts
View file

@ -2,7 +2,6 @@ import TileHierarchy from "./TileHierarchy";
import {UIEventSource} from "../../UIEventSource";
import FeatureSource, {FeatureSourceForLayer, IndexedFeatureSource, Tiled} from "../FeatureSource";
import FilteredLayer from "../../../Models/FilteredLayer";
import {Utils} from "../../../Utils";
import FeatureSourceMerger from "../Sources/FeatureSourceMerger";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";

38
Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts
View file

@ -3,15 +3,29 @@ import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import Loc from "../../../Models/Loc";
import TileHierarchy from "./TileHierarchy";
import {Utils} from "../../../Utils";
import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
export default class TiledFromLocalStorageSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
public tileFreshness : Map<number, Date> = new Map<number, Date>()
public static GetFreshnesses(layerId: string): Map<number, Date> {
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layerId + "-"
const freshnesses = new Map<number, Date>()
for (const key of Object.keys(localStorage)) {
if(!(key.startsWith(prefix) && key.endsWith("-time"))){
continue
}
const index = Number(key.substring(prefix.length, key.length - "-time".length))
const time = Number(localStorage.getItem(key))
const freshness = new Date()
freshness.setTime(time)
freshnesses.set(index, freshness)
}
return freshnesses
}
constructor(layer: FilteredLayer,
handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void,
state: {
@ -33,21 +47,17 @@ public tileFreshness : Map<number, Date> = new Map<number, Date>()
console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", indexes.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
for (const index of indexes) {
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" +index;
const version = localStorage.getItem(prefix+"-format")
if(version === undefined || version !== SaveTileToLocalStorageActor.formatVersion){
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + index;
const version = localStorage.getItem(prefix + "-format")
if (version === undefined || version !== SaveTileToLocalStorageActor.formatVersion) {
// Invalid version! Remove this tile from local storage
localStorage.removeItem(prefix)
localStorage.removeItem(prefix+"-time")
localStorage.removeItem(prefix+"-format")
undefinedTiles.add(index)
console.log("Dropped old format tile", prefix)
continue
}
const data = Number(localStorage.getItem(prefix+"-time"))
const freshness = new Date()
freshness.setTime(data)
this.tileFreshness.set(index, freshness)
}
const zLevels = indexes.map(i => i % 100)
@ -91,8 +101,6 @@ public tileFreshness : Map<number, Date> = new Map<number, Date>()
}
, [layer.isDisplayed, state.leafletMap]).stabilized(50);
neededTiles.addCallbackAndRun(t => console.debug("Tiles to load from localstorage:", t))
neededTiles.addCallbackAndRunD(neededIndexes => {
for (const neededIndex of neededIndexes) {
// We load the features from localStorage