Robin-van-der-Linde/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Better tracking of cached data, only load data if needed

Browse source
This commit is contained in:
pietervdvn 2021-09-30 04:13:23 +02:00
parent 32cbd6e2c1
commit 4f456e8a7f
13 changed files with 349 additions and 185 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
Logic/Actors/GeoLocationHandler.ts
View file

@ -207,6 +207,9 @@ export default class GeoLocationHandler extends VariableUiElement {
});
const map = self._leafletMap.data;
if(map === undefined){
return;
}
const newMarker = L.marker(location.latlng, {icon: icon});
newMarker.addTo(map);

110
Logic/Actors/OverpassFeatureSource.ts
View file

@ -1,5 +1,4 @@
import {UIEventSource} from "../UIEventSource";
import Loc from "../../Models/Loc";
import {Or} from "../Tags/Or";
import {Overpass} from "../Osm/Overpass";
import FeatureSource from "../FeatureSource/FeatureSource";
@ -9,6 +8,8 @@ import SimpleMetaTagger from "../SimpleMetaTagger";
import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig";
import RelationsTracker from "../Osm/RelationsTracker";
import {BBox} from "../BBox";
import Loc from "../../Models/Loc";
import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
export default class OverpassFeatureSource implements FeatureSource {
@ -28,14 +29,7 @@ export default class OverpassFeatureSource implements FeatureSource {
private readonly retries: UIEventSource<number> = new UIEventSource<number>(0);
/**
* The previous bounds for which the query has been run at the given zoom level
*
* Note that some layers only activate on a certain zoom level.
* If the map location changes, we check for each layer if it is loaded:
* we start checking the bounds at the first zoom level the layer might operate. If in bounds - no reload needed, otherwise we continue walking down
*/
private readonly _previousBounds: Map<number, BBox[]> = new Map<number, BBox[]>();
private readonly state: {
readonly locationControl: UIEventSource<Loc>,
readonly layoutToUse: LayoutConfig,
@ -44,11 +38,8 @@ export default class OverpassFeatureSource implements FeatureSource {
readonly currentBounds: UIEventSource<BBox>
}
private readonly _isActive: UIEventSource<boolean>;
private _onUpdated?: (bbox: BBox, dataFreshness: Date) => void;
private readonly onBboxLoaded: (bbox: BBox, date: Date, layers: LayerConfig[]) => void;
/**
* The most important layer should go first, as that one gets first pick for the questions
*/
constructor(
state: {
readonly locationControl: UIEventSource<Loc>,
@ -60,68 +51,25 @@ export default class OverpassFeatureSource implements FeatureSource {
},
options?: {
isActive?: UIEventSource<boolean>,
onUpdated?: (bbox: BBox, freshness: Date) => void,
relationTracker: RelationsTracker
relationTracker: RelationsTracker,
onBboxLoaded?: (bbox: BBox, date: Date, layers: LayerConfig[]) => void
}) {
this.state = state
this._isActive = options.isActive;
this._onUpdated = options.onUpdated;
this.onBboxLoaded = options.onBboxLoaded
this.relationsTracker = options.relationTracker
const location = state.locationControl
const self = this;
for (let i = 0; i < 25; i++) {
// This update removes all data on all layers -> erase the map on lower levels too
this._previousBounds.set(i, []);
}
location.addCallback(() => {
self.update()
});
state.currentBounds.addCallback(_ => {
self.update()
})
}
private GetFilter(interpreterUrl: string): Overpass {
private GetFilter(interpreterUrl: string, layersToDownload: LayerConfig[]): Overpass {
let filters: TagsFilter[] = [];
let extraScripts: string[] = [];
for (const layer of this.state.layoutToUse.layers) {
if (typeof (layer) === "string") {
throw "A layer was not expanded!"
}
if (this.state.locationControl.data.zoom < layer.minzoom) {
continue;
}
if (layer.doNotDownload) {
continue;
}
if (layer.source.geojsonSource !== undefined) {
// Not our responsibility to download this layer!
continue;
}
// Check if data for this layer has already been loaded
let previouslyLoaded = false;
for (let z = layer.minzoom; z < 25 && !previouslyLoaded; z++) {
const previousLoadedBounds = this._previousBounds.get(z);
if (previousLoadedBounds === undefined) {
continue;
}
for (const previousLoadedBound of previousLoadedBounds) {
previouslyLoaded = previouslyLoaded || this.state.currentBounds.data.isContainedIn(previousLoadedBound);
if (previouslyLoaded) {
break;
}
}
}
if (previouslyLoaded) {
continue;
}
for (const layer of layersToDownload) {
if (layer.source.overpassScript !== undefined) {
extraScripts.push(layer.source.overpassScript)
} else {
@ -140,17 +88,17 @@ export default class OverpassFeatureSource implements FeatureSource {
if (!this._isActive.data) {
return;
}
const self = this
this.updateAsync().then(bboxAndDate => {
if (bboxAndDate === undefined || self._onUpdated === undefined) {
const self = this;
this.updateAsync().then(bboxDate => {
if(bboxDate === undefined || self.onBboxLoaded === undefined){
return;
}
const [bbox, date] = bboxAndDate
self._onUpdated(bbox, date);
const [bbox, date, layers] = bboxDate
self.onBboxLoaded(bbox, date, layers)
})
}
private async updateAsync(): Promise<[BBox, Date]> {
private async updateAsync(): Promise<[BBox, Date, LayerConfig[]]> {
if (this.runningQuery.data) {
console.log("Still running a query, not updating");
return undefined;
@ -167,6 +115,26 @@ export default class OverpassFeatureSource implements FeatureSource {
return undefined;
}
const self = this;
const layersToDownload = []
for (const layer of this.state.layoutToUse.layers) {
if (typeof (layer) === "string") {
throw "A layer was not expanded!"
}
if(this.state.locationControl.data.zoom < layer.minzoom){
continue;
}
if (layer.doNotDownload) {
continue;
}
if (layer.source.geojsonSource !== undefined) {
// Not our responsibility to download this layer!
continue;
}
layersToDownload.push(layer)
}
let data: any = undefined
let date: Date = undefined
@ -175,7 +143,8 @@ export default class OverpassFeatureSource implements FeatureSource {
do {
try {
const overpass = this.GetFilter(overpassUrls[lastUsed]);
const overpass = this.GetFilter(overpassUrls[lastUsed], layersToDownload);
if (overpass === undefined) {
return undefined;
@ -208,14 +177,11 @@ export default class OverpassFeatureSource implements FeatureSource {
}
} while (data === undefined);
const z = Math.floor(this.state.locationControl.data.zoom ?? 0);
self._previousBounds.get(z).push(bounds);
self.retries.setData(0);
try {
data.features.forEach(feature => SimpleMetaTagger.objectMetaInfo.applyMetaTagsOnFeature(feature, date));
self.features.setData(data.features.map(f => ({feature: f, freshness: date})));
return [bounds, date];
return [bounds, date, layersToDownload];
} catch (e) {
console.error("Got the overpass response, but could not process it: ", e, e.stack)
} finally {

26
Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts
View file

@ -7,28 +7,30 @@ import {FeatureSourceForLayer} from "../FeatureSource";
export default class SaveTileToLocalStorageActor {
public static readonly storageKey: string = "cached-features";
public static readonly formatVersion : string = "1"
public static readonly formatVersion: string = "1"
constructor(source: FeatureSourceForLayer, tileIndex: number) {
source.features.addCallbackAndRunD(features => {
const key = `${SaveTileToLocalStorageActor.storageKey}-${source.layer.layerDef.id}-${tileIndex}`
const now = new Date().getTime()
if (features.length == 0) {
return;
}
const now = new Date()
try {
localStorage.setItem(key, JSON.stringify(features));
localStorage.setItem(key + "-time", JSON.stringify(now))
localStorage.setItem(key+"-format", SaveTileToLocalStorageActor.formatVersion)
if (features.length > 0) {
localStorage.setItem(key, JSON.stringify(features));
}
// We _still_ write the time to know that this tile is empty!
SaveTileToLocalStorageActor.MarkVisited(source.layer.layerDef.id, tileIndex, now)
} catch (e) {
console.warn("Could not save the features to local storage:", e)
}
})
}
public static MarkVisited(layerId: string, tileId: number, freshness: Date){
const key = `${SaveTileToLocalStorageActor.storageKey}-${layerId}-${tileId}`
localStorage.setItem(key + "-time", JSON.stringify(freshness.getTime()))
localStorage.setItem(key + "-format", SaveTileToLocalStorageActor.formatVersion)
}
}

225
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -25,6 +25,7 @@ import {BBox} from "../BBox";
import OsmFeatureSource from "./TiledFeatureSource/OsmFeatureSource";
import {OsmConnection} from "../Osm/OsmConnection";
import {Tiles} from "../../Models/TileRange";
import TileFreshnessCalculator from "./TileFreshnessCalculator";
export default class FeaturePipeline {
@ -38,9 +39,27 @@ export default class FeaturePipeline {
public readonly newDataLoadedSignal: UIEventSource<FeatureSource> = new UIEventSource<FeatureSource>(undefined)
private readonly overpassUpdater: OverpassFeatureSource
private state: {
readonly filteredLayers: UIEventSource<FilteredLayer[]>,
readonly locationControl: UIEventSource<Loc>,
readonly selectedElement: UIEventSource<any>,
readonly changes: Changes,
readonly layoutToUse: LayoutConfig,
readonly leafletMap: any,
readonly overpassUrl: UIEventSource<string[]>;
readonly overpassTimeout: UIEventSource<number>;
readonly overpassMaxZoom: UIEventSource<number>;
readonly osmConnection: OsmConnection
readonly currentBounds: UIEventSource<BBox>
};
private readonly relationTracker: RelationsTracker
private readonly perLayerHierarchy: Map<string, TileHierarchyMerger>;
private readonly freshnesses = new Map<string, TileFreshnessCalculator>();
private readonly oldestAllowedDate: Date = new Date(new Date().getTime() - 60 * 60 * 24 * 30 * 1000);
private readonly osmSourceZoomLevel = 14
constructor(
handleFeatureSource: (source: FeatureSourceForLayer & Tiled) => void,
state: {
@ -48,7 +67,7 @@ export default class FeaturePipeline {
readonly locationControl: UIEventSource<Loc>,
readonly selectedElement: UIEventSource<any>,
readonly changes: Changes,
readonly layoutToUse: LayoutConfig,
readonly layoutToUse: LayoutConfig,
readonly leafletMap: any,
readonly overpassUrl: UIEventSource<string[]>;
readonly overpassTimeout: UIEventSource<number>;
@ -56,59 +75,15 @@ export default class FeaturePipeline {
readonly osmConnection: OsmConnection
readonly currentBounds: UIEventSource<BBox>
}) {
this.state = state;
const self = this
/**
* Maps tileid onto last download moment
*/
const tileFreshnesses = new UIEventSource<Map<number, Date>>(new Map<number, Date>())
const osmSourceZoomLevel = 14
// milliseconds
const useOsmApi = state.locationControl.map(l => l.zoom > (state.overpassMaxZoom.data ?? 12))
this.relationTracker = new RelationsTracker()
const neededTilesFromOsm = this.getNeededTilesFromOsm()
console.log("Tilefreshnesses are", tileFreshnesses.data)
const oldestAllowedDate = new Date(new Date().getTime() - (60 * 60 * 24 * 30 * 1000));
const neededTilesFromOsm = state.currentBounds.map(bbox => {
if (bbox === undefined) {
return
}
const range = bbox.containingTileRange(osmSourceZoomLevel)
const tileIndexes = []
if (range.total > 100) {
// Too much tiles!
return []
}
Tiles.MapRange(range, (x, y) => {
const i = Tiles.tile_index(osmSourceZoomLevel, x, y);
if (tileFreshnesses.data.get(i) > oldestAllowedDate) {
console.debug("Skipping tile", osmSourceZoomLevel, x, y, "as a decently fresh one is available")
// The cached tiles contain decently fresh data
return;
}
tileIndexes.push(i)
})
return tileIndexes
}, [tileFreshnesses])
const updater = new OverpassFeatureSource(state,
{
relationTracker: this.relationTracker,
isActive: useOsmApi.map(b => !b),
onUpdated: (bbox, freshness) => {
// This callback contains metadata of the overpass call
const range = bbox.containingTileRange(osmSourceZoomLevel)
Tiles.MapRange(range, (x, y) => {
tileFreshnesses.data.set(Tiles.tile_index(osmSourceZoomLevel, x, y), freshness)
})
tileFreshnesses.ping();
}
});
this.overpassUpdater = updater;
this.timeout = updater.timeout
this.sufficientlyZoomed = state.locationControl.map(location => {
if (location?.zoom === undefined) {
@ -119,12 +94,6 @@ export default class FeaturePipeline {
}
);
this.timeout = updater.timeout
// Register everything in the state' 'AllElements'
new RegisteringAllFromFeatureSourceActor(updater)
const perLayerHierarchy = new Map<string, TileHierarchyMerger>()
this.perLayerHierarchy = perLayerHierarchy
@ -140,40 +109,32 @@ export default class FeaturePipeline {
handleFeatureSource(srcFiltered)
self.somethingLoaded.setData(true)
self.freshnesses.get(src.layer.layerDef.id).addTileLoad(src.tileIndex, new Date())
};
function addToHierarchy(src: FeatureSource & Tiled, layerId: string) {
perLayerHierarchy.get(layerId).registerTile(src)
}
for (const filteredLayer of state.filteredLayers.data) {
const hierarchy = new TileHierarchyMerger(filteredLayer, (tile, _) => patchedHandleFeatureSource(tile))
const id = filteredLayer.layerDef.id
perLayerHierarchy.set(id, hierarchy)
const source = filteredLayer.layerDef.source
const hierarchy = new TileHierarchyMerger(filteredLayer, (tile, _) => patchedHandleFeatureSource(tile))
perLayerHierarchy.set(id, hierarchy)
this.freshnesses.set(id, new TileFreshnessCalculator())
if (source.geojsonSource === undefined) {
// This is an OSM layer
// We load the cached values and register them
// Getting data from upstream happens a bit lower
const localStorage = new TiledFromLocalStorageSource(filteredLayer,
new TiledFromLocalStorageSource(filteredLayer,
(src) => {
new RegisteringAllFromFeatureSourceActor(src)
hierarchy.registerTile(src);
src.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(src))
}, state)
localStorage.tileFreshness.forEach((value, key) => {
if (tileFreshnesses.data.has(key)) {
const previous = tileFreshnesses.data.get(key)
if (value < previous) {
tileFreshnesses.data.set(key, value)
}
} else {
tileFreshnesses.data.set(key, value)
}
tileFreshnesses.ping()
TiledFromLocalStorageSource.GetFreshnesses(id).forEach((value, key) => {
self.freshnesses.get(id).addTileLoad(key, value)
})
continue
@ -189,7 +150,7 @@ export default class FeaturePipeline {
dontEnforceMinZoom: true,
registerTile: (tile) => {
new RegisteringAllFromFeatureSourceActor(tile)
addToHierarchy(tile, id)
perLayerHierarchy.get(id).registerTile(tile)
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
}
})
@ -198,7 +159,7 @@ export default class FeaturePipeline {
filteredLayer,
tile => {
new RegisteringAllFromFeatureSourceActor(tile)
addToHierarchy(tile, id)
perLayerHierarchy.get(id).registerTile(tile)
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
},
state
@ -213,14 +174,22 @@ export default class FeaturePipeline {
handleTile: tile => {
new RegisteringAllFromFeatureSourceActor(tile)
new SaveTileToLocalStorageActor(tile, tile.tileIndex)
addToHierarchy(tile, tile.layer.layerDef.id),
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile)
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
},
state: state
state: state,
markTileVisited: (tileId) =>
state.filteredLayers.data.forEach(flayer => {
SaveTileToLocalStorageActor.MarkVisited(flayer.layerDef.id, tileId, new Date())
})
})
const updater = this.initOverpassUpdater(state, useOsmApi)
this.overpassUpdater = updater;
this.timeout = updater.timeout
// Actually load data from the overpass source
new PerLayerFeatureSourceSplitter(state.filteredLayers,
(source) => TiledFeatureSource.createHierarchy(source, {
@ -232,7 +201,7 @@ export default class FeaturePipeline {
registerTile: (tile) => {
// We save the tile data for the given layer to local storage
new SaveTileToLocalStorageActor(tile, tile.tileIndex)
addToHierarchy(new RememberingSource(tile), source.layer.layerDef.id);
perLayerHierarchy.get(source.layer.layerDef.id).registerTile(new RememberingSource(tile))
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
}
@ -247,7 +216,7 @@ export default class FeaturePipeline {
new PerLayerFeatureSourceSplitter(state.filteredLayers,
(perLayer) => {
// We don't bother to split them over tiles as it'll contain little features by default, so we simply add them like this
addToHierarchy(perLayer, perLayer.layer.layerDef.id)
perLayerHierarchy.get(perLayer.layer.layerDef.id).registerTile(perLayer)
// AT last, we always apply the metatags whenever possible
perLayer.features.addCallbackAndRunD(_ => self.applyMetaTags(perLayer))
perLayer.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(perLayer))
@ -270,6 +239,106 @@ export default class FeaturePipeline {
}
private freshnessForVisibleLayers(z: number, x: number, y: number): Date {
let oldestDate = undefined;
for (const flayer of this.state.filteredLayers.data) {
if (!flayer.isDisplayed.data) {
continue
}
if (this.state.locationControl.data.zoom < flayer.layerDef.minzoom) {
continue;
}
const freshness = this.freshnesses.get(flayer.layerDef.id).freshnessFor(z, x, y)
if (freshness === undefined) {
// SOmething is undefined --> we return undefined as we have to download
return undefined
}
if (oldestDate === undefined || oldestDate > freshness) {
oldestDate = freshness
}
}
return oldestDate
}
private getNeededTilesFromOsm(): UIEventSource<number[]> {
const self = this
return this.state.currentBounds.map(bbox => {
if (bbox === undefined) {
return
}
const osmSourceZoomLevel = self.osmSourceZoomLevel
const range = bbox.containingTileRange(osmSourceZoomLevel)
const tileIndexes = []
if (range.total > 100) {
// Too much tiles!
return []
}
Tiles.MapRange(range, (x, y) => {
const i = Tiles.tile_index(osmSourceZoomLevel, x, y);
const oldestDate = self.freshnessForVisibleLayers(osmSourceZoomLevel, x, y)
if (oldestDate !== undefined && oldestDate > this.oldestAllowedDate) {
console.debug("Skipping tile", osmSourceZoomLevel, x, y, "as a decently fresh one is available")
// The cached tiles contain decently fresh data
return;
}
tileIndexes.push(i)
})
return tileIndexes
})
}
private initOverpassUpdater(state: {
layoutToUse: LayoutConfig,
currentBounds: UIEventSource<BBox>,
locationControl: UIEventSource<Loc>,
readonly overpassUrl: UIEventSource<string[]>;
readonly overpassTimeout: UIEventSource<number>;
readonly overpassMaxZoom: UIEventSource<number>,
}, useOsmApi: UIEventSource<boolean>): OverpassFeatureSource {
const minzoom = Math.min(...state.layoutToUse.layers.map(layer => layer.minzoom))
const allUpToDateAndZoomSufficient = state.currentBounds.map(bbox => {
if (bbox === undefined) {
return true
}
let zoom = state.locationControl.data.zoom
if (zoom < minzoom) {
return true;
}
if (zoom > 16) {
zoom = 16
}
if (zoom < 8) {
zoom = zoom + 2
}
const range = bbox.containingTileRange(zoom)
const self = this;
const allFreshnesses = Tiles.MapRange(range, (x, y) => self.freshnessForVisibleLayers(zoom, x, y))
return !allFreshnesses.some(freshness => freshness === undefined || freshness < this.oldestAllowedDate)
}, [state.locationControl])
allUpToDateAndZoomSufficient.addCallbackAndRunD(allUpToDate => console.log("All up to data is: ", allUpToDate))
const self = this;
const updater = new OverpassFeatureSource(state,
{
relationTracker: this.relationTracker,
isActive: useOsmApi.map(b => !b && !allUpToDateAndZoomSufficient.data, [allUpToDateAndZoomSufficient]),
onBboxLoaded: ((bbox, date, downloadedLayers) => {
Tiles.MapRange(bbox.containingTileRange(self.osmSourceZoomLevel), (x, y) => {
downloadedLayers.forEach(layer => {
SaveTileToLocalStorageActor.MarkVisited(layer.id, Tiles.tile_index(this.osmSourceZoomLevel, x, y), date)
})
})
})
});
// Register everything in the state' 'AllElements'
new RegisteringAllFromFeatureSourceActor(updater)
return updater;
}
private applyMetaTags(src: FeatureSourceForLayer) {
const self = this
console.debug("Applying metatagging onto ", src.name)

72
Logic/FeatureSource/TileFreshnessCalculator.ts Normal file
View file

@ -0,0 +1,72 @@
import {Tiles} from "../../Models/TileRange";
export default class TileFreshnessCalculator {
/**
* All the freshnesses per tile index
* @private
*/
private readonly freshnesses = new Map<number, Date>();
/**
* Marks that some data got loaded for this layer
* @param tileId
* @param freshness
*/
public addTileLoad(tileId: number, freshness: Date){
const existingFreshness = this.freshnessFor(...Tiles.tile_from_index(tileId))
if(existingFreshness >= freshness){
return;
}
this.freshnesses.set(tileId, freshness)
// Do we have freshness for the neighbouring tiles? If so, we can mark the tile above as loaded too!
let [z, x, y] = Tiles.tile_from_index(tileId)
if(z === 0){
return;
}
x = x - (x % 2) // Make the tiles always even
y = y - (y % 2)
const ul = this.freshnessFor(z, x, y)?.getTime()
if(ul === undefined){
return
}
const ur = this.freshnessFor(z, x + 1, y)?.getTime()
if(ur === undefined){
return
}
const ll = this.freshnessFor(z, x, y + 1)?.getTime()
if(ll === undefined){
return
}
const lr = this.freshnessFor(z, x + 1, y + 1)?.getTime()
if(lr === undefined){
return
}
const leastFresh = Math.min(ul, ur, ll, lr)
const date = new Date()
date.setTime(leastFresh)
this.addTileLoad(
Tiles.tile_index(z - 1, Math.floor(x / 2), Math.floor(y / 2)),
date
)
}
public freshnessFor(z: number, x: number, y:number): Date {
if(z < 0){
return undefined
}
const tileId = Tiles.tile_index(z, x, y)
if(this.freshnesses.has(tileId)) {
return this.freshnesses.get(tileId)
}
// recurse up
return this.freshnessFor(z - 1, Math.floor(x /2), Math.floor(y / 2))
}
}

12
Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts
View file

@ -22,7 +22,8 @@ export default class OsmFeatureSource {
neededTiles: UIEventSource<number[]>,
state: {
readonly osmConnection: OsmConnection;
};
},
markTileVisited?: (tileId: number) => void
};
private readonly downloadedTiles = new Set<number>()
@ -33,7 +34,8 @@ export default class OsmFeatureSource {
state: {
readonly filteredLayers: UIEventSource<FilteredLayer[]>;
readonly osmConnection: OsmConnection;
};
},
markTileVisited?: (tileId: number) => void
}) {
this.options = options;
this._backend = options.state.osmConnection._oauth_config.url;
@ -84,13 +86,17 @@ export default class OsmFeatureSource {
flatProperties: true
});
console.log("Tile geojson:", z, x, y, "is", geojson)
const index = Tiles.tile_index(z, x, y);
new PerLayerFeatureSourceSplitter(this.filteredLayers,
this.handleTile,
new StaticFeatureSource(geojson.features, false),
{
tileIndex: Tiles.tile_index(z, x, y)
tileIndex:index
}
);
if(this.options.markTileVisited){
this.options.markTileVisited(index)
}
} catch (e) {
console.error("Weird error: ", e)
}

1
Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts
View file

@ -2,7 +2,6 @@ import TileHierarchy from "./TileHierarchy";
import {UIEventSource} from "../../UIEventSource";
import FeatureSource, {FeatureSourceForLayer, IndexedFeatureSource, Tiled} from "../FeatureSource";
import FilteredLayer from "../../../Models/FilteredLayer";
import {Utils} from "../../../Utils";
import FeatureSourceMerger from "../Sources/FeatureSourceMerger";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";

38
Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts
View file

@ -3,15 +3,29 @@ import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import Loc from "../../../Models/Loc";
import TileHierarchy from "./TileHierarchy";
import {Utils} from "../../../Utils";
import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
export default class TiledFromLocalStorageSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
public tileFreshness : Map<number, Date> = new Map<number, Date>()
public static GetFreshnesses(layerId: string): Map<number, Date> {
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layerId + "-"
const freshnesses = new Map<number, Date>()
for (const key of Object.keys(localStorage)) {
if(!(key.startsWith(prefix) && key.endsWith("-time"))){
continue
}
const index = Number(key.substring(prefix.length, key.length - "-time".length))
const time = Number(localStorage.getItem(key))
const freshness = new Date()
freshness.setTime(time)
freshnesses.set(index, freshness)
}
return freshnesses
}
constructor(layer: FilteredLayer,
handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void,
state: {
@ -33,21 +47,17 @@ public tileFreshness : Map<number, Date> = new Map<number, Date>()
console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", indexes.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
for (const index of indexes) {
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" +index;
const version = localStorage.getItem(prefix+"-format")
if(version === undefined || version !== SaveTileToLocalStorageActor.formatVersion){
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + index;
const version = localStorage.getItem(prefix + "-format")
if (version === undefined || version !== SaveTileToLocalStorageActor.formatVersion) {
// Invalid version! Remove this tile from local storage
localStorage.removeItem(prefix)
localStorage.removeItem(prefix+"-time")
localStorage.removeItem(prefix+"-format")
undefinedTiles.add(index)
console.log("Dropped old format tile", prefix)
continue
}
const data = Number(localStorage.getItem(prefix+"-time"))
const freshness = new Date()
freshness.setTime(data)
this.tileFreshness.set(index, freshness)
}
const zLevels = indexes.map(i => i % 100)
@ -91,8 +101,6 @@ public tileFreshness : Map<number, Date> = new Map<number, Date>()
}
, [layer.isDisplayed, state.leafletMap]).stabilized(50);
neededTiles.addCallbackAndRun(t => console.debug("Tiles to load from localstorage:", t))
neededTiles.addCallbackAndRunD(neededIndexes => {
for (const neededIndex of neededIndexes) {
// We load the features from localStorage

9
Logic/SimpleMetaTagger.ts
View file

@ -97,7 +97,7 @@ export default class SimpleMetaTagger {
continue;
}
for (const unit of units) {
if(unit === undefined){
if (unit === undefined) {
continue
}
if (unit.appliesToKeys === undefined) {
@ -108,7 +108,12 @@ export default class SimpleMetaTagger {
continue;
}
const value = feature.properties[key]
const [, denomination] = unit.findDenomination(value)
const denom = unit.findDenomination(value)
if (denom === undefined) {
// no valid value found
break;
}
const [, denomination] = denom;
let canonical = denomination?.canonicalValue(value) ?? undefined;
if (canonical === value) {
break;

2
Models/Constants.ts
View file

@ -2,7 +2,7 @@ import {Utils} from "../Utils";
export default class Constants {
public static vNumber = "0.10.0-rc1";
public static vNumber = "0.10.0-rc2";
public static ImgurApiKey = '7070e7167f0a25a'
public static readonly mapillary_client_token_v3 = 'TXhLaWthQ1d4RUg0czVxaTVoRjFJZzowNDczNjUzNmIyNTQyYzI2'
public static readonly mapillary_client_token_v4 = "MLY|4441509239301885|b40ad2d3ea105435bd40c7e76993ae85"

1
Models/TileRange.ts
View file

@ -107,4 +107,5 @@ export class Tiles {
}
}
}

4
test/TestAll.ts
View file

@ -9,6 +9,7 @@ import UnitsSpec from "./Units.spec";
import RelationSplitHandlerSpec from "./RelationSplitHandler.spec";
import SplitActionSpec from "./SplitAction.spec";
import {Utils} from "../Utils";
import TileFreshnessCalculatorSpec from "./TileFreshnessCalculator.spec";
ScriptUtils.fixUtils()
@ -21,7 +22,8 @@ const allTests = [
new UtilsSpec(),
new UnitsSpec(),
new RelationSplitHandlerSpec(),
new SplitActionSpec()
new SplitActionSpec(),
new TileFreshnessCalculatorSpec()
]
Utils.externalDownloadFunction = async (url) => {

31
test/TileFreshnessCalculator.spec.ts Normal file
View file

@ -0,0 +1,31 @@
import T from "./TestHelper";
import TileFreshnessCalculator from "../Logic/FeatureSource/TileFreshnessCalculator";
import {Tiles} from "../Models/TileRange";
import {equal} from "assert";
export default class TileFreshnessCalculatorSpec extends T {
constructor() {
super("TileFreshnessCalculatorSpec", [
[
"TileFresnessTests",
() => {
const calc = new TileFreshnessCalculator();
// 19/266407/175535
const date = new Date()
date.setTime(42)
calc.addTileLoad(Tiles.tile_index(19, 266406, 175534), date)
equal(42, calc.freshnessFor(19, 266406, 175534).getTime())
equal(42, calc.freshnessFor(20, 266406 * 2, 175534 * 2 + 1).getTime())
equal(undefined, calc.freshnessFor(19, 266406, 175535))
equal(undefined, calc.freshnessFor(18, 266406 / 2, 175534 / 2))
calc.addTileLoad(Tiles.tile_index(19, 266406, 175534+1), date)
calc.addTileLoad(Tiles.tile_index(19, 266406+1, 175534), date)
calc.addTileLoad(Tiles.tile_index(19, 266406+1, 175534+1), date)
equal(42, calc.freshnessFor(18, 266406 / 2, 175534 / 2).getTime())
}
]
])
}
}