contrapunctus/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Merge develop

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-10-11 23:33:09 +02:00
commit 8fdb7a6d7f
22 changed files with 536 additions and 141 deletions
Download patch file Download diff file Expand all files Collapse all files

20
Logic/Actors/OverpassFeatureSource.ts
View file

@ -38,8 +38,7 @@ export default class OverpassFeatureSource implements FeatureSource {
readonly currentBounds: UIEventSource<BBox>
}
private readonly _isActive: UIEventSource<boolean>;
private readonly onBboxLoaded: (bbox: BBox, date: Date, layers: LayerConfig[]) => void;
private readonly onBboxLoaded: (bbox: BBox, date: Date, layers: LayerConfig[], zoomlevel: number) => void;
constructor(
state: {
readonly locationControl: UIEventSource<Loc>,
@ -49,10 +48,11 @@ export default class OverpassFeatureSource implements FeatureSource {
readonly overpassMaxZoom: UIEventSource<number>,
readonly currentBounds: UIEventSource<BBox>
},
options?: {
options: {
padToTiles: UIEventSource<number>,
isActive?: UIEventSource<boolean>,
relationTracker: RelationsTracker,
onBboxLoaded?: (bbox: BBox, date: Date, layers: LayerConfig[]) => void
onBboxLoaded?: (bbox: BBox, date: Date, layers: LayerConfig[], zoomlevel: number) => void
}) {
this.state = state
@ -61,7 +61,7 @@ export default class OverpassFeatureSource implements FeatureSource {
this.relationsTracker = options.relationTracker
const self = this;
state.currentBounds.addCallback(_ => {
self.update()
self.update(options.padToTiles.data)
})
}
@ -84,21 +84,21 @@ export default class OverpassFeatureSource implements FeatureSource {
return new Overpass(new Or(filters), extraScripts, interpreterUrl, this.state.overpassTimeout, this.relationsTracker);
}
private update() {
private update(paddedZoomLevel: number) {
if (!this._isActive.data) {
return;
}
const self = this;
this.updateAsync().then(bboxDate => {
this.updateAsync(paddedZoomLevel).then(bboxDate => {
if(bboxDate === undefined || self.onBboxLoaded === undefined){
return;
}
const [bbox, date, layers] = bboxDate
self.onBboxLoaded(bbox, date, layers)
self.onBboxLoaded(bbox, date, layers, paddedZoomLevel)
})
}
private async updateAsync(): Promise<[BBox, Date, LayerConfig[]]> {
private async updateAsync(padToZoomLevel: number): Promise<[BBox, Date, LayerConfig[]]> {
if (this.runningQuery.data) {
console.log("Still running a query, not updating");
return undefined;
@ -109,7 +109,7 @@ export default class OverpassFeatureSource implements FeatureSource {
return undefined;
}
const bounds = this.state.currentBounds.data?.pad(this.state.layoutToUse.widenFactor)?.expandToTileBounds(14);
const bounds = this.state.currentBounds.data?.pad(this.state.layoutToUse.widenFactor)?.expandToTileBounds(padToZoomLevel);
if (bounds === undefined) {
return undefined;

17
Logic/BBox.ts
View file

@ -116,16 +116,15 @@ export class BBox {
return this.minLat
}
pad(factor: number): BBox {
const latDiff = this.maxLat - this.minLat
const lat = (this.maxLat + this.minLat) / 2
const lonDiff = this.maxLon - this.minLon
const lon = (this.maxLon + this.minLon) / 2
pad(factor: number, maxIncrease = 2): BBox {
const latDiff = Math.min(maxIncrease / 2, Math.abs(this.maxLat - this.minLat) * factor)
const lonDiff =Math.min(maxIncrease / 2, Math.abs(this.maxLon - this.minLon) * factor)
return new BBox([[
lon - lonDiff * factor,
lat - latDiff * factor
], [lon + lonDiff * factor,
lat + latDiff * factor]])
this.minLon - lonDiff,
this.minLat - latDiff
], [this.maxLon + lonDiff,
this.maxLat + latDiff]])
}
toLeaflet() {

4
Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts
View file

@ -4,11 +4,10 @@
* Technically, more an Actor then a featuresource, but it fits more neatly this ay
*/
import {FeatureSourceForLayer} from "../FeatureSource";
import SimpleMetaTagger from "../../SimpleMetaTagger";
export default class SaveTileToLocalStorageActor {
public static readonly storageKey: string = "cached-features";
public static readonly formatVersion: string = "1"
public static readonly formatVersion: string = "2"
constructor(source: FeatureSourceForLayer, tileIndex: number) {
@ -37,6 +36,5 @@ export default class SaveTileToLocalStorageActor {
}catch(e){
console.error("Could not mark tile ", key, "as visited")
}
}
}

32
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -58,7 +58,7 @@ export default class FeaturePipeline {
private readonly freshnesses = new Map<string, TileFreshnessCalculator>();
private readonly oldestAllowedDate: Date = new Date(new Date().getTime() - 60 * 60 * 24 * 30 * 1000);
private readonly osmSourceZoomLevel = 14
private readonly osmSourceZoomLevel = 15
constructor(
handleFeatureSource: (source: FeatureSourceForLayer & Tiled) => void,
@ -147,7 +147,7 @@ export default class FeaturePipeline {
// We split them up into tiles anyway as it is an OSM source
TiledFeatureSource.createHierarchy(src, {
layer: src.layer,
minZoomLevel: 14,
minZoomLevel: this.osmSourceZoomLevel,
dontEnforceMinZoom: true,
registerTile: (tile) => {
new RegisteringAllFromFeatureSourceActor(tile)
@ -155,7 +155,7 @@ export default class FeaturePipeline {
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
}
})
}else{
} else {
new RegisteringAllFromFeatureSourceActor(src)
perLayerHierarchy.get(id).registerTile(src)
src.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(src))
@ -200,7 +200,7 @@ export default class FeaturePipeline {
new PerLayerFeatureSourceSplitter(state.filteredLayers,
(source) => TiledFeatureSource.createHierarchy(source, {
layer: source.layer,
minZoomLevel: 14,
minZoomLevel: source.layer.layerDef.minzoom,
dontEnforceMinZoom: true,
maxFeatureCount: state.layoutToUse.clustering.minNeededElements,
maxZoomLevel: state.layoutToUse.clustering.maxZoom,
@ -235,7 +235,7 @@ export default class FeaturePipeline {
// Whenever fresh data comes in, we need to update the metatagging
self.newDataLoadedSignal.stabilized(1000).addCallback(src => {
self.newDataLoadedSignal.stabilized(1000).addCallback(_ => {
self.updateAllMetaTagging()
})
@ -276,15 +276,15 @@ export default class FeaturePipeline {
const self = this
return this.state.currentBounds.map(bbox => {
if (bbox === undefined) {
return
return undefined
}
if (!isSufficientlyZoomed.data) {
return;
return undefined;
}
const osmSourceZoomLevel = self.osmSourceZoomLevel
const range = bbox.containingTileRange(osmSourceZoomLevel)
const tileIndexes = []
if (range.total > 100) {
if (range.total >= 100) {
// Too much tiles!
return []
}
@ -294,7 +294,7 @@ export default class FeaturePipeline {
if (oldestDate !== undefined && oldestDate > this.oldestAllowedDate) {
console.debug("Skipping tile", osmSourceZoomLevel, x, y, "as a decently fresh one is available")
// The cached tiles contain decently fresh data
return;
return undefined;
}
tileIndexes.push(i)
})
@ -327,28 +327,30 @@ export default class FeaturePipeline {
}
const range = bbox.containingTileRange(zoom)
if (range.total > 100) {
if (range.total >= 5000) {
return false
}
const self = this;
const allFreshnesses = Tiles.MapRange(range, (x, y) => self.freshnessForVisibleLayers(zoom, x, y))
return allFreshnesses.some(freshness => freshness === undefined || freshness < this.oldestAllowedDate)
}, [state.locationControl])
const self = this;
const updater = new OverpassFeatureSource(state,
{
padToTiles: state.locationControl.map(l => Math.min(15, l.zoom + 1)),
relationTracker: this.relationTracker,
isActive: useOsmApi.map(b => !b && overpassIsActive.data, [overpassIsActive]),
onBboxLoaded: ((bbox, date, downloadedLayers) => {
Tiles.MapRange(bbox.containingTileRange(self.osmSourceZoomLevel), (x, y) => {
onBboxLoaded: (bbox, date, downloadedLayers, paddedToZoomLevel) => {
Tiles.MapRange(bbox.containingTileRange(paddedToZoomLevel), (x, y) => {
const tileIndex = Tiles.tile_index(paddedToZoomLevel, x, y)
downloadedLayers.forEach(layer => {
SaveTileToLocalStorageActor.MarkVisited(layer.id, Tiles.tile_index(this.osmSourceZoomLevel, x, y), date)
self.freshnesses.get(layer.id).addTileLoad(tileIndex, date)
SaveTileToLocalStorageActor.MarkVisited(layer.id, tileIndex, date)
})
})
})
}
});

119
Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts
View file

@ -1,14 +1,16 @@
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import Loc from "../../../Models/Loc";
import TileHierarchy from "./TileHierarchy";
import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
export default class TiledFromLocalStorageSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
private readonly layer: FilteredLayer;
private readonly handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void;
private readonly undefinedTiles: Set<number>;
public static GetFreshnesses(layerId: string): Map<number, Date> {
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layerId + "-"
@ -29,14 +31,15 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
constructor(layer: FilteredLayer,
handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void,
state: {
locationControl: UIEventSource<Loc>
leafletMap: any
currentBounds: UIEventSource<BBox>
}) {
this.layer = layer;
this.handleFeatureSource = handleFeatureSource;
const undefinedTiles = new Set<number>()
this.undefinedTiles = new Set<number>()
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-"
// @ts-ignore
const indexes: number[] = Object.keys(localStorage)
const knownTiles: number[] = Object.keys(localStorage)
.filter(key => {
return key.startsWith(prefix) && !key.endsWith("-time") && !key.endsWith("-format");
})
@ -45,8 +48,8 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
})
.filter(i => !isNaN(i))
console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", indexes.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
for (const index of indexes) {
console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", knownTiles.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
for (const index of knownTiles) {
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + index;
const version = localStorage.getItem(prefix + "-format")
@ -55,78 +58,54 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
localStorage.removeItem(prefix)
localStorage.removeItem(prefix+"-time")
localStorage.removeItem(prefix+"-format")
undefinedTiles.add(index)
this. undefinedTiles.add(index)
console.log("Dropped old format tile", prefix)
}
}
const zLevels = indexes.map(i => i % 100)
const indexesSet = new Set(indexes)
const maxZoom = Math.max(...zLevels)
const minZoom = Math.min(...zLevels)
const self = this;
const self = this
state.currentBounds.map(bounds => {
const neededTiles = state.locationControl.map(
location => {
if (!layer.isDisplayed.data) {
// No need to download! - the layer is disabled
return undefined;
}
if (location.zoom < layer.layerDef.minzoom) {
// No need to download! - the layer is disabled
return undefined;
}
// Yup, this is cheating to just get the bounds here
const bounds = state.leafletMap.data?.getBounds()
if (bounds === undefined) {
// We'll retry later
return undefined
}
const needed = []
for (let z = minZoom; z <= maxZoom; z++) {
const tileRange = Tiles.TileRangeBetween(z, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const neededZ = Tiles.MapRange(tileRange, (x, y) => Tiles.tile_index(z, x, y))
.filter(i => !self.loadedTiles.has(i) && !undefinedTiles.has(i) && indexesSet.has(i))
needed.push(...neededZ)
}
if (needed.length === 0) {
return undefined
}
return needed
if(bounds === undefined){
return;
}
, [layer.isDisplayed, state.leafletMap]).stabilized(50);
neededTiles.addCallbackAndRunD(neededIndexes => {
for (const neededIndex of neededIndexes) {
// We load the features from localStorage
try {
const key = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + neededIndex
const data = localStorage.getItem(key)
const features = JSON.parse(data)
const src = {
layer: layer,
features: new UIEventSource<{ feature: any; freshness: Date }[]>(features),
name: "FromLocalStorage(" + key + ")",
tileIndex: neededIndex,
bbox: BBox.fromTileIndex(neededIndex)
}
handleFeatureSource(src, neededIndex)
self.loadedTiles.set(neededIndex, src)
} catch (e) {
console.error("Could not load data tile from local storage due to", e)
undefinedTiles.add(neededIndex)
for (const knownTile of knownTiles) {
if(this.loadedTiles.has(knownTile)){
continue;
}
if(this.undefinedTiles.has(knownTile)){
continue;
}
if(!bounds.overlapsWith(BBox.fromTileIndex(knownTile))){
continue;
}
self.loadTile(knownTile)
}
})
}
private loadTile( neededIndex: number){
try {
const key = SaveTileToLocalStorageActor.storageKey + "-" + this.layer.layerDef.id + "-" + neededIndex
const data = localStorage.getItem(key)
const features = JSON.parse(data)
const src = {
layer: this.layer,
features: new UIEventSource<{ feature: any; freshness: Date }[]>(features),
name: "FromLocalStorage(" + key + ")",
tileIndex: neededIndex,
bbox: BBox.fromTileIndex(neededIndex)
}
this.handleFeatureSource(src, neededIndex)
this.loadedTiles.set(neededIndex, src)
} catch (e) {
console.error("Could not load data tile from local storage due to", e)
this.undefinedTiles.add(neededIndex)
}
}
}

7
Logic/SimpleMetaTagger.ts
View file

@ -209,7 +209,7 @@ export default class SimpleMetaTagger {
configurable: true,
get: () => {
delete feature.properties._isOpen
feature.properties._isOpen = ""
feature.properties._isOpen = undefined
const tagsSource = State.state.allElements.getEventSourceById(feature.properties.id);
tagsSource.addCallbackAndRunD(tags => {
if (tags.opening_hours === undefined || tags._country === undefined) {
@ -230,7 +230,8 @@ export default class SimpleMetaTagger {
const oldNextChange = tags["_isOpen:nextTrigger"] ?? 0;
if (oldNextChange > (new Date()).getTime() &&
tags["_isOpen:oldvalue"] === tags["opening_hours"]) {
tags["_isOpen:oldvalue"] === tags["opening_hours"]
&& tags["_isOpen"] !== undefined) {
// Already calculated and should not yet be triggered
return false;
}
@ -267,7 +268,7 @@ export default class SimpleMetaTagger {
}
})
return feature.properties["_isOpen"]
return undefined
}
})