midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

refactoring

Browse source
This commit is contained in:
Pieter Vander Vennet 2023-03-28 05:13:48 +02:00
parent b94a8f5745
commit 5d0fe31c41
114 changed files with 2412 additions and 2958 deletions
Download patch file Download diff file Expand all files Collapse all files

41
Logic/FeatureSource/Actors/GeoIndexedStore.ts Normal file
View file

@ -0,0 +1,41 @@
import FeatureSource, { FeatureSourceForLayer } from "../FeatureSource"
import { Feature } from "geojson"
import { BBox } from "../../BBox"
import { GeoOperations } from "../../GeoOperations"
import { Store } from "../../UIEventSource"
import FilteredLayer from "../../../Models/FilteredLayer"
/**
* Allows the retrieval of all features in the requested BBox; useful for one-shot queries;
*
* Use a ClippedFeatureSource for a continuously updating featuresource
*/
export default class GeoIndexedStore implements FeatureSource {
public features: Store<Feature[]>
constructor(features: FeatureSource | Store<Feature[]>) {
this.features = features["features"] ?? features
}
/**
* Gets the current features within the given bbox.
*
* @param bbox
* @constructor
*/
public GetFeaturesWithin(bbox: BBox): Feature[] {
// TODO optimize
const bboxFeature = bbox.asGeoJson({})
return this.features.data.filter(
(f) => GeoOperations.intersect(f, bboxFeature) !== undefined
)
}
}
export class GeoIndexedStoreForLayer extends GeoIndexedStore implements FeatureSourceForLayer {
readonly layer: FilteredLayer
constructor(features: FeatureSource | Store<Feature[]>, layer: FilteredLayer) {
super(features)
this.layer = layer
}
}

3
Logic/FeatureSource/Actors/MetaTagRecalculator.ts
View file

@ -1,11 +1,10 @@
import { FeatureSourceForLayer, Tiled } from "../FeatureSource"
import MetaTagging from "../../MetaTagging"
import { ExtraFuncParams } from "../../ExtraFunctions"
import FeaturePipeline from "../FeaturePipeline"
import { BBox } from "../../BBox"
import { UIEventSource } from "../../UIEventSource"
/****
/**
* Concerned with the logic of updating the right layer at the right time
*/
class MetatagUpdater {

36
Logic/FeatureSource/Actors/SaveFeatureSourceToLocalStorage.ts Normal file
View file

@ -0,0 +1,36 @@
import FeatureSource, { Tiled } from "../FeatureSource"
import { Tiles } from "../../../Models/TileRange"
import { IdbLocalStorage } from "../../Web/IdbLocalStorage"
import { UIEventSource } from "../../UIEventSource"
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
import { BBox } from "../../BBox"
import SimpleFeatureSource from "../Sources/SimpleFeatureSource"
import FilteredLayer from "../../../Models/FilteredLayer"
import Loc from "../../../Models/Loc"
import { Feature } from "geojson"
import TileLocalStorage from "./TileLocalStorage"
import { GeoOperations } from "../../GeoOperations"
import { Utils } from "../../../Utils"
/***
* Saves all the features that are passed in to localstorage, so they can be retrieved on the next run
*
* The data is saved in a tiled way on a fixed zoomlevel and is retrievable per layer.
*
* Also see the sibling class
*/
export default class SaveFeatureSourceToLocalStorage {
constructor(layername: string, zoomlevel: number, features: FeatureSource) {
const storage = TileLocalStorage.construct<Feature[]>(layername)
features.features.addCallbackAndRunD((features) => {
const sliced = GeoOperations.slice(zoomlevel, features)
sliced.forEach((features, tileIndex) => {
const src = storage.getTileSource(tileIndex)
if (Utils.sameList(src.data, features)) {
return
}
src.setData(features)
})
})
}
}

149
Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts
View file

@ -1,149 +0,0 @@
import FeatureSource, { Tiled } from "../FeatureSource"
import { Tiles } from "../../../Models/TileRange"
import { IdbLocalStorage } from "../../Web/IdbLocalStorage"
import { UIEventSource } from "../../UIEventSource"
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
import { BBox } from "../../BBox"
import SimpleFeatureSource from "../Sources/SimpleFeatureSource"
import FilteredLayer from "../../../Models/FilteredLayer"
import Loc from "../../../Models/Loc"
import { Feature } from "geojson"
/***
* Saves all the features that are passed in to localstorage, so they can be retrieved on the next run
*
* Technically, more an Actor then a featuresource, but it fits more neatly this way
*/
export default class SaveTileToLocalStorageActor {
private readonly visitedTiles: UIEventSource<Map<number, Date>>
private readonly _layer: LayerConfig
private readonly _flayer: FilteredLayer
private readonly initializeTime = new Date()
constructor(layer: FilteredLayer) {
this._flayer = layer
this._layer = layer.layerDef
this.visitedTiles = IdbLocalStorage.Get("visited_tiles_" + this._layer.id, {
defaultValue: new Map<number, Date>(),
})
this.visitedTiles.stabilized(100).addCallbackAndRunD((tiles) => {
for (const key of Array.from(tiles.keys())) {
const tileFreshness = tiles.get(key)
const toOld =
this.initializeTime.getTime() - tileFreshness.getTime() >
1000 * this._layer.maxAgeOfCache
if (toOld) {
// Purge this tile
this.SetIdb(key, undefined)
console.debug("Purging tile", this._layer.id, key)
tiles.delete(key)
}
}
this.visitedTiles.ping()
return true
})
}
public LoadTilesFromDisk(
currentBounds: UIEventSource<BBox>,
location: UIEventSource<Loc>,
registerFreshness: (tileId: number, freshness: Date) => void,
registerTile: (src: FeatureSource & Tiled) => void
) {
const self = this
const loadedTiles = new Set<number>()
this.visitedTiles.addCallbackD((tiles) => {
if (tiles.size === 0) {
// We don't do anything yet as probably not yet loaded from disk
// We'll unregister later on
return
}
currentBounds.addCallbackAndRunD((bbox) => {
if (self._layer.minzoomVisible > location.data.zoom) {
// Not enough zoom
return
}
// Iterate over all available keys in the local storage, check which are needed and fresh enough
for (const key of Array.from(tiles.keys())) {
const tileFreshness = tiles.get(key)
if (tileFreshness > self.initializeTime) {
// This tile is loaded by another source
continue
}
registerFreshness(key, tileFreshness)
const tileBbox = BBox.fromTileIndex(key)
if (!bbox.overlapsWith(tileBbox)) {
continue
}
if (loadedTiles.has(key)) {
// Already loaded earlier
continue
}
loadedTiles.add(key)
this.GetIdb(key).then((features: Feature[]) => {
if (features === undefined) {
return
}
console.debug("Loaded tile " + self._layer.id + "_" + key + " from disk")
const src = new SimpleFeatureSource(
self._flayer,
key,
new UIEventSource<Feature[]>(features)
)
registerTile(src)
})
}
})
return true // Remove the callback
})
}
public addTile(tile: FeatureSource & Tiled) {
const self = this
tile.features.addCallbackAndRunD((features) => {
const now = new Date()
if (features.length > 0) {
self.SetIdb(tile.tileIndex, features)
}
// We _still_ write the time to know that this tile is empty!
this.MarkVisited(tile.tileIndex, now)
})
}
public poison(lon: number, lat: number) {
for (let z = 0; z < 25; z++) {
const { x, y } = Tiles.embedded_tile(lat, lon, z)
const tileId = Tiles.tile_index(z, x, y)
this.visitedTiles.data.delete(tileId)
}
}
public MarkVisited(tileId: number, freshness: Date) {
this.visitedTiles.data.set(tileId, freshness)
this.visitedTiles.ping()
}
private SetIdb(tileIndex, data) {
try {
IdbLocalStorage.SetDirectly(this._layer.id + "_" + tileIndex, data)
} catch (e) {
console.error(
"Could not save tile to indexed-db: ",
e,
"tileIndex is:",
tileIndex,
"for layer",
this._layer.id
)
}
}
private GetIdb(tileIndex) {
return IdbLocalStorage.GetDirectly(this._layer.id + "_" + tileIndex)
}
}

63
Logic/FeatureSource/Actors/TileLocalStorage.ts Normal file
View file

@ -0,0 +1,63 @@
import { IdbLocalStorage } from "../../Web/IdbLocalStorage"
import { UIEventSource } from "../../UIEventSource"
/**
* A class which allows to read/write a tile to local storage.
*
* Does the heavy lifting for LocalStorageFeatureSource and SaveFeatureToLocalStorage
*/
export default class TileLocalStorage<T> {
private static perLayer: Record<string, TileLocalStorage<any>> = {}
private readonly _layername: string
private readonly cachedSources: Record<number, UIEventSource<T>> = {}
private constructor(layername: string) {
this._layername = layername
}
public static construct<T>(layername: string): TileLocalStorage<T> {
const cached = TileLocalStorage.perLayer[layername]
if (cached) {
return cached
}
const tls = new TileLocalStorage<T>(layername)
TileLocalStorage.perLayer[layername] = tls
return tls
}
/**
* Constructs a UIEventSource element which is synced with localStorage
* @param layername
* @param tileIndex
*/
public getTileSource(tileIndex: number): UIEventSource<T> {
const cached = this.cachedSources[tileIndex]
if (cached) {
return cached
}
const src = UIEventSource.FromPromise(this.GetIdb(tileIndex))
src.addCallbackD((data) => this.SetIdb(tileIndex, data))
this.cachedSources[tileIndex] = src
return src
}
private SetIdb(tileIndex: number, data): void {
try {
IdbLocalStorage.SetDirectly(this._layername + "_" + tileIndex, data)
} catch (e) {
console.error(
"Could not save tile to indexed-db: ",
e,
"tileIndex is:",
tileIndex,
"for layer",
this._layername
)
}
}
private GetIdb(tileIndex: number): Promise<any> {
return IdbLocalStorage.GetDirectly(this._layername + "_" + tileIndex)
}
}

581
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -1,581 +0,0 @@
import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig"
import FilteringFeatureSource from "./Sources/FilteringFeatureSource"
import PerLayerFeatureSourceSplitter from "./PerLayerFeatureSourceSplitter"
import FeatureSource, { FeatureSourceForLayer, IndexedFeatureSource, Tiled } from "./FeatureSource"
import TiledFeatureSource from "./TiledFeatureSource/TiledFeatureSource"
import { Store, UIEventSource } from "../UIEventSource"
import { TileHierarchyTools } from "./TiledFeatureSource/TileHierarchy"
import RememberingSource from "./Sources/RememberingSource"
import OverpassFeatureSource from "../Actors/OverpassFeatureSource"
import GeoJsonSource from "./Sources/GeoJsonSource"
import Loc from "../../Models/Loc"
import RegisteringAllFromFeatureSourceActor from "./Actors/RegisteringAllFromFeatureSourceActor"
import SaveTileToLocalStorageActor from "./Actors/SaveTileToLocalStorageActor"
import DynamicGeoJsonTileSource from "./TiledFeatureSource/DynamicGeoJsonTileSource"
import { TileHierarchyMerger } from "./TiledFeatureSource/TileHierarchyMerger"
import { NewGeometryFromChangesFeatureSource } from "./Sources/NewGeometryFromChangesFeatureSource"
import ChangeGeometryApplicator from "./Sources/ChangeGeometryApplicator"
/**
* Keeps track of the age of the loaded data.
* Has one freshness-Calculator for every layer
* @private
*/
import { BBox } from "../BBox"
import OsmFeatureSource from "./TiledFeatureSource/OsmFeatureSource"
import { Tiles } from "../../Models/TileRange"
import FullNodeDatabaseSource from "./TiledFeatureSource/FullNodeDatabaseSource"
import MapState from "../State/MapState"
import { OsmFeature } from "../../Models/OsmFeature"
import LayerConfig from "../../Models/ThemeConfig/LayerConfig"
import { FilterState } from "../../Models/FilteredLayer"
import { GeoOperations } from "../GeoOperations"
import { Utils } from "../../Utils"
/**
* The features pipeline ties together a myriad of various datasources:
*
* - The Overpass-API
* - The OSM-API
* - Third-party geojson files, either sliced or directly.
*
* In order to truly understand this class, please have a look at the following diagram: https://cdn-images-1.medium.com/fit/c/800/618/1*qTK1iCtyJUr4zOyw4IFD7A.jpeg
*
*
*/
export default class FeaturePipeline {
public readonly sufficientlyZoomed: Store<boolean>
public readonly runningQuery: Store<boolean>
public readonly timeout: UIEventSource<number>
public readonly somethingLoaded: UIEventSource<boolean> = new UIEventSource<boolean>(false)
public readonly newDataLoadedSignal: UIEventSource<FeatureSource> =
new UIEventSource<FeatureSource>(undefined)
/**
* Keeps track of all raw OSM-nodes.
* Only initialized if `ReplaceGeometryAction` is needed somewhere
*/
public readonly fullNodeDatabase?: FullNodeDatabaseSource
private readonly overpassUpdater: OverpassFeatureSource
private state: MapState
private readonly perLayerHierarchy: Map<string, TileHierarchyMerger>
private readonly oldestAllowedDate: Date
private readonly osmSourceZoomLevel
private readonly localStorageSavers = new Map<string, SaveTileToLocalStorageActor>()
private readonly newGeometryHandler: NewGeometryFromChangesFeatureSource
constructor(
handleFeatureSource: (source: FeatureSourceForLayer & Tiled) => void,
state: MapState,
options?: {
/*Used for metatagging - will receive all the sources with changeGeometry applied but without filtering*/
handleRawFeatureSource: (source: FeatureSourceForLayer) => void
}
) {
this.state = state
const self = this
const expiryInSeconds = Math.min(
...(state.layoutToUse?.layers?.map((l) => l.maxAgeOfCache) ?? [])
)
this.oldestAllowedDate = new Date(new Date().getTime() - expiryInSeconds)
this.osmSourceZoomLevel = state.osmApiTileSize.data
const useOsmApi = state.locationControl.map(
(l) => l.zoom > (state.overpassMaxZoom.data ?? 12)
)
state.changes.allChanges.addCallbackAndRun((allChanges) => {
allChanges
.filter((ch) => ch.id < 0 && ch.changes !== undefined)
.map((ch) => ch.changes)
.filter((coor) => coor["lat"] !== undefined && coor["lon"] !== undefined)
.forEach((coor) => {
state.layoutToUse.layers.forEach((l) =>
self.localStorageSavers.get(l.id)?.poison(coor["lon"], coor["lat"])
)
})
})
this.sufficientlyZoomed = state.locationControl.map((location) => {
if (location?.zoom === undefined) {
return false
}
let minzoom = Math.min(
...state.filteredLayers.data.map((layer) => layer.layerDef.minzoom ?? 18)
)
return location.zoom >= minzoom
})
const neededTilesFromOsm = this.getNeededTilesFromOsm(this.sufficientlyZoomed)
const perLayerHierarchy = new Map<string, TileHierarchyMerger>()
this.perLayerHierarchy = perLayerHierarchy
// Given a tile, wraps it and passes it on to render (handled by 'handleFeatureSource'
function patchedHandleFeatureSource(
src: FeatureSourceForLayer & IndexedFeatureSource & Tiled
) {
// This will already contain the merged features for this tile. In other words, this will only be triggered once for every tile
const withChanges = new ChangeGeometryApplicator(src, state.changes)
const srcFiltered = new FilteringFeatureSource(state, src.tileIndex, withChanges)
handleFeatureSource(srcFiltered)
if (options?.handleRawFeatureSource) {
options.handleRawFeatureSource(withChanges)
}
self.somethingLoaded.setData(true)
// We do not mark as visited here, this is the responsability of the code near the actual loader (e.g. overpassLoader and OSMApiFeatureLoader)
}
for (const filteredLayer of state.filteredLayers.data) {
const id = filteredLayer.layerDef.id
const source = filteredLayer.layerDef.source
const hierarchy = new TileHierarchyMerger(filteredLayer, (tile, _) =>
patchedHandleFeatureSource(tile)
)
perLayerHierarchy.set(id, hierarchy)
if (id === "type_node") {
this.fullNodeDatabase = new FullNodeDatabaseSource(filteredLayer, (tile) => {
perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile)
tile.features.addCallbackAndRunD((_) => self.onNewDataLoaded(tile))
})
continue
}
const localTileSaver = new SaveTileToLocalStorageActor(filteredLayer)
this.localStorageSavers.set(filteredLayer.layerDef.id, localTileSaver)
if (source.geojsonSource === undefined) {
// This is an OSM layer
// We load the cached values and register them
// Getting data from upstream happens a bit lower
localTileSaver.LoadTilesFromDisk(
state.currentBounds,
state.locationControl,
(tileIndex, freshness) =>
self.freshnesses.get(id).addTileLoad(tileIndex, freshness),
(tile) => {
console.debug("Loaded tile ", id, tile.tileIndex, "from local cache")
new RegisteringAllFromFeatureSourceActor(tile, state.allElements)
hierarchy.registerTile(tile)
tile.features.addCallbackAndRunD((_) => self.onNewDataLoaded(tile))
}
)
continue
}
if (source.geojsonZoomLevel === undefined) {
// This is a 'load everything at once' geojson layer
const src = new GeoJsonSource(filteredLayer)
if (source.isOsmCacheLayer) {
// We split them up into tiles anyway as it is an OSM source
TiledFeatureSource.createHierarchy(src, {
layer: src.layer,
minZoomLevel: this.osmSourceZoomLevel,
noDuplicates: true,
registerTile: (tile) => {
new RegisteringAllFromFeatureSourceActor(tile, state.allElements)
perLayerHierarchy.get(id).registerTile(tile)
tile.features.addCallbackAndRunD((_) => self.onNewDataLoaded(tile))
},
})
} else {
new RegisteringAllFromFeatureSourceActor(src, state.allElements)
perLayerHierarchy.get(id).registerTile(src)
src.features.addCallbackAndRunD((_) => self.onNewDataLoaded(src))
}
} else {
new DynamicGeoJsonTileSource(
filteredLayer,
(tile) => {
new RegisteringAllFromFeatureSourceActor(tile, state.allElements)
perLayerHierarchy.get(id).registerTile(tile)
tile.features.addCallbackAndRunD((_) => self.onNewDataLoaded(tile))
},
state
)
}
}
const osmFeatureSource = new OsmFeatureSource({
isActive: useOsmApi,
neededTiles: neededTilesFromOsm,
handleTile: (tile) => {
new RegisteringAllFromFeatureSourceActor(tile, state.allElements)
if (tile.layer.layerDef.maxAgeOfCache > 0) {
const saver = self.localStorageSavers.get(tile.layer.layerDef.id)
if (saver === undefined) {
console.error(
"No localStorageSaver found for layer ",
tile.layer.layerDef.id
)
}
saver?.addTile(tile)
}
perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile)
tile.features.addCallbackAndRunD((_) => self.onNewDataLoaded(tile))
},
state: state,
markTileVisited: (tileId) =>
state.filteredLayers.data.forEach((flayer) => {
const layer = flayer.layerDef
if (layer.maxAgeOfCache > 0) {
const saver = self.localStorageSavers.get(layer.id)
if (saver === undefined) {
console.error("No local storage saver found for ", layer.id)
} else {
saver.MarkVisited(tileId, new Date())
}
}
self.freshnesses.get(layer.id).addTileLoad(tileId, new Date())
}),
})
if (this.fullNodeDatabase !== undefined) {
osmFeatureSource.rawDataHandlers.push((osmJson, tileId) =>
this.fullNodeDatabase.handleOsmJson(osmJson, tileId)
)
}
const updater = this.initOverpassUpdater(state, useOsmApi)
this.overpassUpdater = updater
this.timeout = updater.timeout
// Actually load data from the overpass source
new PerLayerFeatureSourceSplitter(
state.filteredLayers,
(source) =>
TiledFeatureSource.createHierarchy(source, {
layer: source.layer,
minZoomLevel: source.layer.layerDef.minzoom,
noDuplicates: true,
maxFeatureCount: state.layoutToUse.clustering.minNeededElements,
maxZoomLevel: state.layoutToUse.clustering.maxZoom,
registerTile: (tile) => {
// We save the tile data for the given layer to local storage - data sourced from overpass
self.localStorageSavers.get(tile.layer.layerDef.id)?.addTile(tile)
perLayerHierarchy
.get(source.layer.layerDef.id)
.registerTile(new RememberingSource(tile))
tile.features.addCallbackAndRunD((f) => {
if (f.length === 0) {
return
}
self.onNewDataLoaded(tile)
})
},
}),
updater,
{
handleLeftovers: (leftOvers) => {
console.warn("Overpass returned a few non-matched features:", leftOvers)
},
}
)
// Also load points/lines that are newly added.
const newGeometry = new NewGeometryFromChangesFeatureSource(
state.changes,
state.allElements,
state.osmConnection._oauth_config.url
)
this.newGeometryHandler = newGeometry
newGeometry.features.addCallbackAndRun((geometries) => {
console.debug("New geometries are:", geometries)
})
new RegisteringAllFromFeatureSourceActor(newGeometry, state.allElements)
// A NewGeometryFromChangesFeatureSource does not split per layer, so we do this next
new PerLayerFeatureSourceSplitter(
state.filteredLayers,
(perLayer) => {
// We don't bother to split them over tiles as it'll contain little features by default, so we simply add them like this
perLayerHierarchy.get(perLayer.layer.layerDef.id).registerTile(perLayer)
// AT last, we always apply the metatags whenever possible
perLayer.features.addCallbackAndRunD((_) => {
self.onNewDataLoaded(perLayer)
})
},
newGeometry,
{
handleLeftovers: (leftOvers) => {
console.warn("Got some leftovers from the filteredLayers: ", leftOvers)
},
}
)
this.runningQuery = updater.runningQuery.map(
(overpass) => {
console.log(
"FeaturePipeline: runningQuery state changed: Overpass",
overpass ? "is querying," : "is idle,",
"osmFeatureSource is",
osmFeatureSource.isRunning
? "is running and needs " +
neededTilesFromOsm.data?.length +
" tiles (already got " +
osmFeatureSource.downloadedTiles.size +
" tiles )"
: "is idle"
)
return overpass || osmFeatureSource.isRunning.data
},
[osmFeatureSource.isRunning]
)
}
public GetAllFeaturesWithin(bbox: BBox): OsmFeature[][] {
const self = this
const tiles: OsmFeature[][] = []
Array.from(this.perLayerHierarchy.keys()).forEach((key) => {
const fetched: OsmFeature[][] = self.GetFeaturesWithin(key, bbox)
tiles.push(...fetched)
})
return tiles
}
public GetAllFeaturesAndMetaWithin(
bbox: BBox,
layerIdWhitelist?: Set<string>
): { features: OsmFeature[]; layer: string }[] {
const self = this
const tiles: { features: any[]; layer: string }[] = []
Array.from(this.perLayerHierarchy.keys()).forEach((key) => {
if (layerIdWhitelist !== undefined && !layerIdWhitelist.has(key)) {
return
}
return tiles.push({
layer: key,
features: [].concat(...self.GetFeaturesWithin(key, bbox)),
})
})
return tiles
}
/**
* Gets all the tiles which overlap with the given BBOX.
* This might imply that extra features might be shown
*/
public GetFeaturesWithin(layerId: string, bbox: BBox): OsmFeature[][] {
if (layerId === "*") {
return this.GetAllFeaturesWithin(bbox)
}
const requestedHierarchy = this.perLayerHierarchy.get(layerId)
if (requestedHierarchy === undefined) {
console.warn(
"Layer ",
layerId,
"is not defined. Try one of ",
Array.from(this.perLayerHierarchy.keys())
)
return undefined
}
return TileHierarchyTools.getTiles(requestedHierarchy, bbox)
.filter((featureSource) => featureSource.features?.data !== undefined)
.map((featureSource) => <OsmFeature[]>featureSource.features.data)
}
public GetTilesPerLayerWithin(
bbox: BBox,
handleTile: (tile: FeatureSourceForLayer & Tiled) => void
) {
Array.from(this.perLayerHierarchy.values()).forEach((hierarchy) => {
TileHierarchyTools.getTiles(hierarchy, bbox).forEach(handleTile)
})
}
private onNewDataLoaded(src: FeatureSource) {
this.newDataLoadedSignal.setData(src)
}
private freshnessForVisibleLayers(z: number, x: number, y: number): Date {
let oldestDate = undefined
for (const flayer of this.state.filteredLayers.data) {
if (!flayer.isDisplayed.data && !flayer.layerDef.forceLoad) {
continue
}
if (this.state.locationControl.data.zoom < flayer.layerDef.minzoom) {
continue
}
if (flayer.layerDef.maxAgeOfCache === 0) {
return undefined
}
const freshnessCalc = this.freshnesses.get(flayer.layerDef.id)
if (freshnessCalc === undefined) {
console.warn("No freshness tracker found for ", flayer.layerDef.id)
return undefined
}
const freshness = freshnessCalc.freshnessFor(z, x, y)
if (freshness === undefined) {
// SOmething is undefined --> we return undefined as we have to download
return undefined
}
if (oldestDate === undefined || oldestDate > freshness) {
oldestDate = freshness
}
}
return oldestDate
}
/*
* Gives an UIEventSource containing the tileIndexes of the tiles that should be loaded from OSM
* */
private getNeededTilesFromOsm(isSufficientlyZoomed: Store<boolean>): Store<number[]> {
const self = this
return this.state.currentBounds.map(
(bbox) => {
if (bbox === undefined) {
return []
}
if (!isSufficientlyZoomed.data) {
return []
}
const osmSourceZoomLevel = self.osmSourceZoomLevel
const range = bbox.containingTileRange(osmSourceZoomLevel)
const tileIndexes = []
if (range.total >= 100) {
// Too much tiles!
return undefined
}
Tiles.MapRange(range, (x, y) => {
const i = Tiles.tile_index(osmSourceZoomLevel, x, y)
const oldestDate = self.freshnessForVisibleLayers(osmSourceZoomLevel, x, y)
if (oldestDate !== undefined && oldestDate > this.oldestAllowedDate) {
console.debug(
"Skipping tile",
osmSourceZoomLevel,
x,
y,
"as a decently fresh one is available"
)
// The cached tiles contain decently fresh data
return undefined
}
tileIndexes.push(i)
})
return tileIndexes
},
[isSufficientlyZoomed]
)
}
private initOverpassUpdater(
state: {
layoutToUse: LayoutConfig
currentBounds: Store<BBox>
locationControl: Store<Loc>
readonly overpassUrl: Store<string[]>
readonly overpassTimeout: Store<number>
readonly overpassMaxZoom: Store<number>
},
useOsmApi: Store<boolean>
): OverpassFeatureSource {
const minzoom = Math.min(...state.layoutToUse.layers.map((layer) => layer.minzoom))
const overpassIsActive = state.currentBounds.map(
(bbox) => {
if (bbox === undefined) {
console.debug("Disabling overpass source: no bbox")
return false
}
let zoom = state.locationControl.data.zoom
if (zoom < minzoom) {
// We are zoomed out over the zoomlevel of any layer
console.debug("Disabling overpass source: zoom < minzoom")
return false
}
const range = bbox.containingTileRange(zoom)
if (range.total >= 5000) {
// Let's assume we don't have so much data cached
return true
}
const self = this
const allFreshnesses = Tiles.MapRange(range, (x, y) =>
self.freshnessForVisibleLayers(zoom, x, y)
)
return allFreshnesses.some(
(freshness) => freshness === undefined || freshness < this.oldestAllowedDate
)
},
[state.locationControl]
)
return new OverpassFeatureSource(state, {
padToTiles: state.locationControl.map((l) => Math.min(15, l.zoom + 1)),
isActive: useOsmApi.map((b) => !b && overpassIsActive.data, [overpassIsActive]),
})
}
/**
* Builds upon 'GetAllFeaturesAndMetaWithin', but does stricter BBOX-checking and applies the filters
*/
public getAllVisibleElementsWithmeta(
bbox: BBox
): { center: [number, number]; element: OsmFeature; layer: LayerConfig }[] {
if (bbox === undefined) {
console.warn("No bbox")
return []
}
const layers = Utils.toIdRecord(this.state.layoutToUse.layers)
const elementsWithMeta: { features: OsmFeature[]; layer: string }[] =
this.GetAllFeaturesAndMetaWithin(bbox)
let elements: { center: [number, number]; element: OsmFeature; layer: LayerConfig }[] = []
let seenElements = new Set<string>()
for (const elementsWithMetaElement of elementsWithMeta) {
const layer = layers[elementsWithMetaElement.layer]
if (layer.title === undefined) {
continue
}
const filtered = this.state.filteredLayers.data.find((fl) => fl.layerDef == layer)
for (let i = 0; i < elementsWithMetaElement.features.length; i++) {
const element = elementsWithMetaElement.features[i]
if (!filtered.isDisplayed.data) {
continue
}
if (seenElements.has(element.properties.id)) {
continue
}
seenElements.add(element.properties.id)
if (!bbox.overlapsWith(BBox.get(element))) {
continue
}
if (layer?.isShown !== undefined && !layer.isShown.matchesProperties(element)) {
continue
}
const activeFilters: FilterState[] = Array.from(
filtered.appliedFilters.data.values()
)
if (
!activeFilters.every(
(filter) =>
filter?.currentFilter === undefined ||
filter?.currentFilter?.matchesProperties(element.properties)
)
) {
continue
}
const center = GeoOperations.centerpointCoordinates(element)
elements.push({
element,
center,
layer: layers[elementsWithMetaElement.layer],
})
}
}
return elements
}
/**
* Inject a new point
*/
InjectNewPoint(geojson) {
this.newGeometryHandler.features.data.push(geojson)
this.newGeometryHandler.features.ping()
}
}

5
Logic/FeatureSource/FeatureSource.ts
View file

@ -1,4 +1,4 @@
import { Store } from "../UIEventSource"
import { Store, UIEventSource } from "../UIEventSource"
import FilteredLayer from "../../Models/FilteredLayer"
import { BBox } from "../BBox"
import { Feature } from "geojson"
@ -6,6 +6,9 @@ import { Feature } from "geojson"
export default interface FeatureSource {
features: Store<Feature[]>
}
export interface WritableFeatureSource extends FeatureSource {
features: UIEventSource<Feature[]>
}
export interface Tiled {
tileIndex: number

64
Logic/FeatureSource/PerLayerFeatureSourceSplitter.ts
View file

@ -1,48 +1,59 @@
import FeatureSource from "./FeatureSource"
import { Store } from "../UIEventSource"
import FeatureSource, { FeatureSourceForLayer } from "./FeatureSource"
import FilteredLayer from "../../Models/FilteredLayer"
import SimpleFeatureSource from "./Sources/SimpleFeatureSource"
import { Feature } from "geojson"
import { Utils } from "../../Utils"
import { UIEventSource } from "../UIEventSource"
/**
* In some rare cases, some elements are shown on multiple layers (when 'passthrough' is enabled)
* If this is the case, multiple objects with a different _matching_layer_id are generated.
* In any case, this featureSource marks the objects with _matching_layer_id
*/
export default class PerLayerFeatureSourceSplitter {
export default class PerLayerFeatureSourceSplitter<
T extends FeatureSourceForLayer = SimpleFeatureSource
> {
public readonly perLayer: ReadonlyMap<string, T>
constructor(
layers: Store<FilteredLayer[]>,
handleLayerData: (source: FeatureSource, layer: FilteredLayer) => void,
layers: FilteredLayer[],
upstream: FeatureSource,
options?: {
tileIndex?: number
constructStore?: (features: UIEventSource<Feature[]>, layer: FilteredLayer) => T
handleLeftovers?: (featuresWithoutLayer: any[]) => void
}
) {
const knownLayers = new Map<string, SimpleFeatureSource>()
const knownLayers = new Map<string, T>()
this.perLayer = knownLayers
const layerSources = new Map<string, UIEventSource<Feature[]>>()
function update() {
const features = upstream.features?.data
const constructStore =
options?.constructStore ?? ((store, layer) => new SimpleFeatureSource(layer, store))
for (const layer of layers) {
const src = new UIEventSource<Feature[]>([])
layerSources.set(layer.layerDef.id, src)
knownLayers.set(layer.layerDef.id, <T>constructStore(src, layer))
}
upstream.features.addCallbackAndRunD((features) => {
if (features === undefined) {
return
}
if (layers.data === undefined || layers.data.length === 0) {
if (layers === undefined) {
return
}
// We try to figure out (for each feature) in which feature store it should be saved.
// Note that this splitter is only run when it is invoked by the overpass feature source, so we can't be sure in which layer it should go
const featuresPerLayer = new Map<string, Feature[]>()
const noLayerFound = []
const noLayerFound: Feature[] = []
for (const layer of layers.data) {
for (const layer of layers) {
featuresPerLayer.set(layer.layerDef.id, [])
}
for (const f of features) {
let foundALayer = false
for (const layer of layers.data) {
for (const layer of layers) {
if (layer.layerDef.source.osmTags.matchesProperties(f.properties)) {
// We have found our matching layer!
featuresPerLayer.get(layer.layerDef.id).push(f)
@ -60,7 +71,7 @@ export default class PerLayerFeatureSourceSplitter {
// At this point, we have our features per layer as a list
// We assign them to the correct featureSources
for (const layer of layers.data) {
for (const layer of layers) {
const id = layer.layerDef.id
const features = featuresPerLayer.get(id)
if (features === undefined) {
@ -68,25 +79,24 @@ export default class PerLayerFeatureSourceSplitter {
continue
}
let featureSource = knownLayers.get(id)
if (featureSource === undefined) {
// Not yet initialized - now is a good time
featureSource = new SimpleFeatureSource(layer)
featureSource.features.setData(features)
knownLayers.set(id, featureSource)
handleLayerData(featureSource, layer)
} else {
featureSource.features.setData(features)
const src = layerSources.get(id)
if (Utils.sameList(src.data, features)) {
return
}
src.setData(features)
}
// AT last, the leftovers are handled
if (options?.handleLeftovers !== undefined && noLayerFound.length > 0) {
options.handleLeftovers(noLayerFound)
}
}
})
}
layers.addCallback((_) => update())
upstream.features.addCallbackAndRunD((_) => update())
public forEach(f: (featureSource: FeatureSourceForLayer) => void) {
for (const fs of this.perLayer.values()) {
f(fs)
}
}
}

17
Logic/FeatureSource/Sources/ClippedFeatureSource.ts Normal file
View file

@ -0,0 +1,17 @@
import FeatureSource from "../FeatureSource"
import { Feature, Polygon } from "geojson"
import StaticFeatureSource from "./StaticFeatureSource"
import { GeoOperations } from "../../GeoOperations"
/**
* Returns a clipped version of the original geojson. Ways which partially intersect the given feature will be split up
*/
export default class ClippedFeatureSource extends StaticFeatureSource {
constructor(features: FeatureSource, clipTo: Feature<Polygon>) {
super(
features.features.mapD((features) => {
return [].concat(features.map((feature) => GeoOperations.clipWith(feature, clipTo)))
})
)
}
}

41
Logic/FeatureSource/Sources/FilteringFeatureSource.ts
View file

@ -1,15 +1,15 @@
import { Store, UIEventSource } from "../../UIEventSource"
import FilteredLayer, { FilterState } from "../../../Models/FilteredLayer"
import FilteredLayer from "../../../Models/FilteredLayer"
import FeatureSource from "../FeatureSource"
import { TagsFilter } from "../../Tags/TagsFilter"
import { Feature } from "geojson"
import { OsmTags } from "../../../Models/OsmFeature"
import { GlobalFilter } from "../../../Models/GlobalFilter"
export default class FilteringFeatureSource implements FeatureSource {
public features: UIEventSource<Feature[]> = new UIEventSource([])
private readonly upstream: FeatureSource
private readonly _fetchStore?: (id: String) => Store<OsmTags>
private readonly _globalFilters?: Store<{ filter: FilterState }[]>
private readonly _fetchStore?: (id: string) => Store<Record<string, string>>
private readonly _globalFilters?: Store<GlobalFilter[]>
private readonly _alreadyRegistered = new Set<Store<any>>()
private readonly _is_dirty = new UIEventSource(false)
private readonly _layer: FilteredLayer
@ -18,8 +18,8 @@ export default class FilteringFeatureSource implements FeatureSource {
constructor(
layer: FilteredLayer,
upstream: FeatureSource,
fetchStore?: (id: String) => Store<OsmTags>,
globalFilters?: Store<{ filter: FilterState }[]>,
fetchStore?: (id: string) => Store<Record<string, string>>,
globalFilters?: Store<GlobalFilter[]>,
metataggingUpdated?: Store<any>
) {
this.upstream = upstream
@ -32,9 +32,11 @@ export default class FilteringFeatureSource implements FeatureSource {
self.update()
})
layer.appliedFilters.addCallback((_) => {
self.update()
})
layer.appliedFilters.forEach((value) =>
value.addCallback((_) => {
self.update()
})
)
this._is_dirty.stabilized(1000).addCallbackAndRunD((dirty) => {
if (dirty) {
@ -58,7 +60,7 @@ export default class FilteringFeatureSource implements FeatureSource {
const layer = this._layer
const features: Feature[] = this.upstream.features.data ?? []
const includedFeatureIds = new Set<string>()
const globalFilters = self._globalFilters?.data?.map((f) => f.filter)
const globalFilters = self._globalFilters?.data?.map((f) => f)
const newFeatures = (features ?? []).filter((f) => {
self.registerCallback(f)
@ -71,19 +73,26 @@ export default class FilteringFeatureSource implements FeatureSource {
return false
}
const tagsFilter = Array.from(layer.appliedFilters?.data?.values() ?? [])
for (const filter of tagsFilter) {
const neededTags: TagsFilter = filter?.currentFilter
for (const filter of layer.layerDef.filters) {
const state = layer.appliedFilters.get(filter.id).data
if (state === undefined) {
continue
}
let neededTags: TagsFilter
if (typeof state === "string") {
// This filter uses fields
} else {
neededTags = filter.options[state].osmTags
}
if (neededTags !== undefined && !neededTags.matchesProperties(f.properties)) {
// Hidden by the filter on the layer itself - we want to hide it no matter what
return false
}
}
for (const filter of globalFilters ?? []) {
const neededTags: TagsFilter = filter?.currentFilter
for (const globalFilter of globalFilters ?? []) {
const neededTags = globalFilter.osmTags
if (neededTags !== undefined && !neededTags.matchesProperties(f.properties)) {
// Hidden by the filter on the layer itself - we want to hide it no matter what
return false
}
}

2
Logic/FeatureSource/Sources/GeoJsonSource.ts
View file

@ -58,7 +58,7 @@ export default class GeoJsonSource implements FeatureSource {
.replace("{x_max}", "" + bounds.maxLon)
}
const eventsource = new UIEventSource<Feature[]>(undefined)
const eventsource = new UIEventSource<Feature[]>([])
if (options?.isActive !== undefined) {
options.isActive.addCallbackAndRunD(async (active) => {
if (!active) {

51
Logic/FeatureSource/LayoutSource.ts → Logic/FeatureSource/Sources/LayoutSource.ts
View file

@ -1,14 +1,15 @@
import FeatureSource from "./FeatureSource"
import { Store } from "../UIEventSource"
import FeatureSwitchState from "../State/FeatureSwitchState"
import OverpassFeatureSource from "../Actors/OverpassFeatureSource"
import { BBox } from "../BBox"
import OsmFeatureSource from "./TiledFeatureSource/OsmFeatureSource"
import { Or } from "../Tags/Or"
import FeatureSourceMerger from "./Sources/FeatureSourceMerger"
import LayerConfig from "../../Models/ThemeConfig/LayerConfig"
import GeoJsonSource from "./Sources/GeoJsonSource"
import DynamicGeoJsonTileSource from "./TiledFeatureSource/DynamicGeoJsonTileSource"
import GeoJsonSource from "./GeoJsonSource"
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
import FeatureSource from "../FeatureSource"
import { Or } from "../../Tags/Or"
import FeatureSwitchState from "../../State/FeatureSwitchState"
import OverpassFeatureSource from "./OverpassFeatureSource"
import { Store } from "../../UIEventSource"
import OsmFeatureSource from "./OsmFeatureSource"
import FeatureSourceMerger from "./FeatureSourceMerger"
import DynamicGeoJsonTileSource from "../TiledFeatureSource/DynamicGeoJsonTileSource"
import { BBox } from "../../BBox"
import LocalStorageFeatureSource from "../TiledFeatureSource/LocalStorageFeatureSource"
/**
* This source will fetch the needed data from various sources for the given layout.
@ -17,22 +18,24 @@ import DynamicGeoJsonTileSource from "./TiledFeatureSource/DynamicGeoJsonTileSou
*/
export default class LayoutSource extends FeatureSourceMerger {
constructor(
filteredLayers: LayerConfig[],
layers: LayerConfig[],
featureSwitches: FeatureSwitchState,
newAndChangedElements: FeatureSource,
mapProperties: { bounds: Store<BBox>; zoom: Store<number> },
backend: string,
isLayerActive: (id: string) => Store<boolean>
isDisplayed: (id: string) => Store<boolean>
) {
const { bounds, zoom } = mapProperties
// remove all 'special' layers
filteredLayers = filteredLayers.filter((flayer) => flayer.source !== null)
layers = layers.filter((flayer) => flayer.source !== null)
const geojsonlayers = filteredLayers.filter(
(flayer) => flayer.source.geojsonSource !== undefined
)
const osmLayers = filteredLayers.filter(
(flayer) => flayer.source.geojsonSource === undefined
const geojsonlayers = layers.filter((layer) => layer.source.geojsonSource !== undefined)
const osmLayers = layers.filter((layer) => layer.source.geojsonSource === undefined)
const fromCache = osmLayers.map(
(l) =>
new LocalStorageFeatureSource(l.id, 15, mapProperties, {
isActive: isDisplayed(l.id),
})
)
const overpassSource = LayoutSource.setupOverpass(osmLayers, bounds, zoom, featureSwitches)
const osmApiSource = LayoutSource.setupOsmApiSource(
@ -43,11 +46,11 @@ export default class LayoutSource extends FeatureSourceMerger {
featureSwitches
)
const geojsonSources: FeatureSource[] = geojsonlayers.map((l) =>
LayoutSource.setupGeojsonSource(l, mapProperties)
LayoutSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id))
)
const expiryInSeconds = Math.min(...(filteredLayers?.map((l) => l.maxAgeOfCache) ?? []))
super(overpassSource, osmApiSource, newAndChangedElements, ...geojsonSources)
const expiryInSeconds = Math.min(...(layers?.map((l) => l.maxAgeOfCache) ?? []))
super(overpassSource, osmApiSource, newAndChangedElements, ...geojsonSources, ...fromCache)
}
private static setupGeojsonSource(
@ -56,6 +59,10 @@ export default class LayoutSource extends FeatureSourceMerger {
isActive?: Store<boolean>
): FeatureSource {
const source = layer.source
isActive = mapProperties.zoom.map(
(z) => (isActive?.data ?? true) && z >= layer.maxzoom,
[isActive]
)
if (source.geojsonZoomLevel === undefined) {
// This is a 'load everything at once' geojson layer
return new GeoJsonSource(layer, { isActive })

3
Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts → Logic/FeatureSource/Sources/OsmFeatureSource.ts
View file

@ -108,6 +108,7 @@ export default class OsmFeatureSource extends FeatureSourceMerger {
}
private async LoadTile(z, x, y): Promise<void> {
console.log("OsmFeatureSource: loading ", z, x, y)
if (z >= 22) {
throw "This is an absurd high zoom level"
}
@ -126,7 +127,7 @@ export default class OsmFeatureSource extends FeatureSourceMerger {
let error = undefined
try {
const osmJson = await Utils.downloadJson(url)
const osmJson = await Utils.downloadJsonCached(url, 2000)
try {
this.rawDataHandlers.forEach((handler) =>
handler(osmJson, Tiles.tile_index(z, x, y))

208
Logic/FeatureSource/Sources/OverpassFeatureSource.ts Normal file
View file

@ -0,0 +1,208 @@
import { Feature } from "geojson"
import FeatureSource from "../FeatureSource"
import { ImmutableStore, Store, UIEventSource } from "../../UIEventSource"
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
import { Or } from "../../Tags/Or"
import LayoutConfig from "../../../Models/ThemeConfig/LayoutConfig"
import { Overpass } from "../../Osm/Overpass"
import { Utils } from "../../../Utils"
import { TagsFilter } from "../../Tags/TagsFilter"
import { BBox } from "../../BBox"
/**
* A wrapper around the 'Overpass'-object.
* It has more logic and will automatically fetch the data for the right bbox and the active layers
*/
export default class OverpassFeatureSource implements FeatureSource {
/**
* The last loaded features, as geojson
*/
public readonly features: UIEventSource<Feature[]> = new UIEventSource(undefined)
public readonly runningQuery: UIEventSource<boolean> = new UIEventSource<boolean>(false)
public readonly timeout: UIEventSource<number> = new UIEventSource<number>(0)
private readonly retries: UIEventSource<number> = new UIEventSource<number>(0)
private readonly state: {
readonly zoom: Store<number>
readonly layoutToUse: LayoutConfig
readonly overpassUrl: Store<string[]>
readonly overpassTimeout: Store<number>
readonly bounds: Store<BBox>
}
private readonly _isActive: Store<boolean>
private readonly padToZoomLevel?: Store<number>
private _lastQueryBBox: BBox
constructor(
state: {
readonly layoutToUse: LayoutConfig
readonly zoom: Store<number>
readonly overpassUrl: Store<string[]>
readonly overpassTimeout: Store<number>
readonly overpassMaxZoom: Store<number>
readonly bounds: Store<BBox>
},
options?: {
padToTiles?: Store<number>
isActive?: Store<boolean>
}
) {
this.state = state
this._isActive = options?.isActive ?? new ImmutableStore(true)
this.padToZoomLevel = options?.padToTiles
const self = this
state.bounds.addCallbackD((_) => {
self.updateAsyncIfNeeded()
})
}
/**
* Creates the 'Overpass'-object for the given layers
* @param interpreterUrl
* @param layersToDownload
* @constructor
* @private
*/
private GetFilter(interpreterUrl: string, layersToDownload: LayerConfig[]): Overpass {
let filters: TagsFilter[] = layersToDownload.map((layer) => layer.source.osmTags)
filters = Utils.NoNull(filters)
if (filters.length === 0) {
return undefined
}
return new Overpass(new Or(filters), [], interpreterUrl, this.state.overpassTimeout)
}
/**
*
* @private
*/
private async updateAsyncIfNeeded(): Promise<void> {
if (!this._isActive?.data) {
console.log("OverpassFeatureSource: not triggering as not active")
return
}
if (this.runningQuery.data) {
console.log("Still running a query, not updating")
return undefined
}
if (this.timeout.data > 0) {
console.log("Still in timeout - not updating")
return undefined
}
const requestedBounds = this.state.bounds.data
if (
this._lastQueryBBox !== undefined &&
requestedBounds.isContainedIn(this._lastQueryBBox)
) {
return undefined
}
const result = await this.updateAsync()
if (!result) {
return
}
const [bounds, date, updatedLayers] = result
this._lastQueryBBox = bounds
}
/**
* Download the relevant data from overpass. Attempt to use a different server; only downloads the relevant layers
* @private
*/
private async updateAsync(): Promise<[BBox, Date, LayerConfig[]]> {
let data: any = undefined
let date: Date = undefined
let lastUsed = 0
const layersToDownload = []
for (const layer of this.state.layoutToUse.layers) {
if (typeof layer === "string") {
throw "A layer was not expanded!"
}
if (layer.source === undefined) {
continue
}
if (this.state.zoom.data < layer.minzoom) {
continue
}
if (layer.doNotDownload) {
continue
}
if (layer.source.geojsonSource !== undefined) {
// Not our responsibility to download this layer!
continue
}
layersToDownload.push(layer)
}
if (layersToDownload.length == 0) {
return
}
const self = this
const overpassUrls = self.state.overpassUrl.data
if (overpassUrls === undefined || overpassUrls.length === 0) {
throw "Panic: overpassFeatureSource didn't receive any overpassUrls"
}
// Note: the bounds are updated between attempts, in case that the user zoomed around
let bounds: BBox
do {
try {
bounds = this.state.bounds.data
?.pad(this.state.layoutToUse.widenFactor)
?.expandToTileBounds(this.padToZoomLevel?.data)
if (bounds === undefined) {
return undefined
}
const overpass = this.GetFilter(overpassUrls[lastUsed], layersToDownload)
if (overpass === undefined) {
return undefined
}
this.runningQuery.setData(true)
;[data, date] = await overpass.queryGeoJson(bounds)
} catch (e) {
self.retries.data++
self.retries.ping()
console.error(`QUERY FAILED due to`, e)
await Utils.waitFor(1000)
if (lastUsed + 1 < overpassUrls.length) {
lastUsed++
console.log("Trying next time with", overpassUrls[lastUsed])
} else {
lastUsed = 0
self.timeout.setData(self.retries.data * 5)
while (self.timeout.data > 0) {
await Utils.waitFor(1000)
self.timeout.data--
self.timeout.ping()
}
}
}
} while (data === undefined && this._isActive.data)
try {
if (data === undefined) {
return undefined
}
// Some metatags are delivered by overpass _without_ underscore-prefix; we fix them below
// TODO FIXME re-enable this data.features.forEach((f) => SimpleMetaTaggers.objectMetaInfo.applyMetaTagsOnFeature(f))
self.features.setData(data.features)
return [bounds, date, layersToDownload]
} catch (e) {
console.error("Got the overpass response, but could not process it: ", e, e.stack)
return undefined
} finally {
self.retries.setData(0)
self.runningQuery.setData(false)
}
}
}

34
Logic/FeatureSource/Sources/RememberingSource.ts
View file

@ -1,34 +0,0 @@
/**
* Every previously added point is remembered, but new points are added.
* Data coming from upstream will always overwrite a previous value
*/
import FeatureSource, { Tiled } from "../FeatureSource"
import { Store, UIEventSource } from "../../UIEventSource"
import { BBox } from "../../BBox"
import { Feature } from "geojson"
export default class RememberingSource implements FeatureSource, Tiled {
public readonly features: Store<Feature[]>
public readonly tileIndex: number
public readonly bbox: BBox
constructor(source: FeatureSource & Tiled) {
const self = this
this.tileIndex = source.tileIndex
this.bbox = source.bbox
const empty = []
const featureSource = new UIEventSource<Feature[]>(empty)
this.features = featureSource
source.features.addCallbackAndRunD((features) => {
const oldFeatures = self.features?.data ?? empty
// Then new ids
const ids = new Set<string>(features.map((f) => f.properties.id + f.geometry.type))
// the old data
const oldData = oldFeatures.filter(
(old) => !ids.has(old.feature.properties.id + old.feature.geometry.type)
)
featureSource.setData([...features, ...oldData])
})
}
}

29
Logic/FeatureSource/Sources/TouchesBboxFeatureSource.ts Normal file
View file

@ -0,0 +1,29 @@
import FeatureSource, { FeatureSourceForLayer } from "../FeatureSource"
import StaticFeatureSource from "./StaticFeatureSource"
import { GeoOperations } from "../../GeoOperations"
import { BBox } from "../../BBox"
import exp from "constants"
import FilteredLayer from "../../../Models/FilteredLayer"
/**
* Results in a feature source which has all the elements that touch the given features
*/
export default class BBoxFeatureSource extends StaticFeatureSource {
constructor(features: FeatureSource, mustTouch: BBox) {
const bbox = mustTouch.asGeoJson({})
super(
features.features.mapD((features) =>
features.filter((feature) => GeoOperations.intersect(feature, bbox) !== undefined)
)
)
}
}
export class BBoxFeatureSourceForLayer extends BBoxFeatureSource implements FeatureSourceForLayer {
constructor(features: FeatureSourceForLayer, mustTouch: BBox) {
super(features, mustTouch)
this.layer = features.layer
}
readonly layer: FilteredLayer
}

4
Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts
View file

@ -84,7 +84,9 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
})
},
mapProperties,
{ isActive: options.isActive }
{
isActive: options?.isActive,
}
)
}
}

3
Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts
View file

@ -5,7 +5,8 @@ import FeatureSource from "../FeatureSource"
import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
/***
* A tiled source which dynamically loads the required tiles at a fixed zoom level
* A tiled source which dynamically loads the required tiles at a fixed zoom level.
* A single featureSource will be initiliased for every tile in view; which will alter be merged into this featureSource
*/
export default class DynamicTileSource extends FeatureSourceMerger {
constructor(

11
Logic/FeatureSource/TiledFeatureSource/FullNodeDatabaseSource.ts
View file

@ -1,11 +1,13 @@
import TileHierarchy from "./TileHierarchy"
import FeatureSource, { FeatureSourceForLayer, Tiled } from "../FeatureSource"
import { OsmNode, OsmObject, OsmWay } from "../../Osm/OsmObject"
import SimpleFeatureSource from "../Sources/SimpleFeatureSource"
import FilteredLayer from "../../../Models/FilteredLayer"
import { UIEventSource } from "../../UIEventSource"
import { OsmTags } from "../../../Models/OsmFeature";
import { BBox } from "../../BBox";
import { Feature, Point } from "geojson";
export default class FullNodeDatabaseSource implements TileHierarchy<FeatureSource & Tiled> {
export default class FullNodeDatabaseSource {
public readonly loadedTiles = new Map<number, FeatureSource & Tiled>()
private readonly onTileLoaded: (tile: Tiled & FeatureSourceForLayer) => void
private readonly layer: FilteredLayer
@ -81,4 +83,9 @@ export default class FullNodeDatabaseSource implements TileHierarchy<FeatureSour
public GetParentWays(nodeId: number): UIEventSource<OsmWay[]> {
return this.parentWays.get(nodeId)
}
getNodesWithin(bBox: BBox) : Feature<Point, OsmTags>[]{
// TODO
throw "TODO"
}
}

28
Logic/FeatureSource/TiledFeatureSource/LocalStorageFeatureSource.ts Normal file
View file

@ -0,0 +1,28 @@
import DynamicTileSource from "./DynamicTileSource"
import { Store } from "../../UIEventSource"
import { BBox } from "../../BBox"
import TileLocalStorage from "../Actors/TileLocalStorage"
import { Feature } from "geojson"
import StaticFeatureSource from "../Sources/StaticFeatureSource"
export default class LocalStorageFeatureSource extends DynamicTileSource {
constructor(
layername: string,
zoomlevel: number,
mapProperties: {
bounds: Store<BBox>
zoom: Store<number>
},
options?: {
isActive?: Store<boolean>
}
) {
const storage = TileLocalStorage.construct<Feature[]>(layername)
super(
zoomlevel,
(tileIndex) => new StaticFeatureSource(storage.getTileSource(tileIndex)),
mapProperties,
options
)
}
}

24
Logic/FeatureSource/TiledFeatureSource/README.md
View file

@ -1,24 +0,0 @@
Data in MapComplete can come from multiple sources.
Currently, they are:
- The Overpass-API
- The OSM-API
- One or more GeoJSON files. This can be a single file or a set of tiled geojson files
- LocalStorage, containing features from a previous visit
- Changes made by the user introducing new features
When the data enters from Overpass or from the OSM-API, they are first distributed per layer:
OVERPASS | ---PerLayerFeatureSource---> FeatureSourceForLayer[]
OSM |
The GeoJSon files (not tiled) are then added to this list
A single FeatureSourcePerLayer is then further handled by splitting it into a tile hierarchy.
In order to keep thins snappy, they are distributed over a tiled database per layer.
## Notes
`cached-featuresbookcases` is the old key used `cahced-features{themeid}` and should be cleaned up

24
Logic/FeatureSource/TiledFeatureSource/TileHierarchy.ts
View file

@ -1,24 +0,0 @@
import FeatureSource, { Tiled } from "../FeatureSource"
import { BBox } from "../../BBox"
export default interface TileHierarchy<T extends FeatureSource> {
/**
* A mapping from 'tile_index' to the actual tile featrues
*/
loadedTiles: Map<number, T>
}
export class TileHierarchyTools {
public static getTiles<T extends FeatureSource & Tiled>(
hierarchy: TileHierarchy<T>,
bbox: BBox
): T[] {
const result: T[] = []
hierarchy.loadedTiles.forEach((tile) => {
if (tile.bbox.overlapsWith(bbox)) {
result.push(tile)
}
})
return result
}
}

58
Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts
View file

@ -1,58 +0,0 @@
import TileHierarchy from "./TileHierarchy"
import { UIEventSource } from "../../UIEventSource"
import FeatureSource, { FeatureSourceForLayer, IndexedFeatureSource, Tiled } from "../FeatureSource"
import FilteredLayer from "../../../Models/FilteredLayer"
import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
import { Tiles } from "../../../Models/TileRange"
import { BBox } from "../../BBox"
export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<
number,
FeatureSourceForLayer & Tiled
>()
public readonly layer: FilteredLayer
private readonly sources: Map<number, UIEventSource<FeatureSource[]>> = new Map<
number,
UIEventSource<FeatureSource[]>
>()
private _handleTile: (src: FeatureSourceForLayer & IndexedFeatureSource, index: number) => void
constructor(
layer: FilteredLayer,
handleTile: (
src: FeatureSourceForLayer & IndexedFeatureSource & Tiled,
index: number
) => void
) {
this.layer = layer
this._handleTile = handleTile
}
/**
* Add another feature source for the given tile.
* Entries for this tile will be merged
* @param src
*/
public registerTile(src: FeatureSource & Tiled) {
const index = src.tileIndex
if (this.sources.has(index)) {
const sources = this.sources.get(index)
sources.data.push(src)
sources.ping()
return
}
// We have to setup
const sources = new UIEventSource<FeatureSource[]>([src])
this.sources.set(index, sources)
const merger = new FeatureSourceMerger(
this.layer,
index,
BBox.fromTile(...Tiles.tile_from_index(index)),
sources
)
this.loadedTiles.set(index, merger)
this._handleTile(merger, index)
}
}

249
Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource.ts
View file

@ -1,249 +0,0 @@
import FeatureSource, { FeatureSourceForLayer, IndexedFeatureSource, Tiled } from "../FeatureSource"
import { Store, UIEventSource } from "../../UIEventSource"
import FilteredLayer from "../../../Models/FilteredLayer"
import TileHierarchy from "./TileHierarchy"
import { Tiles } from "../../../Models/TileRange"
import { BBox } from "../../BBox"
import { Feature } from "geojson";
/**
* Contains all features in a tiled fashion.
* The data will be automatically broken down into subtiles when there are too much features in a single tile or if the zoomlevel is too high
*/
export default class TiledFeatureSource
implements
Tiled,
IndexedFeatureSource,
FeatureSourceForLayer,
TileHierarchy<IndexedFeatureSource & FeatureSourceForLayer & Tiled>
{
public readonly z: number
public readonly x: number
public readonly y: number
public readonly parent: TiledFeatureSource
public readonly root: TiledFeatureSource
public readonly layer: FilteredLayer
/* An index of all known tiles. allTiles[z][x][y].get('layerid') will yield the corresponding tile.
* Only defined on the root element!
*/
public readonly loadedTiles: Map<number, TiledFeatureSource & FeatureSourceForLayer> = undefined
public readonly maxFeatureCount: number
public readonly name
public readonly features: UIEventSource<Feature[]>
public readonly containedIds: Store<Set<string>>
public readonly bbox: BBox
public readonly tileIndex: number
private upper_left: TiledFeatureSource
private upper_right: TiledFeatureSource
private lower_left: TiledFeatureSource
private lower_right: TiledFeatureSource
private readonly maxzoom: number
private readonly options: TiledFeatureSourceOptions
private constructor(
z: number,
x: number,
y: number,
parent: TiledFeatureSource,
options?: TiledFeatureSourceOptions
) {
this.z = z
this.x = x
this.y = y
this.bbox = BBox.fromTile(z, x, y)
this.tileIndex = Tiles.tile_index(z, x, y)
this.name = `TiledFeatureSource(${z},${x},${y})`
this.parent = parent
this.layer = options.layer
options = options ?? {}
this.maxFeatureCount = options?.maxFeatureCount ?? 250
this.maxzoom = options.maxZoomLevel ?? 18
this.options = options
if (parent === undefined) {
throw "Parent is not allowed to be undefined. Use null instead"
}
if (parent === null && z !== 0 && x !== 0 && y !== 0) {
throw "Invalid root tile: z, x and y should all be null"
}
if (parent === null) {
this.root = this
this.loadedTiles = new Map()
} else {
this.root = this.parent.root
this.loadedTiles = this.root.loadedTiles
const i = Tiles.tile_index(z, x, y)
this.root.loadedTiles.set(i, this)
}
this.features = new UIEventSource<any[]>([])
this.containedIds = this.features.map((features) => {
if (features === undefined) {
return undefined
}
return new Set(features.map((f) => f.properties.id))
})
// We register this tile, but only when there is some data in it
if (this.options.registerTile !== undefined) {
this.features.addCallbackAndRunD((features) => {
if (features.length === 0) {
return
}
this.options.registerTile(this)
return true
})
}
}
public static createHierarchy(
features: FeatureSource,
options?: TiledFeatureSourceOptions
): TiledFeatureSource {
options = {
...options,
layer: features["layer"] ?? options.layer,
}
const root = new TiledFeatureSource(0, 0, 0, null, options)
features.features?.addCallbackAndRunD((feats) => root.addFeatures(feats))
return root
}
private isSplitNeeded(featureCount: number) {
if (this.upper_left !== undefined) {
// This tile has been split previously, so we keep on splitting
return true
}
if (this.z >= this.maxzoom) {
// We are not allowed to split any further
return false
}
if (this.options.minZoomLevel !== undefined && this.z < this.options.minZoomLevel) {
// We must have at least this zoom level before we are allowed to start splitting
return true
}
// To much features - we split
return featureCount > this.maxFeatureCount
}
/***
* Adds the list of features to this hierarchy.
* If there are too much features, the list will be broken down and distributed over the subtiles (only retaining features that don't fit a subtile on this level)
* @param features
* @private
*/
private addFeatures(features: Feature[]) {
if (features === undefined || features.length === 0) {
return
}
if (!this.isSplitNeeded(features.length)) {
this.features.setData(features)
return
}
if (this.upper_left === undefined) {
this.upper_left = new TiledFeatureSource(
this.z + 1,
this.x * 2,
this.y * 2,
this,
this.options
)
this.upper_right = new TiledFeatureSource(
this.z + 1,
this.x * 2 + 1,
this.y * 2,
this,
this.options
)
this.lower_left = new TiledFeatureSource(
this.z + 1,
this.x * 2,
this.y * 2 + 1,
this,
this.options
)
this.lower_right = new TiledFeatureSource(
this.z + 1,
this.x * 2 + 1,
this.y * 2 + 1,
this,
this.options
)
}
const ulf = []
const urf = []
const llf = []
const lrf = []
const overlapsboundary = []
for (const feature of features) {
const bbox = BBox.get(feature)
// There are a few strategies to deal with features that cross tile boundaries
if (this.options.noDuplicates) {
// Strategy 1: We put the feature into a somewhat matching tile
if (bbox.overlapsWith(this.upper_left.bbox)) {
ulf.push(feature)
} else if (bbox.overlapsWith(this.upper_right.bbox)) {
urf.push(feature)
} else if (bbox.overlapsWith(this.lower_left.bbox)) {
llf.push(feature)
} else if (bbox.overlapsWith(this.lower_right.bbox)) {
lrf.push(feature)
} else {
overlapsboundary.push(feature)
}
} else if (this.options.minZoomLevel === undefined) {
// Strategy 2: put it into a strictly matching tile (or in this tile, which is slightly too big)
if (bbox.isContainedIn(this.upper_left.bbox)) {
ulf.push(feature)
} else if (bbox.isContainedIn(this.upper_right.bbox)) {
urf.push(feature)
} else if (bbox.isContainedIn(this.lower_left.bbox)) {
llf.push(feature)
} else if (bbox.isContainedIn(this.lower_right.bbox)) {
lrf.push(feature)
} else {
overlapsboundary.push(feature)
}
} else {
// Strategy 3: We duplicate a feature on a boundary into every tile as we need to get to the minZoomLevel
if (bbox.overlapsWith(this.upper_left.bbox)) {
ulf.push(feature)
}
if (bbox.overlapsWith(this.upper_right.bbox)) {
urf.push(feature)
}
if (bbox.overlapsWith(this.lower_left.bbox)) {
llf.push(feature)
}
if (bbox.overlapsWith(this.lower_right.bbox)) {
lrf.push(feature)
}
}
}
this.upper_left.addFeatures(ulf)
this.upper_right.addFeatures(urf)
this.lower_left.addFeatures(llf)
this.lower_right.addFeatures(lrf)
this.features.setData(overlapsboundary)
}
}
export interface TiledFeatureSourceOptions {
readonly maxFeatureCount?: number
readonly maxZoomLevel?: number
readonly minZoomLevel?: number
/**
* IF minZoomLevel is set, and if a feature runs through a tile boundary, it would normally be duplicated.
* Setting 'dontEnforceMinZoomLevel' will assign to feature to some matching subtile.
*/
readonly noDuplicates?: boolean
readonly registerTile?: (tile: TiledFeatureSource & FeatureSourceForLayer & Tiled) => void
readonly layer?: FilteredLayer
}