midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

refactoring: more state splitting, basic layoutFeatureSource

Browse source
This commit is contained in:
Pieter Vander Vennet 2023-03-26 05:58:28 +02:00
parent 8e2f04c0d0
commit b94a8f5745
54 changed files with 1067 additions and 1969 deletions
Download patch file Download diff file Expand all files Collapse all files

44
Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts
View file

@ -1,23 +1,24 @@
import FilteredLayer from "../../../Models/FilteredLayer"
import { FeatureSourceForLayer, Tiled } from "../FeatureSource"
import { UIEventSource } from "../../UIEventSource"
import { Store } from "../../UIEventSource"
import DynamicTileSource from "./DynamicTileSource"
import { Utils } from "../../../Utils"
import GeoJsonSource from "../Sources/GeoJsonSource"
import { BBox } from "../../BBox"
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
export default class DynamicGeoJsonTileSource extends DynamicTileSource {
private static whitelistCache = new Map<string, any>()
constructor(
layer: FilteredLayer,
registerLayer: (layer: FeatureSourceForLayer & Tiled) => void,
state: {
locationControl?: UIEventSource<{ zoom?: number }>
currentBounds: UIEventSource<BBox>
layer: LayerConfig,
mapProperties: {
zoom: Store<number>
bounds: Store<BBox>
},
options?: {
isActive?: Store<boolean>
}
) {
const source = layer.layerDef.source
const source = layer.source
if (source.geojsonZoomLevel === undefined) {
throw "Invalid layer: geojsonZoomLevel expected"
}
@ -30,7 +31,7 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
const whitelistUrl = source.geojsonSource
.replace("{z}", "" + source.geojsonZoomLevel)
.replace("{x}_{y}.geojson", "overview.json")
.replace("{layer}", layer.layerDef.id)
.replace("{layer}", layer.id)
if (DynamicGeoJsonTileSource.whitelistCache.has(whitelistUrl)) {
whitelist = DynamicGeoJsonTileSource.whitelistCache.get(whitelistUrl)
@ -56,14 +57,13 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
DynamicGeoJsonTileSource.whitelistCache.set(whitelistUrl, whitelist)
})
.catch((err) => {
console.warn("No whitelist found for ", layer.layerDef.id, err)
console.warn("No whitelist found for ", layer.id, err)
})
}
}
const blackList = new Set<string>()
super(
layer,
source.geojsonZoomLevel,
(zxy) => {
if (whitelist !== undefined) {
@ -78,25 +78,13 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
}
}
const src = new GeoJsonSource(layer, zxy, {
return new GeoJsonSource(layer, {
zxy,
featureIdBlacklist: blackList,
})
registerLayer(src)
return src
},
state
mapProperties,
{ isActive: options.isActive }
)
}
public static RegisterWhitelist(url: string, json: any) {
const data = new Map<number, Set<number>>()
for (const x in json) {
if (x === "zoom") {
continue
}
data.set(Number(x), new Set(json[x]))
}
DynamicGeoJsonTileSource.whitelistCache.set(url, data)
}
}

116
Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts
View file

@ -1,87 +1,65 @@
import FilteredLayer from "../../../Models/FilteredLayer"
import { FeatureSourceForLayer, Tiled } from "../FeatureSource"
import { UIEventSource } from "../../UIEventSource"
import TileHierarchy from "./TileHierarchy"
import { Store, Stores } from "../../UIEventSource"
import { Tiles } from "../../../Models/TileRange"
import { BBox } from "../../BBox"
import FeatureSource from "../FeatureSource"
import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
/***
* A tiled source which dynamically loads the required tiles at a fixed zoom level
*/
export default class DynamicTileSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled>
private readonly _loadedTiles = new Set<number>()
export default class DynamicTileSource extends FeatureSourceMerger {
constructor(
layer: FilteredLayer,
zoomlevel: number,
constructTile: (zxy: [number, number, number]) => FeatureSourceForLayer & Tiled,
state: {
currentBounds: UIEventSource<BBox>
locationControl?: UIEventSource<{ zoom?: number }>
constructSource: (tileIndex) => FeatureSource,
mapProperties: {
bounds: Store<BBox>
zoom: Store<number>
},
options?: {
isActive?: Store<boolean>
}
) {
const self = this
this.loadedTiles = new Map<number, FeatureSourceForLayer & Tiled>()
const neededTiles = state.currentBounds
.map(
(bounds) => {
if (bounds === undefined) {
// We'll retry later
return undefined
}
if (!layer.isDisplayed.data && !layer.layerDef.forceLoad) {
// No need to download! - the layer is disabled
return undefined
}
if (
state.locationControl?.data?.zoom !== undefined &&
state.locationControl.data.zoom < layer.layerDef.minzoom
) {
// No need to download! - the layer is disabled
return undefined
}
const tileRange = Tiles.TileRangeBetween(
zoomlevel,
bounds.getNorth(),
bounds.getEast(),
bounds.getSouth(),
bounds.getWest()
)
if (tileRange.total > 10000) {
console.error(
"Got a really big tilerange, bounds and location might be out of sync"
super()
const loadedTiles = new Set<number>()
const neededTiles: Store<number[]> = Stores.ListStabilized(
mapProperties.bounds
.mapD(
(bounds) => {
if (options?.isActive?.data === false) {
// No need to download! - the layer is disabled
return undefined
}
const tileRange = Tiles.TileRangeBetween(
zoomlevel,
bounds.getNorth(),
bounds.getEast(),
bounds.getSouth(),
bounds.getWest()
)
return undefined
}
if (tileRange.total > 10000) {
console.error(
"Got a really big tilerange, bounds and location might be out of sync"
)
return undefined
}
const needed = Tiles.MapRange(tileRange, (x, y) =>
Tiles.tile_index(zoomlevel, x, y)
).filter((i) => !self._loadedTiles.has(i))
if (needed.length === 0) {
return undefined
}
return needed
},
[layer.isDisplayed, state.locationControl]
)
.stabilized(250)
const needed = Tiles.MapRange(tileRange, (x, y) =>
Tiles.tile_index(zoomlevel, x, y)
).filter((i) => !loadedTiles.has(i))
if (needed.length === 0) {
return undefined
}
return needed
},
[options?.isActive, mapProperties.zoom]
)
.stabilized(250)
)
neededTiles.addCallbackAndRunD((neededIndexes) => {
console.log("Tiled geojson source ", layer.layerDef.id, " needs", neededIndexes)
if (neededIndexes === undefined) {
return
}
for (const neededIndex of neededIndexes) {
self._loadedTiles.add(neededIndex)
const src = constructTile(Tiles.tile_from_index(neededIndex))
if (src !== undefined) {
self.loadedTiles.set(neededIndex, src)
}
loadedTiles.add(neededIndex)
super.addSource(constructSource(neededIndex))
}
})
}

148
Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts
View file

@ -1,93 +1,68 @@
import { Utils } from "../../../Utils"
import OsmToGeoJson from "osmtogeojson"
import StaticFeatureSource from "../Sources/StaticFeatureSource"
import PerLayerFeatureSourceSplitter from "../PerLayerFeatureSourceSplitter"
import { Store, UIEventSource } from "../../UIEventSource"
import FilteredLayer from "../../../Models/FilteredLayer"
import { FeatureSourceForLayer, Tiled } from "../FeatureSource"
import { ImmutableStore, Store, UIEventSource } from "../../UIEventSource"
import { Tiles } from "../../../Models/TileRange"
import { BBox } from "../../BBox"
import LayoutConfig from "../../../Models/ThemeConfig/LayoutConfig"
import { Or } from "../../Tags/Or"
import { TagsFilter } from "../../Tags/TagsFilter"
import { OsmObject } from "../../Osm/OsmObject"
import { FeatureCollection } from "@turf/turf"
import { Feature } from "geojson"
import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
/**
* If a tile is needed (requested via the UIEventSource in the constructor), will download the appropriate tile and pass it via 'handleTile'
*/
export default class OsmFeatureSource {
public readonly isRunning: UIEventSource<boolean> = new UIEventSource<boolean>(false)
public readonly downloadedTiles = new Set<number>()
public rawDataHandlers: ((osmJson: any, tileId: number) => void)[] = []
export default class OsmFeatureSource extends FeatureSourceMerger {
private readonly _bounds: Store<BBox>
private readonly isActive: Store<boolean>
private readonly _backend: string
private readonly filteredLayers: Store<FilteredLayer[]>
private readonly handleTile: (fs: FeatureSourceForLayer & Tiled) => void
private isActive: Store<boolean>
private options: {
handleTile: (tile: FeatureSourceForLayer & Tiled) => void
isActive: Store<boolean>
neededTiles: Store<number[]>
markTileVisited?: (tileId: number) => void
}
private readonly allowedTags: TagsFilter
public readonly isRunning: UIEventSource<boolean> = new UIEventSource<boolean>(false)
public rawDataHandlers: ((osmJson: any, tileIndex: number) => void)[] = []
private readonly _downloadedTiles: Set<number> = new Set<number>()
private readonly _downloadedData: Feature[][] = []
/**
*
* @param options: allowedFeatures is normally calculated from the layoutToUse
* Downloads data directly from the OSM-api within the given bounds.
* All features which match the TagsFilter 'allowedFeatures' are kept and converted into geojson
*/
constructor(options: {
handleTile: (tile: FeatureSourceForLayer & Tiled) => void
isActive: Store<boolean>
neededTiles: Store<number[]>
state: {
readonly filteredLayers: UIEventSource<FilteredLayer[]>
readonly osmConnection: {
Backend(): string
}
readonly layoutToUse?: LayoutConfig
}
readonly allowedFeatures?: TagsFilter
markTileVisited?: (tileId: number) => void
bounds: Store<BBox>
readonly allowedFeatures: TagsFilter
backend?: "https://openstreetmap.org/" | string
/**
* If given: this featureSwitch will not update if the store contains 'false'
*/
isActive?: Store<boolean>
}) {
this.options = options
this._backend = options.state.osmConnection.Backend()
this.filteredLayers = options.state.filteredLayers.map((layers) =>
layers.filter((layer) => layer.layerDef.source.geojsonSource === undefined)
)
this.handleTile = options.handleTile
this.isActive = options.isActive
const self = this
options.neededTiles.addCallbackAndRunD((neededTiles) => {
self.Update(neededTiles)
})
const neededLayers = (options.state.layoutToUse?.layers ?? [])
.filter((layer) => !layer.doNotDownload)
.filter(
(layer) => layer.source.geojsonSource === undefined || layer.source.isOsmCacheLayer
)
this.allowedTags =
options.allowedFeatures ?? new Or(neededLayers.map((l) => l.source.osmTags))
super()
this._bounds = options.bounds
this.allowedTags = options.allowedFeatures
this.isActive = options.isActive ?? new ImmutableStore(true)
this._backend = options.backend ?? "https://www.openstreetmap.org"
this._bounds.addCallbackAndRunD((bbox) => this.loadData(bbox))
console.log("Allowed tags are:", this.allowedTags)
}
private async Update(neededTiles: number[]) {
if (this.options.isActive?.data === false) {
private async loadData(bbox: BBox) {
if (this.isActive?.data === false) {
console.log("OsmFeatureSource: not triggering: inactive")
return
}
neededTiles = neededTiles.filter((tile) => !this.downloadedTiles.has(tile))
const z = 15
const neededTiles = Tiles.tileRangeFrom(bbox, z)
if (neededTiles.length == 0) {
if (neededTiles.total == 0) {
return
}
this.isRunning.setData(true)
try {
for (const neededTile of neededTiles) {
this.downloadedTiles.add(neededTile)
await this.LoadTile(...Tiles.tile_from_index(neededTile))
}
const tileNumbers = Tiles.MapRange(neededTiles, (x, y) => {
return Tiles.tile_index(z, x, y)
})
await Promise.all(tileNumbers.map((i) => this.LoadTile(...Tiles.tile_from_index(i))))
} catch (e) {
console.error(e)
} finally {
@ -95,6 +70,11 @@ export default class OsmFeatureSource {
}
}
private registerFeatures(features: Feature[]): void {
this._downloadedData.push(features)
super.addData(this._downloadedData)
}
/**
* The requested tile might only contain part of the relation.
*
@ -135,6 +115,11 @@ export default class OsmFeatureSource {
if (z < 14) {
throw `Zoom ${z} is too much for OSM to handle! Use a higher zoom level!`
}
const index = Tiles.tile_index(z, x, y)
if (this._downloadedTiles.has(index)) {
return
}
this._downloadedTiles.add(index)
const bbox = BBox.fromTile(z, x, y)
const url = `${this._backend}/api/0.6/map?bbox=${bbox.minLon},${bbox.minLat},${bbox.maxLon},${bbox.maxLat}`
@ -146,43 +131,28 @@ export default class OsmFeatureSource {
this.rawDataHandlers.forEach((handler) =>
handler(osmJson, Tiles.tile_index(z, x, y))
)
const geojson = <FeatureCollection<any, { id: string }>>OsmToGeoJson(
let features = <Feature<any, { id: string }>[]>OsmToGeoJson(
osmJson,
// @ts-ignore
{
flatProperties: true,
}
)
).features
// The geojson contains _all_ features at the given location
// We only keep what is needed
geojson.features = geojson.features.filter((feature) =>
features = features.filter((feature) =>
this.allowedTags.matchesProperties(feature.properties)
)
for (let i = 0; i < geojson.features.length; i++) {
geojson.features[i] = await this.patchIncompleteRelations(
geojson.features[i],
osmJson
)
for (let i = 0; i < features.length; i++) {
features[i] = await this.patchIncompleteRelations(features[i], osmJson)
}
geojson.features.forEach((f) => {
features.forEach((f) => {
f.properties["_backend"] = this._backend
})
const index = Tiles.tile_index(z, x, y)
new PerLayerFeatureSourceSplitter(
this.filteredLayers,
this.handleTile,
new StaticFeatureSource(geojson.features),
{
tileIndex: index,
}
)
if (this.options.markTileVisited) {
this.options.markTileVisited(index)
}
this.registerFeatures(features)
} catch (e) {
console.error(
"PANIC: got the tile from the OSM-api, but something crashed handling this tile"
@ -202,10 +172,12 @@ export default class OsmFeatureSource {
if (e === "rate limited") {
return
}
await this.LoadTile(z + 1, x * 2, y * 2)
await this.LoadTile(z + 1, 1 + x * 2, y * 2)
await this.LoadTile(z + 1, x * 2, 1 + y * 2)
await this.LoadTile(z + 1, 1 + x * 2, 1 + y * 2)
await Promise.all([
this.LoadTile(z + 1, x * 2, y * 2),
this.LoadTile(z + 1, 1 + x * 2, y * 2),
this.LoadTile(z + 1, x * 2, 1 + y * 2),
this.LoadTile(z + 1, 1 + x * 2, 1 + y * 2),
])
}
if (error !== undefined) {

2
Logic/FeatureSource/TiledFeatureSource/TileHierarchy.ts
View file

@ -1,7 +1,7 @@
import FeatureSource, { Tiled } from "../FeatureSource"
import { BBox } from "../../BBox"
export default interface TileHierarchy<T extends FeatureSource & Tiled> {
export default interface TileHierarchy<T extends FeatureSource> {
/**
* A mapping from 'tile_index' to the actual tile featrues
*/