midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

refactoring

Browse source
This commit is contained in:
Pieter Vander Vennet 2023-03-28 05:13:48 +02:00
parent b94a8f5745
commit 5d0fe31c41
114 changed files with 2412 additions and 2958 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts
View file

@ -84,7 +84,9 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
})
},
mapProperties,
{ isActive: options.isActive }
{
isActive: options?.isActive,
}
)
}
}

3
Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts
View file

@ -5,7 +5,8 @@ import FeatureSource from "../FeatureSource"
import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
/***
* A tiled source which dynamically loads the required tiles at a fixed zoom level
* A tiled source which dynamically loads the required tiles at a fixed zoom level.
* A single featureSource will be initiliased for every tile in view; which will alter be merged into this featureSource
*/
export default class DynamicTileSource extends FeatureSourceMerger {
constructor(

11
Logic/FeatureSource/TiledFeatureSource/FullNodeDatabaseSource.ts
View file

@ -1,11 +1,13 @@
import TileHierarchy from "./TileHierarchy"
import FeatureSource, { FeatureSourceForLayer, Tiled } from "../FeatureSource"
import { OsmNode, OsmObject, OsmWay } from "../../Osm/OsmObject"
import SimpleFeatureSource from "../Sources/SimpleFeatureSource"
import FilteredLayer from "../../../Models/FilteredLayer"
import { UIEventSource } from "../../UIEventSource"
import { OsmTags } from "../../../Models/OsmFeature";
import { BBox } from "../../BBox";
import { Feature, Point } from "geojson";
export default class FullNodeDatabaseSource implements TileHierarchy<FeatureSource & Tiled> {
export default class FullNodeDatabaseSource {
public readonly loadedTiles = new Map<number, FeatureSource & Tiled>()
private readonly onTileLoaded: (tile: Tiled & FeatureSourceForLayer) => void
private readonly layer: FilteredLayer
@ -81,4 +83,9 @@ export default class FullNodeDatabaseSource implements TileHierarchy<FeatureSour
public GetParentWays(nodeId: number): UIEventSource<OsmWay[]> {
return this.parentWays.get(nodeId)
}
getNodesWithin(bBox: BBox) : Feature<Point, OsmTags>[]{
// TODO
throw "TODO"
}
}

28
Logic/FeatureSource/TiledFeatureSource/LocalStorageFeatureSource.ts Normal file
View file

@ -0,0 +1,28 @@
import DynamicTileSource from "./DynamicTileSource"
import { Store } from "../../UIEventSource"
import { BBox } from "../../BBox"
import TileLocalStorage from "../Actors/TileLocalStorage"
import { Feature } from "geojson"
import StaticFeatureSource from "../Sources/StaticFeatureSource"
export default class LocalStorageFeatureSource extends DynamicTileSource {
constructor(
layername: string,
zoomlevel: number,
mapProperties: {
bounds: Store<BBox>
zoom: Store<number>
},
options?: {
isActive?: Store<boolean>
}
) {
const storage = TileLocalStorage.construct<Feature[]>(layername)
super(
zoomlevel,
(tileIndex) => new StaticFeatureSource(storage.getTileSource(tileIndex)),
mapProperties,
options
)
}
}

187
Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts
View file

@ -1,187 +0,0 @@
import { Utils } from "../../../Utils"
import OsmToGeoJson from "osmtogeojson"
import { ImmutableStore, Store, UIEventSource } from "../../UIEventSource"
import { Tiles } from "../../../Models/TileRange"
import { BBox } from "../../BBox"
import { TagsFilter } from "../../Tags/TagsFilter"
import { OsmObject } from "../../Osm/OsmObject"
import { Feature } from "geojson"
import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
/**
* If a tile is needed (requested via the UIEventSource in the constructor), will download the appropriate tile and pass it via 'handleTile'
*/
export default class OsmFeatureSource extends FeatureSourceMerger {
private readonly _bounds: Store<BBox>
private readonly isActive: Store<boolean>
private readonly _backend: string
private readonly allowedTags: TagsFilter
public readonly isRunning: UIEventSource<boolean> = new UIEventSource<boolean>(false)
public rawDataHandlers: ((osmJson: any, tileIndex: number) => void)[] = []
private readonly _downloadedTiles: Set<number> = new Set<number>()
private readonly _downloadedData: Feature[][] = []
/**
* Downloads data directly from the OSM-api within the given bounds.
* All features which match the TagsFilter 'allowedFeatures' are kept and converted into geojson
*/
constructor(options: {
bounds: Store<BBox>
readonly allowedFeatures: TagsFilter
backend?: "https://openstreetmap.org/" | string
/**
* If given: this featureSwitch will not update if the store contains 'false'
*/
isActive?: Store<boolean>
}) {
super()
this._bounds = options.bounds
this.allowedTags = options.allowedFeatures
this.isActive = options.isActive ?? new ImmutableStore(true)
this._backend = options.backend ?? "https://www.openstreetmap.org"
this._bounds.addCallbackAndRunD((bbox) => this.loadData(bbox))
console.log("Allowed tags are:", this.allowedTags)
}
private async loadData(bbox: BBox) {
if (this.isActive?.data === false) {
console.log("OsmFeatureSource: not triggering: inactive")
return
}
const z = 15
const neededTiles = Tiles.tileRangeFrom(bbox, z)
if (neededTiles.total == 0) {
return
}
this.isRunning.setData(true)
try {
const tileNumbers = Tiles.MapRange(neededTiles, (x, y) => {
return Tiles.tile_index(z, x, y)
})
await Promise.all(tileNumbers.map((i) => this.LoadTile(...Tiles.tile_from_index(i))))
} catch (e) {
console.error(e)
} finally {
this.isRunning.setData(false)
}
}
private registerFeatures(features: Feature[]): void {
this._downloadedData.push(features)
super.addData(this._downloadedData)
}
/**
* The requested tile might only contain part of the relation.
*
* This method will download the full relation and return it as geojson if it was incomplete.
* If the feature is already complete (or is not a relation), the feature will be returned
*/
private async patchIncompleteRelations(
feature: { properties: { id: string } },
originalJson: { elements: { type: "node" | "way" | "relation"; id: number }[] }
): Promise<any> {
if (!feature.properties.id.startsWith("relation")) {
return feature
}
const relationSpec = originalJson.elements.find(
(f) => "relation/" + f.id === feature.properties.id
)
const members: { type: string; ref: number }[] = relationSpec["members"]
for (const member of members) {
const isFound = originalJson.elements.some(
(f) => f.id === member.ref && f.type === member.type
)
if (isFound) {
continue
}
// This member is missing. We redownload the entire relation instead
console.debug("Fetching incomplete relation " + feature.properties.id)
return (await OsmObject.DownloadObjectAsync(feature.properties.id)).asGeoJson()
}
return feature
}
private async LoadTile(z, x, y): Promise<void> {
if (z >= 22) {
throw "This is an absurd high zoom level"
}
if (z < 14) {
throw `Zoom ${z} is too much for OSM to handle! Use a higher zoom level!`
}
const index = Tiles.tile_index(z, x, y)
if (this._downloadedTiles.has(index)) {
return
}
this._downloadedTiles.add(index)
const bbox = BBox.fromTile(z, x, y)
const url = `${this._backend}/api/0.6/map?bbox=${bbox.minLon},${bbox.minLat},${bbox.maxLon},${bbox.maxLat}`
let error = undefined
try {
const osmJson = await Utils.downloadJson(url)
try {
this.rawDataHandlers.forEach((handler) =>
handler(osmJson, Tiles.tile_index(z, x, y))
)
let features = <Feature<any, { id: string }>[]>OsmToGeoJson(
osmJson,
// @ts-ignore
{
flatProperties: true,
}
).features
// The geojson contains _all_ features at the given location
// We only keep what is needed
features = features.filter((feature) =>
this.allowedTags.matchesProperties(feature.properties)
)
for (let i = 0; i < features.length; i++) {
features[i] = await this.patchIncompleteRelations(features[i], osmJson)
}
features.forEach((f) => {
f.properties["_backend"] = this._backend
})
this.registerFeatures(features)
} catch (e) {
console.error(
"PANIC: got the tile from the OSM-api, but something crashed handling this tile"
)
error = e
}
} catch (e) {
console.error(
"Could not download tile",
z,
x,
y,
"due to",
e,
"; retrying with smaller bounds"
)
if (e === "rate limited") {
return
}
await Promise.all([
this.LoadTile(z + 1, x * 2, y * 2),
this.LoadTile(z + 1, 1 + x * 2, y * 2),
this.LoadTile(z + 1, x * 2, 1 + y * 2),
this.LoadTile(z + 1, 1 + x * 2, 1 + y * 2),
])
}
if (error !== undefined) {
throw error
}
}
}

24
Logic/FeatureSource/TiledFeatureSource/README.md
View file

@ -1,24 +0,0 @@
Data in MapComplete can come from multiple sources.
Currently, they are:
- The Overpass-API
- The OSM-API
- One or more GeoJSON files. This can be a single file or a set of tiled geojson files
- LocalStorage, containing features from a previous visit
- Changes made by the user introducing new features
When the data enters from Overpass or from the OSM-API, they are first distributed per layer:
OVERPASS | ---PerLayerFeatureSource---> FeatureSourceForLayer[]
OSM |
The GeoJSon files (not tiled) are then added to this list
A single FeatureSourcePerLayer is then further handled by splitting it into a tile hierarchy.
In order to keep thins snappy, they are distributed over a tiled database per layer.
## Notes
`cached-featuresbookcases` is the old key used `cahced-features{themeid}` and should be cleaned up

24
Logic/FeatureSource/TiledFeatureSource/TileHierarchy.ts
View file

@ -1,24 +0,0 @@
import FeatureSource, { Tiled } from "../FeatureSource"
import { BBox } from "../../BBox"
export default interface TileHierarchy<T extends FeatureSource> {
/**
* A mapping from 'tile_index' to the actual tile featrues
*/
loadedTiles: Map<number, T>
}
export class TileHierarchyTools {
public static getTiles<T extends FeatureSource & Tiled>(
hierarchy: TileHierarchy<T>,
bbox: BBox
): T[] {
const result: T[] = []
hierarchy.loadedTiles.forEach((tile) => {
if (tile.bbox.overlapsWith(bbox)) {
result.push(tile)
}
})
return result
}
}

58
Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts
View file

@ -1,58 +0,0 @@
import TileHierarchy from "./TileHierarchy"
import { UIEventSource } from "../../UIEventSource"
import FeatureSource, { FeatureSourceForLayer, IndexedFeatureSource, Tiled } from "../FeatureSource"
import FilteredLayer from "../../../Models/FilteredLayer"
import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
import { Tiles } from "../../../Models/TileRange"
import { BBox } from "../../BBox"
export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<
number,
FeatureSourceForLayer & Tiled
>()
public readonly layer: FilteredLayer
private readonly sources: Map<number, UIEventSource<FeatureSource[]>> = new Map<
number,
UIEventSource<FeatureSource[]>
>()
private _handleTile: (src: FeatureSourceForLayer & IndexedFeatureSource, index: number) => void
constructor(
layer: FilteredLayer,
handleTile: (
src: FeatureSourceForLayer & IndexedFeatureSource & Tiled,
index: number
) => void
) {
this.layer = layer
this._handleTile = handleTile
}
/**
* Add another feature source for the given tile.
* Entries for this tile will be merged
* @param src
*/
public registerTile(src: FeatureSource & Tiled) {
const index = src.tileIndex
if (this.sources.has(index)) {
const sources = this.sources.get(index)
sources.data.push(src)
sources.ping()
return
}
// We have to setup
const sources = new UIEventSource<FeatureSource[]>([src])
this.sources.set(index, sources)
const merger = new FeatureSourceMerger(
this.layer,
index,
BBox.fromTile(...Tiles.tile_from_index(index)),
sources
)
this.loadedTiles.set(index, merger)
this._handleTile(merger, index)
}
}

249
Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource.ts
View file

@ -1,249 +0,0 @@
import FeatureSource, { FeatureSourceForLayer, IndexedFeatureSource, Tiled } from "../FeatureSource"
import { Store, UIEventSource } from "../../UIEventSource"
import FilteredLayer from "../../../Models/FilteredLayer"
import TileHierarchy from "./TileHierarchy"
import { Tiles } from "../../../Models/TileRange"
import { BBox } from "../../BBox"
import { Feature } from "geojson";
/**
* Contains all features in a tiled fashion.
* The data will be automatically broken down into subtiles when there are too much features in a single tile or if the zoomlevel is too high
*/
export default class TiledFeatureSource
implements
Tiled,
IndexedFeatureSource,
FeatureSourceForLayer,
TileHierarchy<IndexedFeatureSource & FeatureSourceForLayer & Tiled>
{
public readonly z: number
public readonly x: number
public readonly y: number
public readonly parent: TiledFeatureSource
public readonly root: TiledFeatureSource
public readonly layer: FilteredLayer
/* An index of all known tiles. allTiles[z][x][y].get('layerid') will yield the corresponding tile.
* Only defined on the root element!
*/
public readonly loadedTiles: Map<number, TiledFeatureSource & FeatureSourceForLayer> = undefined
public readonly maxFeatureCount: number
public readonly name
public readonly features: UIEventSource<Feature[]>
public readonly containedIds: Store<Set<string>>
public readonly bbox: BBox
public readonly tileIndex: number
private upper_left: TiledFeatureSource
private upper_right: TiledFeatureSource
private lower_left: TiledFeatureSource
private lower_right: TiledFeatureSource
private readonly maxzoom: number
private readonly options: TiledFeatureSourceOptions
private constructor(
z: number,
x: number,
y: number,
parent: TiledFeatureSource,
options?: TiledFeatureSourceOptions
) {
this.z = z
this.x = x
this.y = y
this.bbox = BBox.fromTile(z, x, y)
this.tileIndex = Tiles.tile_index(z, x, y)
this.name = `TiledFeatureSource(${z},${x},${y})`
this.parent = parent
this.layer = options.layer
options = options ?? {}
this.maxFeatureCount = options?.maxFeatureCount ?? 250
this.maxzoom = options.maxZoomLevel ?? 18
this.options = options
if (parent === undefined) {
throw "Parent is not allowed to be undefined. Use null instead"
}
if (parent === null && z !== 0 && x !== 0 && y !== 0) {
throw "Invalid root tile: z, x and y should all be null"
}
if (parent === null) {
this.root = this
this.loadedTiles = new Map()
} else {
this.root = this.parent.root
this.loadedTiles = this.root.loadedTiles
const i = Tiles.tile_index(z, x, y)
this.root.loadedTiles.set(i, this)
}
this.features = new UIEventSource<any[]>([])
this.containedIds = this.features.map((features) => {
if (features === undefined) {
return undefined
}
return new Set(features.map((f) => f.properties.id))
})
// We register this tile, but only when there is some data in it
if (this.options.registerTile !== undefined) {
this.features.addCallbackAndRunD((features) => {
if (features.length === 0) {
return
}
this.options.registerTile(this)
return true
})
}
}
public static createHierarchy(
features: FeatureSource,
options?: TiledFeatureSourceOptions
): TiledFeatureSource {
options = {
...options,
layer: features["layer"] ?? options.layer,
}
const root = new TiledFeatureSource(0, 0, 0, null, options)
features.features?.addCallbackAndRunD((feats) => root.addFeatures(feats))
return root
}
private isSplitNeeded(featureCount: number) {
if (this.upper_left !== undefined) {
// This tile has been split previously, so we keep on splitting
return true
}
if (this.z >= this.maxzoom) {
// We are not allowed to split any further
return false
}
if (this.options.minZoomLevel !== undefined && this.z < this.options.minZoomLevel) {
// We must have at least this zoom level before we are allowed to start splitting
return true
}
// To much features - we split
return featureCount > this.maxFeatureCount
}
/***
* Adds the list of features to this hierarchy.
* If there are too much features, the list will be broken down and distributed over the subtiles (only retaining features that don't fit a subtile on this level)
* @param features
* @private
*/
private addFeatures(features: Feature[]) {
if (features === undefined || features.length === 0) {
return
}
if (!this.isSplitNeeded(features.length)) {
this.features.setData(features)
return
}
if (this.upper_left === undefined) {
this.upper_left = new TiledFeatureSource(
this.z + 1,
this.x * 2,
this.y * 2,
this,
this.options
)
this.upper_right = new TiledFeatureSource(
this.z + 1,
this.x * 2 + 1,
this.y * 2,
this,
this.options
)
this.lower_left = new TiledFeatureSource(
this.z + 1,
this.x * 2,
this.y * 2 + 1,
this,
this.options
)
this.lower_right = new TiledFeatureSource(
this.z + 1,
this.x * 2 + 1,
this.y * 2 + 1,
this,
this.options
)
}
const ulf = []
const urf = []
const llf = []
const lrf = []
const overlapsboundary = []
for (const feature of features) {
const bbox = BBox.get(feature)
// There are a few strategies to deal with features that cross tile boundaries
if (this.options.noDuplicates) {
// Strategy 1: We put the feature into a somewhat matching tile
if (bbox.overlapsWith(this.upper_left.bbox)) {
ulf.push(feature)
} else if (bbox.overlapsWith(this.upper_right.bbox)) {
urf.push(feature)
} else if (bbox.overlapsWith(this.lower_left.bbox)) {
llf.push(feature)
} else if (bbox.overlapsWith(this.lower_right.bbox)) {
lrf.push(feature)
} else {
overlapsboundary.push(feature)
}
} else if (this.options.minZoomLevel === undefined) {
// Strategy 2: put it into a strictly matching tile (or in this tile, which is slightly too big)
if (bbox.isContainedIn(this.upper_left.bbox)) {
ulf.push(feature)
} else if (bbox.isContainedIn(this.upper_right.bbox)) {
urf.push(feature)
} else if (bbox.isContainedIn(this.lower_left.bbox)) {
llf.push(feature)
} else if (bbox.isContainedIn(this.lower_right.bbox)) {
lrf.push(feature)
} else {
overlapsboundary.push(feature)
}
} else {
// Strategy 3: We duplicate a feature on a boundary into every tile as we need to get to the minZoomLevel
if (bbox.overlapsWith(this.upper_left.bbox)) {
ulf.push(feature)
}
if (bbox.overlapsWith(this.upper_right.bbox)) {
urf.push(feature)
}
if (bbox.overlapsWith(this.lower_left.bbox)) {
llf.push(feature)
}
if (bbox.overlapsWith(this.lower_right.bbox)) {
lrf.push(feature)
}
}
}
this.upper_left.addFeatures(ulf)
this.upper_right.addFeatures(urf)
this.lower_left.addFeatures(llf)
this.lower_right.addFeatures(lrf)
this.features.setData(overlapsboundary)
}
}
export interface TiledFeatureSourceOptions {
readonly maxFeatureCount?: number
readonly maxZoomLevel?: number
readonly minZoomLevel?: number
/**
* IF minZoomLevel is set, and if a feature runs through a tile boundary, it would normally be duplicated.
* Setting 'dontEnforceMinZoomLevel' will assign to feature to some matching subtile.
*/
readonly noDuplicates?: boolean
readonly registerTile?: (tile: TiledFeatureSource & FeatureSourceForLayer & Tiled) => void
readonly layer?: FilteredLayer
}