midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Refactoring: port import flow

Browse source
This commit is contained in:
Pieter Vander Vennet 2023-06-01 02:52:21 +02:00
parent 8ed4da4e9d
commit ace7caada1
48 changed files with 852 additions and 574 deletions
Download patch file Download diff file Expand all files Collapse all files

38
Logic/FeatureSource/Actors/TileLocalStorage.ts
View file

@ -1,5 +1,5 @@
import { IdbLocalStorage } from "../../Web/IdbLocalStorage"
import { UIEventSource } from "../../UIEventSource"
import {IdbLocalStorage} from "../../Web/IdbLocalStorage"
import {UIEventSource} from "../../UIEventSource"
/**
* A class which allows to read/write a tile to local storage.
@ -10,23 +10,25 @@ import { UIEventSource } from "../../UIEventSource"
*/
export default class TileLocalStorage<T> {
private static perLayer: Record<string, TileLocalStorage<any>> = {}
private static readonly useIndexedDb = typeof indexedDB !== "undefined"
private readonly _layername: string
private readonly inUse = new UIEventSource(false)
private readonly cachedSources: Record<number, UIEventSource<T> & { flush: () => void }> = {}
private readonly _maxAgeSeconds: number;
private static readonly useIndexedDb = typeof indexedDB !== "undefined"
private constructor(layername: string) {
private constructor(layername: string, maxAgeSeconds: number) {
this._layername = layername
this._maxAgeSeconds = maxAgeSeconds;
}
public static construct<T>(backend: string, layername: string): TileLocalStorage<T> {
public static construct<T>(backend: string, layername: string, maxAgeS: number): TileLocalStorage<T> {
const key = backend + "_" + layername
const cached = TileLocalStorage.perLayer[key]
if (cached) {
return cached
}
const tls = new TileLocalStorage<T>(key)
const tls = new TileLocalStorage<T>(key, maxAgeS)
TileLocalStorage.perLayer[key] = tls
return tls
}
@ -50,13 +52,15 @@ export default class TileLocalStorage<T> {
}
private async SetIdb(tileIndex: number, data: any): Promise<void> {
if(!TileLocalStorage.useIndexedDb){
if (!TileLocalStorage.useIndexedDb) {
return
}
try {
await this.inUse.AsPromise((inUse) => !inUse)
this.inUse.setData(true)
await IdbLocalStorage.SetDirectly(this._layername + "_" + tileIndex, data)
await IdbLocalStorage.SetDirectly(this._layername + "_" + tileIndex + "_date", Date.now())
this.inUse.setData(false)
} catch (e) {
console.error(
@ -72,10 +76,24 @@ export default class TileLocalStorage<T> {
}
}
private GetIdb(tileIndex: number): Promise<any> {
if(!TileLocalStorage.useIndexedDb){
private async GetIdb(tileIndex: number): Promise<any> {
if (!TileLocalStorage.useIndexedDb) {
return undefined
}
return IdbLocalStorage.GetDirectly(this._layername + "_" + tileIndex)
const date = <any>await IdbLocalStorage.GetDirectly(this._layername + "_" + tileIndex + "_date")
const maxAge = this._maxAgeSeconds
const timeDiff = Date.now() - date
if (timeDiff >= maxAge) {
console.log("Dropping cache for", this._layername, tileIndex, "out of date")
await IdbLocalStorage.SetDirectly(this._layername + "_" + tileIndex, undefined)
return undefined
}
const data = await IdbLocalStorage.GetDirectly(this._layername + "_" + tileIndex)
return <any>data
}
invalidate(zoomlevel: number, tileIndex) {
this.getTileSource(tileIndex).setData(undefined)
}
}

5
Logic/FeatureSource/FeatureSource.ts
View file

@ -10,11 +10,6 @@ export interface WritableFeatureSource<T extends Feature = Feature> extends Feat
features: UIEventSource<T[]>
}
export interface Tiled {
tileIndex: number
bbox: BBox
}
/**
* A feature source which only contains features for the defined layer
*/

15
Logic/FeatureSource/Sources/LayoutSource.ts
View file

@ -10,6 +10,7 @@ import FeatureSourceMerger from "./FeatureSourceMerger"
import DynamicGeoJsonTileSource from "../TiledFeatureSource/DynamicGeoJsonTileSource"
import {BBox} from "../../BBox"
import LocalStorageFeatureSource from "../TiledFeatureSource/LocalStorageFeatureSource"
import FullNodeDatabaseSource from "../TiledFeatureSource/FullNodeDatabaseSource";
/**
* This source will fetch the needed data from various sources for the given layout.
@ -27,7 +28,8 @@ export default class LayoutSource extends FeatureSourceMerger {
featureSwitches: FeatureSwitchState,
mapProperties: { bounds: Store<BBox>; zoom: Store<number> },
backend: string,
isDisplayed: (id: string) => Store<boolean>
isDisplayed: (id: string) => Store<boolean>,
fullNodeDatabaseSource?: FullNodeDatabaseSource
) {
const { bounds, zoom } = mapProperties
// remove all 'special' layers
@ -39,6 +41,7 @@ export default class LayoutSource extends FeatureSourceMerger {
(l) =>
new LocalStorageFeatureSource(backend, l.id, 15, mapProperties, {
isActive: isDisplayed(l.id),
maxAge: l.maxAgeOfCache
})
)
@ -55,7 +58,8 @@ export default class LayoutSource extends FeatureSourceMerger {
bounds,
zoom,
backend,
featureSwitches
featureSwitches,
fullNodeDatabaseSource
)
const geojsonSources: FeatureSource[] = geojsonlayers.map((l) =>
LayoutSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id))
@ -96,7 +100,8 @@ export default class LayoutSource extends FeatureSourceMerger {
bounds: Store<BBox>,
zoom: Store<number>,
backend: string,
featureSwitches: FeatureSwitchState
featureSwitches: FeatureSwitchState,
fullNodeDatabase: FullNodeDatabaseSource
): OsmFeatureSource | undefined {
if (osmLayers.length == 0) {
return undefined
@ -121,8 +126,8 @@ export default class LayoutSource extends FeatureSourceMerger {
bounds,
backend,
isActive,
patchRelations: true
patchRelations: true,
fullNodeDatabase
})
}

25
Logic/FeatureSource/Sources/OsmFeatureSource.ts
View file

@ -7,6 +7,7 @@ import { TagsFilter } from "../../Tags/TagsFilter"
import { Feature } from "geojson"
import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
import OsmObjectDownloader from "../../Osm/OsmObjectDownloader"
import FullNodeDatabaseSource from "../TiledFeatureSource/FullNodeDatabaseSource";
/**
* If a tile is needed (requested via the UIEventSource in the constructor), will download the appropriate tile and pass it via 'handleTile'
@ -16,9 +17,19 @@ export default class OsmFeatureSource extends FeatureSourceMerger {
private readonly isActive: Store<boolean>
private readonly _backend: string
private readonly allowedTags: TagsFilter
private options: {
bounds: Store<BBox>
readonly allowedFeatures: TagsFilter
backend?: "https://openstreetmap.org/" | string
/**
* If given: this featureSwitch will not update if the store contains 'false'
*/
isActive?: Store<boolean>,
patchRelations?: true | boolean,
fullNodeDatabase?: FullNodeDatabaseSource
};
public readonly isRunning: UIEventSource<boolean> = new UIEventSource<boolean>(false)
public rawDataHandlers: ((osmJson: any, tileIndex: number) => void)[] = []
private readonly _downloadedTiles: Set<number> = new Set<number>()
private readonly _downloadedData: Feature[][] = []
@ -35,9 +46,11 @@ export default class OsmFeatureSource extends FeatureSourceMerger {
* If given: this featureSwitch will not update if the store contains 'false'
*/
isActive?: Store<boolean>,
patchRelations?: true | boolean
patchRelations?: true | boolean,
fullNodeDatabase?: FullNodeDatabaseSource
}) {
super()
this.options = options;
this._bounds = options.bounds
this.allowedTags = options.allowedFeatures
this.isActive = options.isActive ?? new ImmutableStore(true)
@ -119,7 +132,7 @@ export default class OsmFeatureSource extends FeatureSourceMerger {
return feature
}
private async LoadTile(z, x, y): Promise<void> {
private async LoadTile(z: number, x: number, y: number): Promise<void> {
console.log("OsmFeatureSource: loading ", z, x, y, "from", this._backend)
if (z >= 22) {
throw "This is an absurd high zoom level"
@ -141,9 +154,7 @@ export default class OsmFeatureSource extends FeatureSourceMerger {
try {
const osmJson = await Utils.downloadJsonCached(url, 2000)
try {
this.rawDataHandlers.forEach((handler) =>
handler(osmJson, Tiles.tile_index(z, x, y))
)
this.options?.fullNodeDatabase?.handleOsmJson(osmJson, z, x, y)
let features = <Feature<any, { id: string }>[]>OsmToGeoJson(
osmJson,
// @ts-ignore
@ -181,7 +192,7 @@ export default class OsmFeatureSource extends FeatureSourceMerger {
y,
"due to",
e,
"; retrying with smaller bounds"
e === "rate limited" ? "; stopping now" : "; retrying with smaller bounds"
)
if (e === "rate limited") {
return

59
Logic/FeatureSource/TiledFeatureSource/FullNodeDatabaseSource.ts
View file

@ -1,31 +1,25 @@
import {FeatureSource, FeatureSourceForLayer, Tiled } from "../FeatureSource"
import { OsmNode, OsmObject, OsmWay } from "../../Osm/OsmObject"
import SimpleFeatureSource from "../Sources/SimpleFeatureSource"
import FilteredLayer from "../../../Models/FilteredLayer"
import { UIEventSource } from "../../UIEventSource"
import { OsmTags } from "../../../Models/OsmFeature";
import { BBox } from "../../BBox";
import { Feature, Point } from "geojson";
import {OsmNode, OsmObject, OsmWay} from "../../Osm/OsmObject"
import {UIEventSource} from "../../UIEventSource"
import {BBox} from "../../BBox";
import StaticFeatureSource from "../Sources/StaticFeatureSource";
import {Tiles} from "../../../Models/TileRange";
export default class FullNodeDatabaseSource {
public readonly loadedTiles = new Map<number, FeatureSource & Tiled>()
private readonly onTileLoaded: (tile: Tiled & FeatureSourceForLayer) => void
private readonly layer: FilteredLayer
private readonly loadedTiles = new Map<number, Map<number, OsmNode>>()
private readonly nodeByIds = new Map<number, OsmNode>()
private readonly parentWays = new Map<number, UIEventSource<OsmWay[]>>()
constructor(layer: FilteredLayer, onTileLoaded: (tile: Tiled & FeatureSourceForLayer) => void) {
this.onTileLoaded = onTileLoaded
this.layer = layer
if (this.layer === undefined) {
throw "Layer is undefined"
}
}
private smallestZoom = 99
private largestZoom = 0
public handleOsmJson(osmJson: any, tileId: number) {
public handleOsmJson(osmJson: any, z: number, x: number, y: number) : void {
const allObjects = OsmObject.ParseObjects(osmJson.elements)
const nodesById = new Map<number, OsmNode>()
this.smallestZoom = Math.min(this.smallestZoom, z)
this.largestZoom = Math.max(this.largestZoom, z)
for (const osmObj of allObjects) {
if (osmObj.type !== "node") {
continue
@ -59,10 +53,9 @@ export default class FullNodeDatabaseSource {
osmNode.asGeoJson()
)
const featureSource = new SimpleFeatureSource(this.layer, tileId)
featureSource.features.setData(asGeojsonFeatures)
this.loadedTiles.set(tileId, featureSource)
this.onTileLoaded(featureSource)
const featureSource = new StaticFeatureSource(asGeojsonFeatures)
const tileId = Tiles.tile_index(z, x, y)
this.loadedTiles.set(tileId, nodesById)
}
/**
@ -76,7 +69,7 @@ export default class FullNodeDatabaseSource {
}
/**
* Gets the parent way list
* Gets all the ways that the given node is a part of
* @param nodeId
* @constructor
*/
@ -84,8 +77,20 @@ export default class FullNodeDatabaseSource {
return this.parentWays.get(nodeId)
}
getNodesWithin(bBox: BBox) : Feature<Point, OsmTags>[]{
// TODO
throw "TODO"
/**
* Gets (at least) all nodes which are part of this BBOX; might also return some nodes that fall outside of the bbox but are closeby
* @param bbox
*/
getNodesWithin(bbox: BBox) : Map<number, OsmNode>{
const allById = new Map<number, OsmNode>()
for (let z = this.smallestZoom; z < this.largestZoom; z++) {
const range = Tiles.tileRangeFrom(bbox, z)
Tiles.MapRange(range, (x, y ) => {
const tileId = Tiles.tile_index(z, x, y)
const nodesById = this.loadedTiles.get(tileId)
nodesById?.forEach((v,k) => allById.set(k,v))
})
}
return allById
}
}

23
Logic/FeatureSource/TiledFeatureSource/LocalStorageFeatureSource.ts
View file

@ -1,9 +1,11 @@
import DynamicTileSource from "./DynamicTileSource"
import { Store } from "../../UIEventSource"
import { BBox } from "../../BBox"
import {Store} from "../../UIEventSource"
import {BBox} from "../../BBox"
import TileLocalStorage from "../Actors/TileLocalStorage"
import { Feature } from "geojson"
import {Feature} from "geojson"
import StaticFeatureSource from "../Sources/StaticFeatureSource"
import {constants} from "http2";
import HTTP_STATUS_CONTINUE = module
export default class LocalStorageFeatureSource extends DynamicTileSource {
constructor(
@ -15,18 +17,25 @@ export default class LocalStorageFeatureSource extends DynamicTileSource {
zoom: Store<number>
},
options?: {
isActive?: Store<boolean>
isActive?: Store<boolean>,
maxAge?: number // In seconds
}
) {
const storage = TileLocalStorage.construct<Feature[]>(backend, layername)
const storage = TileLocalStorage.construct<Feature[]>(backend, layername, options?.maxAge ?? 24 * 60 * 60)
super(
zoomlevel,
(tileIndex) =>
new StaticFeatureSource(
storage
.getTileSource(tileIndex)
.map((features) =>
features?.filter((f) => !f.properties.id.match(/(node|way)\/-[0-9]+/))
.mapD((features) => {
if (features.length === undefined) {
console.trace("These are not features:", features)
storage.invalidate(zoomlevel, tileIndex)
return []
}
return features.filter((f) => !f.properties.id.match(/(node|way)\/-[0-9]+/));
}
)
),
mapProperties,