From 1476ab0407322a101279f824c524e8c924bed0d5 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Thu, 6 May 2021 03:03:54 +0200 Subject: [PATCH] Experimenting with using the overpass API directly --- InitUiElements.ts | 3 + Logic/FeatureSource/GeoJsonSource.ts | 106 +++++++++-------- Logic/FeatureSource/OsmApiFeatureSource.ts | 80 ++++++++++++- Logic/Osm/OsmObject.ts | 125 +++++++++++++++++---- State.ts | 3 +- Utils.ts | 23 +++- 6 files changed, 251 insertions(+), 89 deletions(-) diff --git a/InitUiElements.ts b/InitUiElements.ts index d7eeded2a5..23b8cd4f88 100644 --- a/InitUiElements.ts +++ b/InitUiElements.ts @@ -394,6 +394,9 @@ export class InitUiElements { const updater = new LoadFromOverpass(state.locationControl, state.layoutToUse, state.leafletMap); State.state.layerUpdater = updater; + + + const source = new FeaturePipeline(state.filteredLayers, updater, state.osmApiFeatureSource, diff --git a/Logic/FeatureSource/GeoJsonSource.ts b/Logic/FeatureSource/GeoJsonSource.ts index ce329b0a1e..beff2c801c 100644 --- a/Logic/FeatureSource/GeoJsonSource.ts +++ b/Logic/FeatureSource/GeoJsonSource.ts @@ -20,9 +20,9 @@ export default class GeoJsonSource implements FeatureSource { private readonly layerId: string; private readonly seenids: Set = new Set() - private constructor(locationControl: UIEventSource, - flayer: { isDisplayed: UIEventSource, layerDef: LayerConfig }, - onFail?: ((errorMsg: any) => void)) { + private constructor(locationControl: UIEventSource, + flayer: { isDisplayed: UIEventSource, layerDef: LayerConfig }, + onFail?: ((errorMsg: any) => void)) { this.layerId = flayer.layerDef.id; let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id); this.name = "GeoJsonSource of " + url; @@ -33,7 +33,7 @@ export default class GeoJsonSource implements FeatureSource { if (zoomLevel === undefined) { // This is a classic, static geojson layer if (onFail === undefined) { - onFail = errorMsg => { + onFail = _ => { } } this.onFail = onFail; @@ -43,57 +43,6 @@ export default class GeoJsonSource implements FeatureSource { this.ConfigureDynamicLayer(url, zoomLevel, locationControl, flayer) } } - - private ConfigureDynamicLayer(url: string, zoomLevel: number, locationControl: UIEventSource, flayer: { isDisplayed: UIEventSource, layerDef: LayerConfig }){ - // This is a dynamic template with a fixed zoom level - url = url.replace("{z}", "" + zoomLevel) - const loadedTiles = new Set(); - const self = this; - this.onFail = (msg, url) => { - console.warn(`Could not load geojson layer from`, url, "due to", msg) - loadedTiles.add(url); // We add the url to the 'loadedTiles' in order to not reload it in the future - } - - const neededTiles = locationControl.map( - location => { - // Yup, this is cheating to just get the bounds here - const bounds = State.state.leafletMap.data.getBounds() - const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest()) - const needed = new Set(); - for (let x = tileRange.xstart; x <= tileRange.xend; x++) { - for (let y = tileRange.ystart; y <= tileRange.yend; y++) { - let neededUrl = url.replace("{x}", "" + x).replace("{y}", "" + y); - needed.add(neededUrl) - } - } - return needed; - } - , [flayer.isDisplayed]); - neededTiles.stabilized(250).addCallback((needed: Set) => { - if (needed === undefined) { - return; - } - if (!flayer.isDisplayed.data) { - // No need to download! - the layer is disabled - return; - } - - if(locationControl.data.zoom < flayer.layerDef.minzoom){ - return; - } - - needed.forEach(neededTile => { - if (loadedTiles.has(neededTile)) { - return; - } - - loadedTiles.add(neededTile) - self.LoadJSONFrom(neededTile) - - }) - }) - - } /** * Merges together the layers which have the same source @@ -152,6 +101,53 @@ export default class GeoJsonSource implements FeatureSource { } + private ConfigureDynamicLayer(url: string, zoomLevel: number, locationControl: UIEventSource, flayer: { isDisplayed: UIEventSource, layerDef: LayerConfig }) { + // This is a dynamic template with a fixed zoom level + url = url.replace("{z}", "" + zoomLevel) + const loadedTiles = new Set(); + const self = this; + this.onFail = (msg, url) => { + console.warn(`Could not load geojson layer from`, url, "due to", msg) + loadedTiles.add(url); // We add the url to the 'loadedTiles' in order to not reload it in the future + } + + const neededTiles = locationControl.map( + _ => { + // Yup, this is cheating to just get the bounds here + const bounds = State.state.leafletMap.data.getBounds() + const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest()) + const needed = Utils.MapRange(tileRange, (x, y) => { + return url.replace("{x}", "" + x).replace("{y}", "" + y); + }) + return new Set(needed); + } + , [flayer.isDisplayed]); + neededTiles.stabilized(250).addCallback((needed: Set) => { + if (needed === undefined) { + return; + } + if (!flayer.isDisplayed.data) { + // No need to download! - the layer is disabled + return; + } + + if (locationControl.data.zoom < flayer.layerDef.minzoom) { + return; + } + + needed.forEach(neededTile => { + if (loadedTiles.has(neededTile)) { + return; + } + + loadedTiles.add(neededTile) + self.LoadJSONFrom(neededTile) + + }) + }) + + } + private LoadJSONFrom(url: string) { const eventSource = this.features; const self = this; diff --git a/Logic/FeatureSource/OsmApiFeatureSource.ts b/Logic/FeatureSource/OsmApiFeatureSource.ts index 909aa1f4df..4805511d62 100644 --- a/Logic/FeatureSource/OsmApiFeatureSource.ts +++ b/Logic/FeatureSource/OsmApiFeatureSource.ts @@ -1,21 +1,93 @@ import FeatureSource from "./FeatureSource"; import {UIEventSource} from "../UIEventSource"; import {OsmObject} from "../Osm/OsmObject"; +import State from "../../State"; +import {Utils} from "../../Utils"; +import Loc from "../../Models/Loc"; export default class OsmApiFeatureSource implements FeatureSource { public readonly features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]); public readonly name: string = "OsmApiFeatureSource"; - - - public load(id: string){ + private readonly loadedTiles: Set = new Set(); + + constructor(location: UIEventSource) { + /* const self = this + location.addCallback(_ => { + self.loadArea() + }) + */ + + } + + + public load(id: string) { console.log("Updating from OSM API: ", id) OsmObject.DownloadObject(id, (element, meta) => { const geojson = element.asGeoJson(); console.warn(geojson) geojson.id = geojson.properties.id; - this.features.setData([{feature:geojson, freshness: meta["_last_edit:timestamp"]}]) + this.features.setData([{feature: geojson, freshness: meta["_last_edit:timestamp"]}]) }) } + /** + * Loads the current inview-area + */ + public loadArea(z: number = 16): boolean { + const layers = State.state.filteredLayers.data; + + const disabledLayers = layers.filter(layer => layer.layerDef.source.overpassScript !== undefined || layer.layerDef.source.geojsonSource !== undefined) + if (disabledLayers.length > 0) { + return false; + } + const loc = State.state.locationControl.data; + if (loc.zoom < 16) { + return false; + } + if (State.state.leafletMap.data === undefined) { + return false; // Not yet inited + } + const bounds = State.state.leafletMap.data.getBounds() + const tileRange = Utils.TileRangeBetween(z, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest()) + const self = this; + Utils.MapRange(tileRange, (x, y) => { + const key = x + "/" + y; + if (self.loadedTiles.has(key)) { + return; + } + + self.loadedTiles.add(key); + + const bounds = Utils.tile_bounds(z, x, y); + console.log("Loading OSM data tile", z, x, y, " with bounds", bounds) + OsmObject.LoadArea(bounds, objects => { + const keptGeoJson: {feature:any, freshness: Date}[] = [] + // Which layer does the object match? + for (const object of objects) { + + for (const flayer of layers) { + const layer = flayer.layerDef; + const tags = object.tags + const doesMatch = layer.source.osmTags.matchesProperties(tags); + if (doesMatch) { + const geoJson = object.asGeoJson(); + geoJson._matching_layer_id = layer.id + keptGeoJson.push({feature: geoJson, freshness: object.timestamp}) + break; + } + + } + + } + + self.features.setData(keptGeoJson) + }); + + }); + + return true; + + } + } \ No newline at end of file diff --git a/Logic/Osm/OsmObject.ts b/Logic/Osm/OsmObject.ts index 4fe9e5b035..ed74876be2 100644 --- a/Logic/Osm/OsmObject.ts +++ b/Logic/Osm/OsmObject.ts @@ -9,6 +9,7 @@ export abstract class OsmObject { tags: {} = {}; version: number; public changed: boolean = false; + timestamp: Date; protected constructor(type: string, id: number) { this.id = id; @@ -24,7 +25,6 @@ export abstract class OsmObject { const idN = splitted[1]; const newContinuation = (element: OsmObject, meta: OsmObjectMeta) => { - console.log("Received: ", element, "with meta", meta); continuation(element, meta); } @@ -39,6 +39,75 @@ export abstract class OsmObject { } } + public static DownloadHistory(id: string, continuation: (versions: OsmObject[]) => void): void { + const splitted = id.split("/"); + const type = splitted[0]; + const idN = splitted[1]; + $.getJSON("https://openStreetMap.org/api/0.6/" + type + "/" + idN + "/history", data => { + const elements: any[] = data.elements; + const osmObjects: OsmObject[] = [] + for (const element of elements) { + let osmObject: OsmObject = null + switch (type) { + case("node"): + osmObject = new OsmNode(idN); + break; + case("way"): + osmObject = new OsmWay(idN); + break; + case("relation"): + osmObject = new OsmRelation(idN); + break; + } + osmObject?.LoadData(element); + osmObject?.SaveExtraData(element, []); + osmObjects.push(osmObject) + } + continuation(osmObjects) + }) + } + + //Loads an area from the OSM-api. + // bounds should be: [[maxlat, minlon], [minlat, maxlon]] (same as Utils.tile_bounds) + public static LoadArea(bounds: [[number, number], [number, number]], callback: (objects: OsmObject[]) => void) { + const minlon = bounds[0][1] + const maxlon = bounds[1][1] + const minlat = bounds[1][0] + const maxlat = bounds[0][0]; + const url = `https://www.openstreetmap.org/api/0.6/map.json?bbox=${minlon},${minlat},${maxlon},${maxlat}` + $.getJSON(url, data => { + const elements: any[] = data.elements; + const objects: OsmObject[] = []; + const allNodes: Map = new Map() + for (const element of elements) { + const type = element.type; + const idN = element.id; + let osmObject: OsmObject = null + switch (type) { + case("node"): + const node = new OsmNode(idN); + allNodes.set(idN, node); + osmObject = node + node.SaveExtraData(element); + break; + case("way"): + osmObject = new OsmWay(idN); + const nodes = element.nodes.map(i => allNodes.get(i)); + osmObject.SaveExtraData(element, nodes) + break; + case("relation"): + osmObject = new OsmRelation(idN); + osmObject.SaveExtraData(element, []) + break; + } + osmObject.LoadData(element) + objects.push(osmObject) + } + callback(objects); + + }) + } + public static DownloadAll(neededIds, knownElements: any = {}, continuation: ((knownObjects: any) => void)) { // local function which downloads all the objects one by one // this is one big loop, running one download, then rerunning the entire function @@ -53,7 +122,6 @@ export abstract class OsmObject { return; } - console.log("Downloading ", neededId); OsmObject.DownloadObject(neededId, function (element) { knownElements[neededId] = element; // assign the element for later, continue downloading the next element @@ -90,17 +158,10 @@ export abstract class OsmObject { const url = "https://www.openstreetmap.org/api/0.6/" + this.type + "/" + this.id + full; $.getJSON(url, function (data) { const element = data.elements[data.elements.length - 1]; - self.tags = element.tags; - const tgs = self.tags; - tgs["_last_edit:contributor"] = element.user - tgs["_last_edit:contributor:uid"] = element.uid - tgs["_last_edit:changeset"] = element.changeset - tgs["_last_edit:timestamp"] = element.timestamp - tgs["_version_number"] = element.version - tgs["id"] = self.type+"/"+self.id; - self.version = element.version; + self.LoadData(element) self.SaveExtraData(element, data.elements); + continuation(self, { "_last_edit:contributor": element.user, "_last_edit:contributor:uid": element.uid, @@ -119,7 +180,7 @@ export abstract class OsmObject { if (oldV == v) { return; } - console.log("WARNING: overwriting ", oldV, " with ", v, " for key ", k) + console.log("Overwriting ", oldV, " with ", v, " for key ", k) } this.tags[k] = v; if (v === undefined || v === "") { @@ -136,6 +197,25 @@ export abstract class OsmObject { } return 'version="' + this.version + '"'; } + + private LoadData(element: any): void { + this.tags = element.tags ?? this.tags; + this.version = element.version; + this.timestamp = element.timestamp; + const tgs = this.tags; + if(element.tags === undefined){ + // Simple node which is part of a way - not important + return; + } + tgs["_last_edit:contributor"] = element.user + tgs["_last_edit:contributor:uid"] = element.uid + tgs["_last_edit:changeset"] = element.changeset + tgs["_last_edit:timestamp"] = element.timestamp + tgs["_version_number"] = element.version + tgs["id"] = this.type + "/" + this.id; + + + } } @@ -219,21 +299,20 @@ export class OsmWay extends OsmObject { ' \n'; } - SaveExtraData(element, allNodes) { - console.log("Way-extras: ", allNodes) + SaveExtraData(element, allNodes: OsmNode[]) { + + let latSum = 0 + let lonSum = 0 - for (const node of allNodes) { - if (node.type === "node") { - const n = new OsmNode(node.id); - n.SaveExtraData(node) - const cp = n.centerpoint(); - this.coordinates.push(cp); - } + const cp = node.centerpoint(); + this.coordinates.push(cp); + latSum = cp[0] + lonSum = cp[1] } let count = this.coordinates.length; - this.lat = this.coordinates.map(c => c[0]).reduce((a, b) => a + b, 0) / count; - this.lon = this.coordinates.map(c => c[1]).reduce((a, b) => a + b, 0) / count; + this.lat = latSum / count; + this.lon = lonSum / count; this.nodes = element.nodes; } diff --git a/State.ts b/State.ts index 79437b0edf..96dbacbeda 100644 --- a/State.ts +++ b/State.ts @@ -59,7 +59,7 @@ export default class State { public layerUpdater: UpdateFromOverpass; - public osmApiFeatureSource : OsmApiFeatureSource = new OsmApiFeatureSource(); + public osmApiFeatureSource : OsmApiFeatureSource ; public filteredLayers: UIEventSource<{ @@ -220,6 +220,7 @@ export default class State { this.allElements = new ElementStorage(); this.changes = new Changes(); + this.osmApiFeatureSource = new OsmApiFeatureSource(this.locationControl) new PendingChangesUploader(this.changes, this.selectedElement); diff --git a/Utils.ts b/Utils.ts index aca8abc38b..f780e7f8d5 100644 --- a/Utils.ts +++ b/Utils.ts @@ -189,7 +189,7 @@ export class Utils { * @param z * @param x * @param y - * @returns [[lat, lon], [lat, lon]] + * @returns [[maxlat, minlon], [minlat, maxlon]] */ static tile_bounds(z: number, x: number, y: number): [[number, number], [number, number]] { return [[Utils.tile2lat(y, z), Utils.tile2long(x, z)], [Utils.tile2lat(y + 1, z), Utils.tile2long(x + 1, z)]] @@ -201,8 +201,8 @@ export class Utils { static embedded_tile(lat: number, lon: number, z: number): { x: number, y: number, z: number } { return {x: Utils.lon2tile(lon, z), y: Utils.lat2tile(lat, z), z: z} } - - static TileRangeBetween(zoomlevel: number, lat0: number, lon0: number, lat1:number, lon1: number) : TileRange{ + + static TileRangeBetween(zoomlevel: number, lat0: number, lon0: number, lat1: number, lon1: number): TileRange { const t0 = Utils.embedded_tile(lat0, lon0, zoomlevel) const t1 = Utils.embedded_tile(lat1, lon1, zoomlevel) @@ -211,12 +211,12 @@ export class Utils { const ystart = Math.min(t0.y, t1.y) const yend = Math.max(t0.y, t1.y) const total = (1 + xend - xstart) * (1 + yend - ystart) - + return { xstart: xstart, xend: xend, ystart: ystart, - yend: yend, + yend: yend, total: total, zoomlevel: zoomlevel } @@ -274,8 +274,18 @@ export class Utils { private static lat2tile(lat, zoom) { return (Math.floor((1 - Math.log(Math.tan(lat * Math.PI / 180) + 1 / Math.cos(lat * Math.PI / 180)) / Math.PI) / 2 * Math.pow(2, zoom))); } -} + public static MapRange (tileRange: TileRange, f: (x: number, y: number) => T): T[] { + const result : T[] = [] + for (let x = tileRange.xstart; x <= tileRange.xend; x++) { + for (let y = tileRange.ystart; y <= tileRange.yend; y++) { + const t= f(x, y); + result.push(t) + } + } + return result; + } +} export interface TileRange{ xstart: number, @@ -284,4 +294,5 @@ export interface TileRange{ yend: number, total: number, zoomlevel: number + } \ No newline at end of file