midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Experimenting with using the overpass API directly

Browse source
This commit is contained in:
pietervdvn 2021-05-06 03:03:54 +02:00
parent a0c1bc2137
commit 1476ab0407
6 changed files with 251 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
InitUiElements.ts
View file

@ -394,6 +394,9 @@ export class InitUiElements {
const updater = new LoadFromOverpass(state.locationControl, state.layoutToUse, state.leafletMap);
State.state.layerUpdater = updater;
const source = new FeaturePipeline(state.filteredLayers,
updater,
state.osmApiFeatureSource,

106
Logic/FeatureSource/GeoJsonSource.ts
View file

@ -20,9 +20,9 @@ export default class GeoJsonSource implements FeatureSource {
private readonly layerId: string;
private readonly seenids: Set<string> = new Set<string>()
private constructor(locationControl: UIEventSource<Loc>,
flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig },
onFail?: ((errorMsg: any) => void)) {
private constructor(locationControl: UIEventSource<Loc>,
flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig },
onFail?: ((errorMsg: any) => void)) {
this.layerId = flayer.layerDef.id;
let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id);
this.name = "GeoJsonSource of " + url;
@ -33,7 +33,7 @@ export default class GeoJsonSource implements FeatureSource {
if (zoomLevel === undefined) {
// This is a classic, static geojson layer
if (onFail === undefined) {
onFail = errorMsg => {
onFail = _ => {
}
}
this.onFail = onFail;
@ -43,57 +43,6 @@ export default class GeoJsonSource implements FeatureSource {
this.ConfigureDynamicLayer(url, zoomLevel, locationControl, flayer)
}
}
private ConfigureDynamicLayer(url: string, zoomLevel: number, locationControl: UIEventSource<Loc>, flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }){
// This is a dynamic template with a fixed zoom level
url = url.replace("{z}", "" + zoomLevel)
const loadedTiles = new Set<string>();
const self = this;
this.onFail = (msg, url) => {
console.warn(`Could not load geojson layer from`, url, "due to", msg)
loadedTiles.add(url); // We add the url to the 'loadedTiles' in order to not reload it in the future
}
const neededTiles = locationControl.map(
location => {
// Yup, this is cheating to just get the bounds here
const bounds = State.state.leafletMap.data.getBounds()
const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const needed = new Set<string>();
for (let x = tileRange.xstart; x <= tileRange.xend; x++) {
for (let y = tileRange.ystart; y <= tileRange.yend; y++) {
let neededUrl = url.replace("{x}", "" + x).replace("{y}", "" + y);
needed.add(neededUrl)
}
}
return needed;
}
, [flayer.isDisplayed]);
neededTiles.stabilized(250).addCallback((needed: Set<string>) => {
if (needed === undefined) {
return;
}
if (!flayer.isDisplayed.data) {
// No need to download! - the layer is disabled
return;
}
if(locationControl.data.zoom < flayer.layerDef.minzoom){
return;
}
needed.forEach(neededTile => {
if (loadedTiles.has(neededTile)) {
return;
}
loadedTiles.add(neededTile)
self.LoadJSONFrom(neededTile)
})
})
}
/**
* Merges together the layers which have the same source
@ -152,6 +101,53 @@ export default class GeoJsonSource implements FeatureSource {
}
private ConfigureDynamicLayer(url: string, zoomLevel: number, locationControl: UIEventSource<Loc>, flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }) {
// This is a dynamic template with a fixed zoom level
url = url.replace("{z}", "" + zoomLevel)
const loadedTiles = new Set<string>();
const self = this;
this.onFail = (msg, url) => {
console.warn(`Could not load geojson layer from`, url, "due to", msg)
loadedTiles.add(url); // We add the url to the 'loadedTiles' in order to not reload it in the future
}
const neededTiles = locationControl.map(
_ => {
// Yup, this is cheating to just get the bounds here
const bounds = State.state.leafletMap.data.getBounds()
const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const needed = Utils.MapRange(tileRange, (x, y) => {
return url.replace("{x}", "" + x).replace("{y}", "" + y);
})
return new Set<string>(needed);
}
, [flayer.isDisplayed]);
neededTiles.stabilized(250).addCallback((needed: Set<string>) => {
if (needed === undefined) {
return;
}
if (!flayer.isDisplayed.data) {
// No need to download! - the layer is disabled
return;
}
if (locationControl.data.zoom < flayer.layerDef.minzoom) {
return;
}
needed.forEach(neededTile => {
if (loadedTiles.has(neededTile)) {
return;
}
loadedTiles.add(neededTile)
self.LoadJSONFrom(neededTile)
})
})
}
private LoadJSONFrom(url: string) {
const eventSource = this.features;
const self = this;

80
Logic/FeatureSource/OsmApiFeatureSource.ts
View file

@ -1,21 +1,93 @@
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import {OsmObject} from "../Osm/OsmObject";
import State from "../../State";
import {Utils} from "../../Utils";
import Loc from "../../Models/Loc";
export default class OsmApiFeatureSource implements FeatureSource {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]);
public readonly name: string = "OsmApiFeatureSource";
public load(id: string){
private readonly loadedTiles: Set<string> = new Set<string>();
constructor(location: UIEventSource<Loc>) {
/* const self = this
location.addCallback(_ => {
self.loadArea()
})
*/
}
public load(id: string) {
console.log("Updating from OSM API: ", id)
OsmObject.DownloadObject(id, (element, meta) => {
const geojson = element.asGeoJson();
console.warn(geojson)
geojson.id = geojson.properties.id;
this.features.setData([{feature:geojson, freshness: meta["_last_edit:timestamp"]}])
this.features.setData([{feature: geojson, freshness: meta["_last_edit:timestamp"]}])
})
}
/**
* Loads the current inview-area
*/
public loadArea(z: number = 16): boolean {
const layers = State.state.filteredLayers.data;
const disabledLayers = layers.filter(layer => layer.layerDef.source.overpassScript !== undefined || layer.layerDef.source.geojsonSource !== undefined)
if (disabledLayers.length > 0) {
return false;
}
const loc = State.state.locationControl.data;
if (loc.zoom < 16) {
return false;
}
if (State.state.leafletMap.data === undefined) {
return false; // Not yet inited
}
const bounds = State.state.leafletMap.data.getBounds()
const tileRange = Utils.TileRangeBetween(z, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const self = this;
Utils.MapRange(tileRange, (x, y) => {
const key = x + "/" + y;
if (self.loadedTiles.has(key)) {
return;
}
self.loadedTiles.add(key);
const bounds = Utils.tile_bounds(z, x, y);
console.log("Loading OSM data tile", z, x, y, " with bounds", bounds)
OsmObject.LoadArea(bounds, objects => {
const keptGeoJson: {feature:any, freshness: Date}[] = []
// Which layer does the object match?
for (const object of objects) {
for (const flayer of layers) {
const layer = flayer.layerDef;
const tags = object.tags
const doesMatch = layer.source.osmTags.matchesProperties(tags);
if (doesMatch) {
const geoJson = object.asGeoJson();
geoJson._matching_layer_id = layer.id
keptGeoJson.push({feature: geoJson, freshness: object.timestamp})
break;
}
}
}
self.features.setData(keptGeoJson)
});
});
return true;
}
}

125
Logic/Osm/OsmObject.ts
View file

@ -9,6 +9,7 @@ export abstract class OsmObject {
tags: {} = {};
version: number;
public changed: boolean = false;
timestamp: Date;
protected constructor(type: string, id: number) {
this.id = id;
@ -24,7 +25,6 @@ export abstract class OsmObject {
const idN = splitted[1];
const newContinuation = (element: OsmObject, meta: OsmObjectMeta) => {
console.log("Received: ", element, "with meta", meta);
continuation(element, meta);
}
@ -39,6 +39,75 @@ export abstract class OsmObject {
}
}
public static DownloadHistory(id: string, continuation: (versions: OsmObject[]) => void): void {
const splitted = id.split("/");
const type = splitted[0];
const idN = splitted[1];
$.getJSON("https://openStreetMap.org/api/0.6/" + type + "/" + idN + "/history", data => {
const elements: any[] = data.elements;
const osmObjects: OsmObject[] = []
for (const element of elements) {
let osmObject: OsmObject = null
switch (type) {
case("node"):
osmObject = new OsmNode(idN);
break;
case("way"):
osmObject = new OsmWay(idN);
break;
case("relation"):
osmObject = new OsmRelation(idN);
break;
}
osmObject?.LoadData(element);
osmObject?.SaveExtraData(element, []);
osmObjects.push(osmObject)
}
continuation(osmObjects)
})
}
//Loads an area from the OSM-api.
// bounds should be: [[maxlat, minlon], [minlat, maxlon]] (same as Utils.tile_bounds)
public static LoadArea(bounds: [[number, number], [number, number]], callback: (objects: OsmObject[]) => void) {
const minlon = bounds[0][1]
const maxlon = bounds[1][1]
const minlat = bounds[1][0]
const maxlat = bounds[0][0];
const url = `https://www.openstreetmap.org/api/0.6/map.json?bbox=${minlon},${minlat},${maxlon},${maxlat}`
$.getJSON(url, data => {
const elements: any[] = data.elements;
const objects: OsmObject[] = [];
const allNodes: Map<number, OsmNode> = new Map<number, OsmNode>()
for (const element of elements) {
const type = element.type;
const idN = element.id;
let osmObject: OsmObject = null
switch (type) {
case("node"):
const node = new OsmNode(idN);
allNodes.set(idN, node);
osmObject = node
node.SaveExtraData(element);
break;
case("way"):
osmObject = new OsmWay(idN);
const nodes = element.nodes.map(i => allNodes.get(i));
osmObject.SaveExtraData(element, nodes)
break;
case("relation"):
osmObject = new OsmRelation(idN);
osmObject.SaveExtraData(element, [])
break;
}
osmObject.LoadData(element)
objects.push(osmObject)
}
callback(objects);
})
}
public static DownloadAll(neededIds, knownElements: any = {}, continuation: ((knownObjects: any) => void)) {
// local function which downloads all the objects one by one
// this is one big loop, running one download, then rerunning the entire function
@ -53,7 +122,6 @@ export abstract class OsmObject {
return;
}
console.log("Downloading ", neededId);
OsmObject.DownloadObject(neededId,
function (element) {
knownElements[neededId] = element; // assign the element for later, continue downloading the next element
@ -90,17 +158,10 @@ export abstract class OsmObject {
const url = "https://www.openstreetmap.org/api/0.6/" + this.type + "/" + this.id + full;
$.getJSON(url, function (data) {
const element = data.elements[data.elements.length - 1];
self.tags = element.tags;
const tgs = self.tags;
tgs["_last_edit:contributor"] = element.user
tgs["_last_edit:contributor:uid"] = element.uid
tgs["_last_edit:changeset"] = element.changeset
tgs["_last_edit:timestamp"] = element.timestamp
tgs["_version_number"] = element.version
tgs["id"] = self.type+"/"+self.id;
self.version = element.version;
self.LoadData(element)
self.SaveExtraData(element, data.elements);
continuation(self, {
"_last_edit:contributor": element.user,
"_last_edit:contributor:uid": element.uid,
@ -119,7 +180,7 @@ export abstract class OsmObject {
if (oldV == v) {
return;
}
console.log("WARNING: overwriting ", oldV, " with ", v, " for key ", k)
console.log("Overwriting ", oldV, " with ", v, " for key ", k)
}
this.tags[k] = v;
if (v === undefined || v === "") {
@ -136,6 +197,25 @@ export abstract class OsmObject {
}
return 'version="' + this.version + '"';
}
private LoadData(element: any): void {
this.tags = element.tags ?? this.tags;
this.version = element.version;
this.timestamp = element.timestamp;
const tgs = this.tags;
if(element.tags === undefined){
// Simple node which is part of a way - not important
return;
}
tgs["_last_edit:contributor"] = element.user
tgs["_last_edit:contributor:uid"] = element.uid
tgs["_last_edit:changeset"] = element.changeset
tgs["_last_edit:timestamp"] = element.timestamp
tgs["_version_number"] = element.version
tgs["id"] = this.type + "/" + this.id;
}
}
@ -219,21 +299,20 @@ export class OsmWay extends OsmObject {
' </way>\n';
}
SaveExtraData(element, allNodes) {
console.log("Way-extras: ", allNodes)
SaveExtraData(element, allNodes: OsmNode[]) {
let latSum = 0
let lonSum = 0
for (const node of allNodes) {
if (node.type === "node") {
const n = new OsmNode(node.id);
n.SaveExtraData(node)
const cp = n.centerpoint();
this.coordinates.push(cp);
}
const cp = node.centerpoint();
this.coordinates.push(cp);
latSum = cp[0]
lonSum = cp[1]
}
let count = this.coordinates.length;
this.lat = this.coordinates.map(c => c[0]).reduce((a, b) => a + b, 0) / count;
this.lon = this.coordinates.map(c => c[1]).reduce((a, b) => a + b, 0) / count;
this.lat = latSum / count;
this.lon = lonSum / count;
this.nodes = element.nodes;
}

3
State.ts
View file

@ -59,7 +59,7 @@ export default class State {
public layerUpdater: UpdateFromOverpass;
public osmApiFeatureSource : OsmApiFeatureSource = new OsmApiFeatureSource();
public osmApiFeatureSource : OsmApiFeatureSource ;
public filteredLayers: UIEventSource<{
@ -220,6 +220,7 @@ export default class State {
this.allElements = new ElementStorage();
this.changes = new Changes();
this.osmApiFeatureSource = new OsmApiFeatureSource(this.locationControl)
new PendingChangesUploader(this.changes, this.selectedElement);

23
Utils.ts
View file

@ -189,7 +189,7 @@ export class Utils {
* @param z
* @param x
* @param y
* @returns [[lat, lon], [lat, lon]]
* @returns [[maxlat, minlon], [minlat, maxlon]]
*/
static tile_bounds(z: number, x: number, y: number): [[number, number], [number, number]] {
return [[Utils.tile2lat(y, z), Utils.tile2long(x, z)], [Utils.tile2lat(y + 1, z), Utils.tile2long(x + 1, z)]]
@ -201,8 +201,8 @@ export class Utils {
static embedded_tile(lat: number, lon: number, z: number): { x: number, y: number, z: number } {
return {x: Utils.lon2tile(lon, z), y: Utils.lat2tile(lat, z), z: z}
}
static TileRangeBetween(zoomlevel: number, lat0: number, lon0: number, lat1:number, lon1: number) : TileRange{
static TileRangeBetween(zoomlevel: number, lat0: number, lon0: number, lat1: number, lon1: number): TileRange {
const t0 = Utils.embedded_tile(lat0, lon0, zoomlevel)
const t1 = Utils.embedded_tile(lat1, lon1, zoomlevel)
@ -211,12 +211,12 @@ export class Utils {
const ystart = Math.min(t0.y, t1.y)
const yend = Math.max(t0.y, t1.y)
const total = (1 + xend - xstart) * (1 + yend - ystart)
return {
xstart: xstart,
xend: xend,
ystart: ystart,
yend: yend,
yend: yend,
total: total,
zoomlevel: zoomlevel
}
@ -274,8 +274,18 @@ export class Utils {
private static lat2tile(lat, zoom) {
return (Math.floor((1 - Math.log(Math.tan(lat * Math.PI / 180) + 1 / Math.cos(lat * Math.PI / 180)) / Math.PI) / 2 * Math.pow(2, zoom)));
}
}
public static MapRange<T> (tileRange: TileRange, f: (x: number, y: number) => T): T[] {
const result : T[] = []
for (let x = tileRange.xstart; x <= tileRange.xend; x++) {
for (let y = tileRange.ystart; y <= tileRange.yend; y++) {
const t= f(x, y);
result.push(t)
}
}
return result;
}
}
export interface TileRange{
xstart: number,
@ -284,4 +294,5 @@ export interface TileRange{
yend: number,
total: number,
zoomlevel: number
}