Bastian-Greshake-Tzovaras/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Download needed relations completely, fixes 873

Browse source
This commit is contained in:
Pieter Vander Vennet 2022-06-24 18:12:39 +02:00
parent 732d4621ae
commit 4fd40c6935
6 changed files with 418 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

72
Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts
View file

@ -10,6 +10,7 @@ import {BBox} from "../../BBox";
import LayoutConfig from "../../../Models/ThemeConfig/LayoutConfig";
import {Or} from "../../Tags/Or";
import {TagsFilter} from "../../Tags/TagsFilter";
import {OsmObject} from "../../Osm/OsmObject";
/**
* If a tile is needed (requested via the UIEventSource in the constructor), will download the appropriate tile and pass it via 'handleTile'
@ -31,7 +32,7 @@ export default class OsmFeatureSource {
private readonly allowedTags: TagsFilter;
/**
*
*
* @param options: allowedFeatures is normally calculated from the layoutToUse
*/
constructor(options: {
@ -81,7 +82,7 @@ export default class OsmFeatureSource {
for (const neededTile of neededTiles) {
this.downloadedTiles.add(neededTile)
this.LoadTile(...Tiles.tile_from_index(neededTile))
await this.LoadTile(...Tiles.tile_from_index(neededTile))
}
} catch (e) {
console.error(e)
@ -90,7 +91,33 @@ export default class OsmFeatureSource {
}
}
private LoadTile(z, x, y): void {
/**
* The requested tile might only contain part of the relation.
*
* This method will download the full relation and return it as geojson if it was incomplete.
* If the feature is already complete (or is not a relation), the feature will be returned
*/
private async patchIncompleteRelations(feature: {properties: {id: string}},
originalJson: {elements: {type: "node" | "way" | "relation", id: number, } []}): Promise<any> {
if(!feature.properties.id.startsWith("relation")){
return feature
}
const relationSpec = originalJson.elements.find(f => "relation/"+f.id === feature.properties.id)
const members : {type: string, ref: number}[] = relationSpec["members"]
for (const member of members) {
const isFound = originalJson.elements.some(f => f.id === member.ref && f.type === member.type)
if (isFound) {
continue
}
// This member is missing. We redownload the entire relation instead
console.debug("Fetching incomplete relation "+feature.properties.id)
return (await OsmObject.DownloadObjectAsync(feature.properties.id)).asGeoJson()
}
return feature;
}
private async LoadTile(z, x, y): Promise<void> {
if (z > 25) {
throw "This is an absurd high zoom level"
}
@ -102,8 +129,11 @@ export default class OsmFeatureSource {
const bbox = BBox.fromTile(z, x, y)
const url = `${this._backend}/api/0.6/map?bbox=${bbox.minLon},${bbox.minLat},${bbox.maxLon},${bbox.maxLat}`
Utils.downloadJson(url).then(osmJson => {
let error = undefined;
try {
const osmJson = await Utils.downloadJson(url)
try {
console.log("Got tile", z, x, y, "from the osm api")
this.rawDataHandlers.forEach(handler => handler(osmJson, Tiles.tile_index(z, x, y)))
const geojson = OsmToGeoJson.default(osmJson,
@ -112,10 +142,15 @@ export default class OsmFeatureSource {
flatProperties: true
});
// The geojson contains _all_ features at the given location
// We only keep what is needed
geojson.features = geojson.features.filter(feature => this.allowedTags.matchesProperties(feature.properties))
for (let i = 0; i < geojson.features.length; i++) {
geojson.features[i] = await this.patchIncompleteRelations(geojson.features[i], osmJson)
}
geojson.features.forEach(f => {
f.properties["_backend"] = this._backend
})
@ -131,22 +166,25 @@ export default class OsmFeatureSource {
if (this.options.markTileVisited) {
this.options.markTileVisited(index)
}
} catch (e) {
console.error("Weird error: ", e)
}catch(e){
console.error("PANIC: got the tile from the OSM-api, but something crashed handling this tile")
error = e;
}
})
.catch(e => {
console.error("Could not download tile", z, x, y, "due to", e, "; retrying with smaller bounds")
if (e === "rate limited") {
return;
}
this.LoadTile(z + 1, x * 2, y * 2)
this.LoadTile(z + 1, 1 + x * 2, y * 2)
this.LoadTile(z + 1, x * 2, 1 + y * 2)
this.LoadTile(z + 1, 1 + x * 2, 1 + y * 2)
} catch (e) {
console.error("Could not download tile", z, x, y, "due to", e, "; retrying with smaller bounds")
if (e === "rate limited") {
return;
})
}
await this.LoadTile(z + 1, x * 2, y * 2)
await this.LoadTile(z + 1, 1 + x * 2, y * 2)
await this.LoadTile(z + 1, x * 2, 1 + y * 2)
await this.LoadTile(z + 1, 1 + x * 2, 1 + y * 2)
}
if(error !== undefined){
throw error;
}
}

39
Logic/Osm/OsmObject.ts
View file

@ -1,8 +1,8 @@
import {Utils} from "../../Utils";
import * as polygon_features from "../../assets/polygon-features.json";
import {Store, Stores, UIEventSource} from "../UIEventSource";
import {Store, UIEventSource} from "../UIEventSource";
import {BBox} from "../BBox";
import * as OsmToGeoJson from "osmtogeojson";
export abstract class OsmObject {
@ -38,6 +38,7 @@ export abstract class OsmObject {
throw "Backend URL must begin with http"
}
this.backendURL = url;
this.DownloadObject("id/5")
}
public static DownloadObject(id: string, forceRefresh: boolean = false): Store<OsmObject> {
@ -77,10 +78,10 @@ export abstract class OsmObject {
return undefined;
}
const full = (id.startsWith("way")) ? "/full" : "";
const full = (!id.startsWith("node")) ? "/full" : "";
const url = `${OsmObject.backendURL}api/0.6/${id}${full}`;
const rawData = await Utils.downloadJsonCached(url, 1000)
if(rawData === undefined){
const rawData = await Utils.downloadJsonCached(url, 10000)
if (rawData === undefined) {
return undefined
}
// A full query might contain more then just the requested object (e.g. nodes that are part of a way, where we only want the way)
@ -127,7 +128,7 @@ export abstract class OsmObject {
return data.elements.map(wayInfo => {
const rel = new OsmRelation(wayInfo.id)
rel.LoadData(wayInfo)
rel.SaveExtraData(wayInfo)
rel.SaveExtraData(wayInfo, undefined)
return rel
})
}
@ -196,7 +197,13 @@ export abstract class OsmObject {
break;
case("relation"):
osmObject = new OsmRelation(idN);
osmObject.SaveExtraData(element, [])
const allGeojsons = OsmToGeoJson.default({elements},
// @ts-ignore
{
flatProperties: true
});
const feature = allGeojsons.features.find(f => f.id === osmObject.type + "/" + osmObject.id)
osmObject.SaveExtraData(element, feature)
break;
}
@ -218,7 +225,7 @@ export abstract class OsmObject {
if (!tags.hasOwnProperty(tagsKey)) {
continue
}
const polyGuide : { values: Set<string>; blacklist: boolean } = OsmObject.polygonFeatures.get(tagsKey)
const polyGuide: { values: Set<string>; blacklist: boolean } = OsmObject.polygonFeatures.get(tagsKey)
if (polyGuide === undefined) {
continue
}
@ -228,12 +235,12 @@ export abstract class OsmObject {
}
// is the key contained? Then we have a match if the value is contained
const doesMatch = polyGuide.values.has(tags[tagsKey])
if(polyGuide.blacklist){
if (polyGuide.blacklist) {
return !doesMatch
}
return doesMatch
}
return false;
}
@ -260,7 +267,7 @@ export abstract class OsmObject {
public abstract asGeoJson(): any;
abstract SaveExtraData(element: any, allElements: OsmObject[]);
abstract SaveExtraData(element: any, allElements: OsmObject[] | any);
/**
* Generates the changeset-XML for tags
@ -431,7 +438,7 @@ export class OsmWay extends OsmObject {
private isPolygon(): boolean {
// Compare lat and lon seperately, as the coordinate array might not be a reference to the same object
if (this.coordinates[0][0] !== this.coordinates[this.coordinates.length - 1][0] ||
this.coordinates[0][1] !== this.coordinates[this.coordinates.length - 1][1] ) {
this.coordinates[0][1] !== this.coordinates[this.coordinates.length - 1][1]) {
return false; // Not closed
}
return OsmObject.isPolygon(this.tags)
@ -447,6 +454,8 @@ export class OsmRelation extends OsmObject {
role: string
}[];
private geojson = undefined
constructor(id: number) {
super("relation", id);
}
@ -472,11 +481,15 @@ ${members}${tags} </relation>
}
SaveExtraData(element) {
SaveExtraData(element, geojson) {
this.members = element.members;
this.geojson = geojson
}
asGeoJson(): any {
if (this.geojson !== undefined) {
return this.geojson;
}
throw "Not Implemented"
}
}