Download needed relations completely, fixes 873

This commit is contained in:
Pieter Vander Vennet 2022-06-24 18:12:39 +02:00
parent 732d4621ae
commit 4fd40c6935
6 changed files with 418 additions and 88 deletions

View file

@ -10,6 +10,7 @@ import {BBox} from "../../BBox";
import LayoutConfig from "../../../Models/ThemeConfig/LayoutConfig";
import {Or} from "../../Tags/Or";
import {TagsFilter} from "../../Tags/TagsFilter";
import {OsmObject} from "../../Osm/OsmObject";
/**
* If a tile is needed (requested via the UIEventSource in the constructor), will download the appropriate tile and pass it via 'handleTile'
@ -81,7 +82,7 @@ export default class OsmFeatureSource {
for (const neededTile of neededTiles) {
this.downloadedTiles.add(neededTile)
this.LoadTile(...Tiles.tile_from_index(neededTile))
await this.LoadTile(...Tiles.tile_from_index(neededTile))
}
} catch (e) {
console.error(e)
@ -90,7 +91,33 @@ export default class OsmFeatureSource {
}
}
private LoadTile(z, x, y): void {
/**
* The requested tile might only contain part of the relation.
*
* This method will download the full relation and return it as geojson if it was incomplete.
* If the feature is already complete (or is not a relation), the feature will be returned
*/
private async patchIncompleteRelations(feature: {properties: {id: string}},
originalJson: {elements: {type: "node" | "way" | "relation", id: number, } []}): Promise<any> {
if(!feature.properties.id.startsWith("relation")){
return feature
}
const relationSpec = originalJson.elements.find(f => "relation/"+f.id === feature.properties.id)
const members : {type: string, ref: number}[] = relationSpec["members"]
for (const member of members) {
const isFound = originalJson.elements.some(f => f.id === member.ref && f.type === member.type)
if (isFound) {
continue
}
// This member is missing. We redownload the entire relation instead
console.debug("Fetching incomplete relation "+feature.properties.id)
return (await OsmObject.DownloadObjectAsync(feature.properties.id)).asGeoJson()
}
return feature;
}
private async LoadTile(z, x, y): Promise<void> {
if (z > 25) {
throw "This is an absurd high zoom level"
}
@ -102,8 +129,11 @@ export default class OsmFeatureSource {
const bbox = BBox.fromTile(z, x, y)
const url = `${this._backend}/api/0.6/map?bbox=${bbox.minLon},${bbox.minLat},${bbox.maxLon},${bbox.maxLat}`
Utils.downloadJson(url).then(osmJson => {
let error = undefined;
try {
const osmJson = await Utils.downloadJson(url)
try {
console.log("Got tile", z, x, y, "from the osm api")
this.rawDataHandlers.forEach(handler => handler(osmJson, Tiles.tile_index(z, x, y)))
const geojson = OsmToGeoJson.default(osmJson,
@ -112,10 +142,15 @@ export default class OsmFeatureSource {
flatProperties: true
});
// The geojson contains _all_ features at the given location
// We only keep what is needed
geojson.features = geojson.features.filter(feature => this.allowedTags.matchesProperties(feature.properties))
for (let i = 0; i < geojson.features.length; i++) {
geojson.features[i] = await this.patchIncompleteRelations(geojson.features[i], osmJson)
}
geojson.features.forEach(f => {
f.properties["_backend"] = this._backend
})
@ -131,22 +166,25 @@ export default class OsmFeatureSource {
if (this.options.markTileVisited) {
this.options.markTileVisited(index)
}
} catch (e) {
console.error("Weird error: ", e)
}catch(e){
console.error("PANIC: got the tile from the OSM-api, but something crashed handling this tile")
error = e;
}
})
.catch(e => {
} catch (e) {
console.error("Could not download tile", z, x, y, "due to", e, "; retrying with smaller bounds")
if (e === "rate limited") {
return;
}
this.LoadTile(z + 1, x * 2, y * 2)
this.LoadTile(z + 1, 1 + x * 2, y * 2)
this.LoadTile(z + 1, x * 2, 1 + y * 2)
this.LoadTile(z + 1, 1 + x * 2, 1 + y * 2)
return;
})
await this.LoadTile(z + 1, x * 2, y * 2)
await this.LoadTile(z + 1, 1 + x * 2, y * 2)
await this.LoadTile(z + 1, x * 2, 1 + y * 2)
await this.LoadTile(z + 1, 1 + x * 2, 1 + y * 2)
}
if(error !== undefined){
throw error;
}
}

View file

@ -1,8 +1,8 @@
import {Utils} from "../../Utils";
import * as polygon_features from "../../assets/polygon-features.json";
import {Store, Stores, UIEventSource} from "../UIEventSource";
import {Store, UIEventSource} from "../UIEventSource";
import {BBox} from "../BBox";
import * as OsmToGeoJson from "osmtogeojson";
export abstract class OsmObject {
@ -38,6 +38,7 @@ export abstract class OsmObject {
throw "Backend URL must begin with http"
}
this.backendURL = url;
this.DownloadObject("id/5")
}
public static DownloadObject(id: string, forceRefresh: boolean = false): Store<OsmObject> {
@ -77,10 +78,10 @@ export abstract class OsmObject {
return undefined;
}
const full = (id.startsWith("way")) ? "/full" : "";
const full = (!id.startsWith("node")) ? "/full" : "";
const url = `${OsmObject.backendURL}api/0.6/${id}${full}`;
const rawData = await Utils.downloadJsonCached(url, 1000)
if(rawData === undefined){
const rawData = await Utils.downloadJsonCached(url, 10000)
if (rawData === undefined) {
return undefined
}
// A full query might contain more then just the requested object (e.g. nodes that are part of a way, where we only want the way)
@ -127,7 +128,7 @@ export abstract class OsmObject {
return data.elements.map(wayInfo => {
const rel = new OsmRelation(wayInfo.id)
rel.LoadData(wayInfo)
rel.SaveExtraData(wayInfo)
rel.SaveExtraData(wayInfo, undefined)
return rel
})
}
@ -196,7 +197,13 @@ export abstract class OsmObject {
break;
case("relation"):
osmObject = new OsmRelation(idN);
osmObject.SaveExtraData(element, [])
const allGeojsons = OsmToGeoJson.default({elements},
// @ts-ignore
{
flatProperties: true
});
const feature = allGeojsons.features.find(f => f.id === osmObject.type + "/" + osmObject.id)
osmObject.SaveExtraData(element, feature)
break;
}
@ -218,7 +225,7 @@ export abstract class OsmObject {
if (!tags.hasOwnProperty(tagsKey)) {
continue
}
const polyGuide : { values: Set<string>; blacklist: boolean } = OsmObject.polygonFeatures.get(tagsKey)
const polyGuide: { values: Set<string>; blacklist: boolean } = OsmObject.polygonFeatures.get(tagsKey)
if (polyGuide === undefined) {
continue
}
@ -228,7 +235,7 @@ export abstract class OsmObject {
}
// is the key contained? Then we have a match if the value is contained
const doesMatch = polyGuide.values.has(tags[tagsKey])
if(polyGuide.blacklist){
if (polyGuide.blacklist) {
return !doesMatch
}
return doesMatch
@ -260,7 +267,7 @@ export abstract class OsmObject {
public abstract asGeoJson(): any;
abstract SaveExtraData(element: any, allElements: OsmObject[]);
abstract SaveExtraData(element: any, allElements: OsmObject[] | any);
/**
* Generates the changeset-XML for tags
@ -431,7 +438,7 @@ export class OsmWay extends OsmObject {
private isPolygon(): boolean {
// Compare lat and lon seperately, as the coordinate array might not be a reference to the same object
if (this.coordinates[0][0] !== this.coordinates[this.coordinates.length - 1][0] ||
this.coordinates[0][1] !== this.coordinates[this.coordinates.length - 1][1] ) {
this.coordinates[0][1] !== this.coordinates[this.coordinates.length - 1][1]) {
return false; // Not closed
}
return OsmObject.isPolygon(this.tags)
@ -447,6 +454,8 @@ export class OsmRelation extends OsmObject {
role: string
}[];
private geojson = undefined
constructor(id: number) {
super("relation", id);
}
@ -472,11 +481,15 @@ ${members}${tags} </relation>
}
SaveExtraData(element) {
SaveExtraData(element, geojson) {
this.members = element.members;
this.geojson = geojson
}
asGeoJson(): any {
if (this.geojson !== undefined) {
return this.geojson;
}
throw "Not Implemented"
}
}

View file

@ -10,15 +10,205 @@ import {Tag} from "../../../Logic/Tags/Tag";
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig";
import {expect} from "chai";
console.log(process.cwd())
let data = JSON.parse(readFileSync("./test/Logic/FeatureSource/osmdata.json", "utf8"))
const expected = {
"type": "Feature",
"id": "relation/5759328",
"properties": {
"timestamp": "2022-06-10T00:46:55Z",
"version": 6,
"changeset": 122187206,
"user": "Pieter Vander Vennet",
"uid": 3818858,
"amenity": "school",
"isced:2011:level": "vocational_lower_secondary;vocational_upper_secondary",
"name": "Koninklijk Technisch Atheneum Pro Technica",
"school:gender": "mixed",
"type": "multipolygon",
"website": "http://ktahalle.be/",
"id": "relation/5759328",
"_backend":"https://osm.org"
},
"geometry": {
"type": "MultiPolygon",
"coordinates": [
[
[
[
4.2461832,
50.7335751
],
[
4.2463167,
50.7336785
],
[
4.2463473,
50.7337021
],
[
4.2464497,
50.7337814
],
[
4.2471698,
50.7343389
],
[
4.2469541,
50.7344768
],
[
4.2467571,
50.7346116
],
[
4.2467727,
50.7346199
],
[
4.2465714,
50.7347511
],
[
4.2462398,
50.7349687
],
[
4.2453546,
50.734601
],
[
4.2451895,
50.7345103
],
[
4.2448867,
50.7342629
],
[
4.244899,
50.7342069
],
[
4.2461832,
50.7335751
]
]
],
[
[
[
4.2444209,
50.7353737
],
[
4.2439986,
50.7352034
],
[
4.2440303,
50.7351755
],
[
4.2440602,
50.7351058
],
[
4.2439776,
50.7350326
],
[
4.2439558,
50.7350132
],
[
4.2438246,
50.7348961
],
[
4.2437848,
50.73486
],
[
4.2436555,
50.7347455
],
[
4.2435905,
50.734689
],
[
4.2435494,
50.7346601
],
[
4.2435038,
50.7346256
],
[
4.2434769,
50.7346026
],
[
4.2430948,
50.734275
],
[
4.2427978,
50.7340052
],
[
4.2430556,
50.7338391
],
[
4.2438957,
50.7334942
],
[
4.2440204,
50.7336368
],
[
4.2442806,
50.7338922
],
[
4.2444173,
50.7340119
],
[
4.2447379,
50.7342925
],
[
4.2450107,
50.7345294
],
[
4.2450236,
50.7346021
],
[
4.2449643,
50.7347019
],
[
4.244711,
50.7350821
],
[
4.2444209,
50.7353737
]
]
]
]
}
}
describe("OsmFeatureSource", () => {
it("should work", (done) => {
ScriptUtils.fixUtils()
Utils.injectJsonDownloadForTests("https://osm.org/api/0.6/map?bbox=4.24346923828125,50.732978448277514,4.2462158203125,50.73471682490244", data)
function test(done: () => void){
let fetchedTile = undefined;
const neededTiles = new UIEventSource<number[]>([Tiles.tile_index(17, 67081, 44033)]);
new OsmFeatureSource({
@ -41,6 +231,7 @@ describe("OsmFeatureSource", () => {
_backend: 'https://osm.org'
})
expect(data.geometry.type).eq("MultiPolygon")
expect(data).deep.eq(expected)
done()
},
isActive: new UIEventSource<boolean>(true),
@ -66,5 +257,22 @@ describe("OsmFeatureSource", () => {
])
}
})
}
describe("OsmFeatureSource", () => {
it("downloading the full school should give a multipolygon", (done) => {
ScriptUtils.fixUtils()
let data = JSON.parse(readFileSync("./test/Logic/FeatureSource/osmdata.json", "utf8"))
Utils.injectJsonDownloadForTests("https://osm.org/api/0.6/map?bbox=4.24346923828125,50.732978448277514,4.2462158203125,50.73471682490244", data)
test(done)
})
it("downloading the partial school polygon should give a multipolygon", (done) => {
ScriptUtils.fixUtils()
Utils.injectJsonDownloadForTests("https://www.openstreetmap.org/api/0.6/relation/5759328/full", JSON.parse(readFileSync("./test/data/relation_5759328.json","UTF-8")))
let data = JSON.parse(readFileSync("./test/Logic/FeatureSource/small_box.json", "utf8"))
Utils.injectJsonDownloadForTests("https://osm.org/api/0.6/map?bbox=4.24346923828125,50.732978448277514,4.2462158203125,50.73471682490244", data)
test(done)
})
})

File diff suppressed because one or more lines are too long

View file

@ -71,7 +71,7 @@ describe("RelationSplitHandler", () => {
)
Utils.injectJsonDownloadForTests(
"https://www.openstreetmap.org/api/0.6/relation/9572808",
"https://www.openstreetmap.org/api/0.6/relation/9572808/full",
{
"version": "0.6",
"generator": "CGImap 0.8.5 (3128319 spike-07.openstreetmap.org)",
@ -266,7 +266,7 @@ describe("RelationSplitHandler", () => {
}
)
Utils.injectJsonDownloadForTests(
"https://www.openstreetmap.org/api/0.6/relation/4374576",
"https://www.openstreetmap.org/api/0.6/relation/4374576/full",
{
"version": "0.6",
"generator": "CGImap 0.8.5 (1266692 spike-06.openstreetmap.org)",
@ -552,7 +552,7 @@ describe("RelationSplitHandler", () => {
it("should split all cycling relation (split 295132739)",
async () => {
// Lets mimick a split action of https://www.openstreetmap.org/way/295132739
// Lets mimic a split action of https://www.openstreetmap.org/way/295132739
const relation: OsmRelation = <OsmRelation>await OsmObject.DownloadObjectAsync("relation/9572808")
const originalNodeIds = [5273988967,

View file

@ -2,13 +2,73 @@ import {describe} from 'mocha'
import {expect} from 'chai'
import {OsmObject} from "../../../Logic/Osm/OsmObject";
import {Utils} from "../../../Utils";
import ScriptUtils from "../../../scripts/ScriptUtils";
import {readFileSync} from "fs";
describe("OsmObject", () => {
describe("download referencing ways", () => {
Utils.injectJsonDownloadForTests(
"https://www.openstreetmap.org/api/0.6/node/1124134958/ways", {"version":"0.6","generator":"CGImap 0.8.6 (49805 spike-06.openstreetmap.org)","copyright":"OpenStreetMap and contributors","attribution":"http://www.openstreetmap.org/copyright","license":"http://opendatacommons.org/licenses/odbl/1-0/","elements":[{"type":"way","id":97038428,"timestamp":"2019-06-19T12:26:24Z","version":6,"changeset":71399984,"user":"Pieter Vander Vennet","uid":3818858,"nodes":[1124134958,323729212,323729351,2542460408,187073405],"tags":{"highway":"residential","name":"Brugs-Kerkhofstraat","sett:pattern":"arc","surface":"sett"}},{"type":"way","id":97038434,"timestamp":"2019-06-19T12:26:24Z","version":5,"changeset":71399984,"user":"Pieter Vander Vennet","uid":3818858,"nodes":[1124134958,1124135024,187058607],"tags":{"bicycle":"use_sidepath","highway":"residential","name":"Kerkhofblommenstraat","sett:pattern":"arc","surface":"sett"}},{"type":"way","id":97038435,"timestamp":"2017-12-21T21:41:08Z","version":4,"changeset":54826837,"user":"Jakka","uid":2403313,"nodes":[1124134958,2576628889,1124135035,5298371485,5298371495],"tags":{"bicycle":"use_sidepath","highway":"residential","name":"Kerkhofblommenstraat"}},{"type":"way","id":251446313,"timestamp":"2019-01-07T19:22:47Z","version":4,"changeset":66106872,"user":"M!dgard","uid":763799,"nodes":[1124134958,5243143198,4555715455],"tags":{"foot":"yes","highway":"service"}}]})
"https://www.openstreetmap.org/api/0.6/node/1124134958/ways", {
"version": "0.6",
"generator": "CGImap 0.8.6 (49805 spike-06.openstreetmap.org)",
"copyright": "OpenStreetMap and contributors",
"attribution": "http://www.openstreetmap.org/copyright",
"license": "http://opendatacommons.org/licenses/odbl/1-0/",
"elements": [{
"type": "way",
"id": 97038428,
"timestamp": "2019-06-19T12:26:24Z",
"version": 6,
"changeset": 71399984,
"user": "Pieter Vander Vennet",
"uid": 3818858,
"nodes": [1124134958, 323729212, 323729351, 2542460408, 187073405],
"tags": {
"highway": "residential",
"name": "Brugs-Kerkhofstraat",
"sett:pattern": "arc",
"surface": "sett"
}
}, {
"type": "way",
"id": 97038434,
"timestamp": "2019-06-19T12:26:24Z",
"version": 5,
"changeset": 71399984,
"user": "Pieter Vander Vennet",
"uid": 3818858,
"nodes": [1124134958, 1124135024, 187058607],
"tags": {
"bicycle": "use_sidepath",
"highway": "residential",
"name": "Kerkhofblommenstraat",
"sett:pattern": "arc",
"surface": "sett"
}
}, {
"type": "way",
"id": 97038435,
"timestamp": "2017-12-21T21:41:08Z",
"version": 4,
"changeset": 54826837,
"user": "Jakka",
"uid": 2403313,
"nodes": [1124134958, 2576628889, 1124135035, 5298371485, 5298371495],
"tags": {"bicycle": "use_sidepath", "highway": "residential", "name": "Kerkhofblommenstraat"}
}, {
"type": "way",
"id": 251446313,
"timestamp": "2019-01-07T19:22:47Z",
"version": 4,
"changeset": 66106872,
"user": "M!dgard",
"uid": 763799,
"nodes": [1124134958, 5243143198, 4555715455],
"tags": {"foot": "yes", "highway": "service"}
}]
})
it("should download referencing ways",
@ -19,5 +79,15 @@ describe("OsmObject", () => {
expect(ways).not.undefined
expect(ways).length(4)
})
it("should download full OSM-relations", async () => {
ScriptUtils.fixUtils()
Utils.injectJsonDownloadForTests("https://www.openstreetmap.org/api/0.6/relation/5759328/full", JSON.parse(readFileSync("./test/data/relation_5759328.json","UTF-8")))
const r = await OsmObject.DownloadObjectAsync("relation/5759328").then(x => x)
const geojson = r.asGeoJson();
expect(geojson.geometry.type).eq("MultiPolygon")
})
})
})