From 41aa6e871fa0138b7126f24a88c7cbd55623954a Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Sun, 11 Jun 2023 11:42:41 +0200 Subject: [PATCH 1/9] Docs: write down test protocol --- Docs/UserTests/Test_protocol.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 Docs/UserTests/Test_protocol.md diff --git a/Docs/UserTests/Test_protocol.md b/Docs/UserTests/Test_protocol.md new file mode 100644 index 000000000..ae4c5c8cb --- /dev/null +++ b/Docs/UserTests/Test_protocol.md @@ -0,0 +1,33 @@ +# Test Protocol + +This is the default test protocol for performing basic tasks with MapComplete. + +## Exploring the interface + +Open up the homepage. + +1. What does the participant think the website does? +2. Can you find a map with a topic that is interesting to you? + +## Exploring the map + +1. Can the participant find a POI nearby the current location? +2. Can the participant open up basic information? +3. Can the participant filter for data? +4. Can the participant change the background layer? +5. What are the legal requirements of using the map? + +## Making changes + +1. Can the participant login or create a new account? +2. Can the participant make a change to an existing POI? +3. Can the participant add a picture? +4. Can the participant add a new POI? + +## Advanced features + +1. Can the participant change the language of the interface to their preferred language? +2. Can the participant export the data in view for analysis? +3. Can the participant create a PDF-map for print? +4. Can they change the license of their picture? Can they explain what the CC0 and their chosen licenses imply? +5. Can they find the chatroom? From 7dd3b4be4dc1340685ff522c0c5ba6765a227eb6 Mon Sep 17 00:00:00 2001 From: bxl-forever Date: Sun, 11 Jun 2023 14:43:15 +0200 Subject: [PATCH 2/9] Updated rainbow crossing theme to use surface:colour on footways instead of crossing:marking (deprecated) --- Docs/Layers/rainbow_crossing_high_zoom.md | 10 ++++----- Docs/Layers/rainbow_crossings.md | 6 +++--- .../mapcomplete_rainbow_crossings.json | 12 +++++------ .../rainbow_crossings/rainbow_crossings.json | 21 ++++++++++++------- .../rainbow_crossings/rainbow_crossings.json | 4 ++-- 5 files changed, 29 insertions(+), 24 deletions(-) diff --git a/Docs/Layers/rainbow_crossing_high_zoom.md b/Docs/Layers/rainbow_crossing_high_zoom.md index dcd1e19bf..42b883275 100644 --- a/Docs/Layers/rainbow_crossing_high_zoom.md +++ b/Docs/Layers/rainbow_crossing_high_zoom.md @@ -41,10 +41,10 @@ Elements must have the all of following tags to be shown on this layer: - highway=crossing - - crossing:marking=rainbow + - surface:colour=rainbow -[Execute on overpass](http://overpass-turbo.eu/?Q=%5Bout%3Ajson%5D%5Btimeout%3A90%5D%3B(%20%20%20%20nwr%5B%22crossing%3Amarking%22%3D%22rainbow%22%5D%5B%22highway%22%3D%22crossing%22%5D(%7B%7Bbbox%7D%7D)%3B%0A)%3Bout%20body%3B%3E%3Bout%20skel%20qt%3B) +[Execute on overpass](http://overpass-turbo.eu/?Q=%5Bout%3Ajson%5D%5Btimeout%3A90%5D%3B(%20%20%20%20nwr%5B%22surface%3Acolour%22%3D%22rainbow%22%5D%5B%22highway%22%3D%22crossing%22%5D(%7B%7Bbbox%7D%7D)%3B%0A)%3Bout%20body%3B%3E%3Bout%20skel%20qt%3B) @@ -77,9 +77,9 @@ The question is *Does this crossing has rainbow paintings?* - - *This crossing has rainbow paintings* corresponds with `crossing:marking=rainbow` - - *No rainbow paintings here* corresponds with `not:crossing:marking=rainbow` - - *No rainbow paintings here* corresponds with `crossing:marking!=rainbow` + - *This crossing has rainbow paintings* corresponds with `surface:colour=rainbow` + - *No rainbow paintings here* corresponds with `not:surface:colour=rainbow` + - *No rainbow paintings here* corresponds with `surface:colour!=rainbow` - This option cannot be chosen as answer diff --git a/Docs/Layers/rainbow_crossings.md b/Docs/Layers/rainbow_crossings.md index 37cac2f73..6e2babfd0 100644 --- a/Docs/Layers/rainbow_crossings.md +++ b/Docs/Layers/rainbow_crossings.md @@ -77,9 +77,9 @@ The question is *Does this crossing has rainbow paintings?* - - *This crossing has rainbow paintings* corresponds with `crossing:marking=rainbow` - - *No rainbow paintings here* corresponds with `not:crossing:marking=rainbow` - - *No rainbow paintings here* corresponds with `crossing:marking!=rainbow` + - *This crossing has rainbow paintings* corresponds with `surface:colour=rainbow` + - *No rainbow paintings here* corresponds with `not:surface:colour=rainbow` + - *No rainbow paintings here* corresponds with `surface:colour!=rainbow` - This option cannot be chosen as answer diff --git a/Docs/TagInfo/mapcomplete_rainbow_crossings.json b/Docs/TagInfo/mapcomplete_rainbow_crossings.json index 3068b29ed..c506f32c5 100644 --- a/Docs/TagInfo/mapcomplete_rainbow_crossings.json +++ b/Docs/TagInfo/mapcomplete_rainbow_crossings.json @@ -11,7 +11,7 @@ }, "tags": [ { - "key": "highway", + "key": "footway", "description": "The MapComplete theme Rainbow pedestrian crossings has a layer Crossings with rainbow paintings showing features with this tag", "value": "crossing" }, @@ -32,14 +32,14 @@ "description": "The layer 'Crossings with rainbow paintings allows to upload images and adds them under the 'image'-tag (and image:0, image:1, ... for multiple images). Furhtermore, this layer shows images based on the keys image, wikidata, wikipedia, wikimedia_commons and mapillary" }, { - "key": "crossing:marking", - "description": "Layer 'Crossings with rainbow paintings' shows crossing:marking=rainbow with a fixed text, namely 'This crossing has rainbow paintings' and allows to pick this as a default answer (in the MapComplete.osm.be theme 'Rainbow pedestrian crossings') (This is only shown if highway=crossing)", + "key": "surface:colour", + "description": "Layer 'Crossings with rainbow paintings' shows surface:colour=rainbow with a fixed text, namely 'This crossing has rainbow paintings' and allows to pick this as a default answer (in the MapComplete.osm.be theme 'Rainbow pedestrian crossings') (This is only shown if footway=crossing)", "value": "rainbow" }, { - "key": "not:crossing:marking", - "description": "Layer 'Crossings with rainbow paintings' shows not:crossing:marking=rainbow with a fixed text, namely 'No rainbow paintings here' and allows to pick this as a default answer (in the MapComplete.osm.be theme 'Rainbow pedestrian crossings') (This is only shown if highway=crossing)", + "key": "not:surface:colour", + "description": "Layer 'Crossings with rainbow paintings' shows not:surface:colour=rainbow with a fixed text, namely 'No rainbow paintings here' and allows to pick this as a default answer (in the MapComplete.osm.be theme 'Rainbow pedestrian crossings') (This is only shown if footway=crossing)", "value": "rainbow" } ] -} \ No newline at end of file +} diff --git a/assets/layers/rainbow_crossings/rainbow_crossings.json b/assets/layers/rainbow_crossings/rainbow_crossings.json index 27dc9c0e6..3586a0d42 100644 --- a/assets/layers/rainbow_crossings/rainbow_crossings.json +++ b/assets/layers/rainbow_crossings/rainbow_crossings.json @@ -13,7 +13,12 @@ "ca": "Una capa que mostra els passos de vianants pintats amb de l'arc de Sant Martí" }, "source": { - "osmTags": "highway=crossing" + "osmTags": { + "and": [ + "highway=footway", + "footway=crossing" + ] + } }, "minzoom": 17, "title": { @@ -34,7 +39,7 @@ "fr": "un passage" }, "tags": [ - "highway=crossing" + "footway=crossing" ], "description": { "en": "Pedestrian crossing", @@ -62,10 +67,10 @@ "fr": "Ce passage a-t-il une peinture arc-en-ciel ?", "ca": "Aquest pas de vianants està pintat amb l'arc de Sant Martí?" }, - "condition": "highway=crossing", + "condition": "footway=crossing", "mappings": [ { - "if": "crossing:marking=rainbow", + "if": "surface:colour=rainbow", "then": { "en": "This crossing has rainbow paintings", "de": "Der Überweg hat eine Markierung in Regenbogenfarben", @@ -78,7 +83,7 @@ } }, { - "if": "not:crossing:marking=rainbow", + "if": "not:surface:colour=rainbow", "then": { "en": "No rainbow paintings here", "de": "Hier gibt es kein Markierung in Regenbogenfarben", @@ -88,7 +93,7 @@ "icon": "./assets/themes/rainbow_crossings/crossing.svg" }, { - "if": "crossing:marking!=rainbow", + "if": "surface:colour!=rainbow", "then": { "en": "No rainbow paintings here", "de": "Hier gibt es kein Markierung in Regenbogenfarben", @@ -107,7 +112,7 @@ "render": "./assets/themes/rainbow_crossings/crossing.svg", "mappings": [ { - "if": "crossing:marking=rainbow", + "if": "surface:colour=rainbow", "then": "./assets/themes/rainbow_crossings/logo.svg" } ] @@ -119,4 +124,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/assets/themes/rainbow_crossings/rainbow_crossings.json b/assets/themes/rainbow_crossings/rainbow_crossings.json index 09fc00113..85aedd23b 100644 --- a/assets/themes/rainbow_crossings/rainbow_crossings.json +++ b/assets/themes/rainbow_crossings/rainbow_crossings.json @@ -38,7 +38,7 @@ "source": { "osmTags": { "and+": [ - "crossing:marking=rainbow" + "surface:colour=rainbow" ] } } @@ -53,4 +53,4 @@ } } ] -} \ No newline at end of file +} From 6f7437aa46885102f6c5960d94dce0ed0f874d00 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Sun, 11 Jun 2023 16:57:52 +0200 Subject: [PATCH 3/9] Chore: move one-off import script to directory --- scripts/{ => importscripts}/extractBikeRental.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) rename scripts/{ => importscripts}/extractBikeRental.ts (97%) diff --git a/scripts/extractBikeRental.ts b/scripts/importscripts/extractBikeRental.ts similarity index 97% rename from scripts/extractBikeRental.ts rename to scripts/importscripts/extractBikeRental.ts index f9d9aeeac..5a6d5098a 100644 --- a/scripts/extractBikeRental.ts +++ b/scripts/importscripts/extractBikeRental.ts @@ -1,5 +1,9 @@ import * as fs from "fs" -import { OH } from "../UI/OpeningHours/OpeningHours" +import { OH } from "../../UI/OpeningHours/OpeningHours" + +/* + * One-shot script to import data about bicycle rental from West-Flanders + */ function extractValue(vs: { __value }[]) { if (vs === undefined) { From 8eda65a24f129a406c23b7a248bc7b3d3c716b09 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Sun, 5 Mar 2023 23:42:49 +0100 Subject: [PATCH 4/9] Conflation script --- scripts/conflate.ts | 211 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 scripts/conflate.ts diff --git a/scripts/conflate.ts b/scripts/conflate.ts new file mode 100644 index 000000000..17d84292f --- /dev/null +++ b/scripts/conflate.ts @@ -0,0 +1,211 @@ +import Script from "./Script" +import fs from "fs" +import { Feature } from "geojson" +import { GeoOperations } from "../Logic/GeoOperations" +import { Utils } from "../Utils" +import { OsmObject } from "../Logic/Osm/OsmObject" + +export class Conflate extends Script { + constructor() { + super( + [ + "Conflation script", + "", + "This script is meant to be used to prepare imports. It takes one 'OSM'-dataset and one external dataset and tries to find an OSM-id for every external item.", + "", + "Arguments:", + "osm_file.geojson external_file.geojson [search_range]", + "- osm_file.geojson: a file exported from overpass, including meta (note: filename MUST contain either OSM or OpenStreetMap)", + "- external_file.geojson: the data to import. Tags should be prepared to have an OSM-name", + "- search_range: max distance at which a match will occur", + ].join("\n") + ) + } + + private async findTimeFork( + externalName: string, + osmName: string, + osmId: string + ): Promise<{ earliestDateOfImport; latestDateOfImport }> { + const history = await OsmObject.DownloadHistory(osmId).AsPromise((h) => h.length > 0) + let earliest: Date = undefined + let latest: Date = undefined + for (const historyElement of history) { + const csTime = new Date(historyElement.tags["_last_edit:timestamp"]) + if (isNaN(csTime.getTime())) { + console.error("Could not parse" + historyElement.tags["_last_edit:timestamp"]) + return undefined + } + const nameIdentical = historyElement.tags.name === externalName + if (nameIdentical) { + if (earliest == undefined) { + earliest = csTime + } + latest = csTime + } + } + + if (history.at(-1).tags.name === externalName) { + // Not changed yet, so no actual hint about when this import could have happened + latest = new Date() + } + + if (this.earliestDate === undefined || earliest?.getTime() > this.earliestDate?.getTime()) { + this.earliestDate = earliest + } + if (this.latestDate === undefined || latest?.getTime() < this.latestDate?.getTime()) { + this.latestDate = latest + } + + return { earliestDateOfImport: earliest, latestDateOfImport: latest } + } + + private earliestDate: Date = undefined + private latestDate: Date = undefined + + async main(args: string[]): Promise { + const [osm_file_path, external_file_path] = args + let max_range = 50 + if (args.length === 3) { + max_range = Number(args[2]) + } + if ( + osm_file_path.toLowerCase().indexOf("osm") < 0 && + osm_file_path.toLowerCase().indexOf("openstreetmap") < 0 + ) { + throw "OSM File path must contain 'osm' or 'openStreetMap'" + } + + if ( + external_file_path.toLowerCase().indexOf("osm") >= 0 || + external_file_path.toLowerCase().indexOf("openstreetmap") >= 0 + ) { + throw "External File path may not contain 'osm' or 'openStreetMap'" + } + + const external_features: Feature[] = JSON.parse( + fs.readFileSync(external_file_path, { encoding: "utf-8" }) + ).features + const osm_features: Feature[] = JSON.parse( + fs.readFileSync(osm_file_path, { encoding: "utf-8" }) + ).features + + const match_lengths: (string | number)[][] = [ + [ + "osm_id", + "external_index", + "match_distance", + "name_levenshtein_distance", + "osm_data", + "external_data", + "status", + ], + ] + for (let i = 0; i < external_features.length; i++) { + // console.log("Inspecting " + (i + 1) + "/" + external_features.length) + const externalFeature = external_features[i] + const possibleMatches: number[] = [] + for (const osmFeature of osm_features) { + const d = GeoOperations.distanceBetween( + GeoOperations.centerpointCoordinates(externalFeature), + GeoOperations.centerpointCoordinates(osmFeature) + ) + + if (d === 0) { + console.log( + "Found an exact match (name match: ", + osmFeature.properties.name === externalFeature.properties.name, + osmFeature.properties.name, + externalFeature.properties.name + ) + continue + } + continue + if (d < max_range) { + console.log("Found a match") + match_lengths.push([ + osmFeature.properties["@id"], + (i + " " + possibleMatches.join(",")).trim(), + d, + this.levenshteinDistancePharmacy( + externalFeature.properties.name, + osmFeature.properties.name + ), + externalFeature.properties.status, + ...this.conflate(osmFeature.properties, externalFeature.properties), + ]) + possibleMatches.push(osmFeature.properties["@id"]) + /* + possibleMatches.push({ + osmFeature, + d, + nameDist: Utils.levenshteinDistance( + osmFeature.properties.name, + externalFeature.properties.name + ), + })//*/ + } + // possibleMatches.sort((a, b) => b.d - a.d) + } + } + match_lengths.sort((a, b) => b[1] - a[1]) + console.log( + "The import probably happened between ", + this.earliestDate?.toISOString(), + "and", + this.latestDate?.toISOString() + ) + fs.writeFileSync( + "../onwheels-data-prep/match_lengths.tsv", + match_lengths.map((l) => l.join("\t")).join("\n") + ) + console.log(match_lengths) + } + + private levenshteinDistancePharmacy(a?: string, b?: string) { + a ??= "" + b ??= "" + a = a.toLowerCase() + b = b.toLowerCase() + return Math.min( + ...["", "pharmacie", "apotheek", "pharmacie de", "apotheke"].map((prefix) => + Math.min( + Utils.levenshteinDistance(a, prefix + b), + Utils.levenshteinDistance(prefix + a, b) + ) + ) + ) + } + + private conflate( + osmFeature: Record, + externalFeature: Record + ): string[] { + const r: string[] = [] + + for (const externalFeatureKey in externalFeature) { + if ( + [ + "status", + "healthcare", + "unmeasurable_reason", + "timestamp_created", + "timestamp_last_modified", + ].indexOf(externalFeatureKey) >= 0 + ) { + continue + } + const v = externalFeature[externalFeatureKey] + const osmV = osmFeature[externalFeatureKey] + if (osmV === undefined) { + r.push("+" + externalFeatureKey + "=" + v) + } else if (osmV !== v) { + r.push("~" + externalFeatureKey + "=" + v + " (osm: " + osmV + ")") + } + } + + return r.map((l) => l.replace(/\n/g, "\\n")) + } +} + +new Conflate().run() From 99cb879cfe09bcb5e5fb5f9dcae96f0a908f1f2d Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Tue, 21 Mar 2023 20:01:11 +0100 Subject: [PATCH 5/9] Some more experimentation with the conflation script --- package.json | 3 +- scripts/ScriptUtils.ts | 27 ++- scripts/conflate.ts | 393 +++++++++++++++++++++++++++++++---------- 3 files changed, 321 insertions(+), 102 deletions(-) diff --git a/package.json b/package.json index 072f4113b..7b37663fd 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,8 @@ "weblate-merge": "git remote update weblate-github; git merge weblate-github/weblate-mapcomplete-core weblate-github/weblate-mapcomplete-layers weblate-github/weblate-mapcomplete-layer-translations", "weblate-fix-heavy": "git fetch weblate-hosted-layers; git fetch weblate-hosted-core; git merge weblate-hosted-layers/master weblate-hosted-core/master ", "housekeeping": "git pull && npm run weblate-fix-heavy && npm run generate && npm run generate:docs && npm run generate:contributor-list && vite-node scripts/fetchLanguages.ts && npm run format && git add assets/ langs/ Docs/ **/*.ts Docs/* && git commit -m 'chore: automated housekeeping...'", - "parseSchools": "vite-node scripts/schools/amendSchoolData.ts" + "parseSchools": "vite-node scripts/schools/amendSchoolData.ts", + "conflate": "vite-node scripts/conflate.ts -- ../onwheels-data-prep/osm_pharmacies.geojson ../onwheels-data-prep/OnWheelsData_apotheek.geojson" }, "keywords": [ "OpenStreetMap", diff --git a/scripts/ScriptUtils.ts b/scripts/ScriptUtils.ts index af36d962f..86a72a505 100644 --- a/scripts/ScriptUtils.ts +++ b/scripts/ScriptUtils.ts @@ -146,17 +146,20 @@ export default class ScriptUtils { private static async DownloadJSON(url: string, headers?: any): Promise { const data = await ScriptUtils.Download(url, headers) - return JSON.parse(data.content) + return JSON.parse(data["content"]) } - private static Download(url: string, headers?: any): Promise<{ content: string }> { + public static Download( + url: string, + headers?: any + ): Promise<{ content: string } | { redirect: string }> { return new Promise((resolve, reject) => { try { headers = headers ?? {} headers.accept = "application/json" - console.log(" > ScriptUtils.DownloadJson(", url, ")") + console.log(" > ScriptUtils.Download(", url, ")") const urlObj = new URL(url) - https.get( + const request = https.get( { host: urlObj.host, path: urlObj.pathname + urlObj.search, @@ -173,10 +176,26 @@ export default class ScriptUtils { }) res.addListener("end", function () { + if (res.statusCode === 301 || res.statusCode === 302) { + console.log("Got a redirect:", res.headers.location) + resolve({ redirect: res.headers.location }) + } + if (res.statusCode >= 400) { + console.log( + "Error while fetching ", + url, + "due to", + res.statusMessage + ) + reject(res.statusCode) + } resolve({ content: parts.join("") }) }) } ) + request.on("error", function (e) { + reject(e) + }) } catch (e) { reject(e) } diff --git a/scripts/conflate.ts b/scripts/conflate.ts index 17d84292f..175072227 100644 --- a/scripts/conflate.ts +++ b/scripts/conflate.ts @@ -4,8 +4,31 @@ import { Feature } from "geojson" import { GeoOperations } from "../Logic/GeoOperations" import { Utils } from "../Utils" import { OsmObject } from "../Logic/Osm/OsmObject" +import { PhoneTextField, UrlTextfieldDef } from "../UI/Input/ValidatedTextField" +import { OsmId } from "../Models/OsmFeature" +import ScriptUtils from "./ScriptUtils" + +interface PossibleMatch { + /** + * Distance in meter between the OSM-data and the external dataset + */ + d: number + + osm_feature: Feature + external_feature: Feature +} + +interface ReplayResult { + certainly_imported?: boolean + possibly_imported?: boolean + resting_properties?: Record +} export class Conflate extends Script { + private earliestDate: Date = undefined + private latestDate: Date = undefined + private readonly historyCacheDir = "/tmp/cache/" + constructor() { super( [ @@ -22,10 +45,88 @@ export class Conflate extends Script { ) } + async main(args: string[]): Promise { + const [osm_file_path, external_file_path] = args + let max_range = 50 + if (args.length === 3) { + max_range = Number(args[2]) + } + if ( + osm_file_path.toLowerCase().indexOf("osm") < 0 && + osm_file_path.toLowerCase().indexOf("openstreetmap") < 0 + ) { + throw "OSM File path must contain 'osm' or 'openStreetMap'" + } + + if ( + external_file_path.toLowerCase().indexOf("osm") >= 0 || + external_file_path.toLowerCase().indexOf("openstreetmap") >= 0 + ) { + throw "External File path may not contain 'osm' or 'openStreetMap'" + } + + const external_features: Feature[] = JSON.parse( + fs.readFileSync(external_file_path, { encoding: "utf-8" }) + ).features + const osm_features: Feature[] = JSON.parse( + fs.readFileSync(osm_file_path, { encoding: "utf-8" }) + ).features + + const bestMatches = await this.calculateMatches(external_features, osm_features, max_range) + const unmatched = external_features.filter( + (f) => !bestMatches.some((matched) => matched.match.external_feature === f) + ) + const match_lengths: (string | number)[][] = [ + [ + "osm_id", + "match_distance", + "osm_name", + "imported", + "status_external", + "...properties_differences", + ], + ] + for (const { match, replayed } of bestMatches) { + const { external_feature, d, osm_feature } = match + const { possibly_imported, certainly_imported, resting_properties } = replayed + const status = resting_properties["status"] + delete resting_properties["status"] + if (Object.keys(resting_properties).length === 0) { + continue + } + match_lengths.push([ + osm_feature.properties["@id"], + d, + osm_feature.properties.name, + certainly_imported ? "import" : possibly_imported ? "prob import" : "new", + status, + JSON.stringify(resting_properties), + ]) + } + + fs.writeFileSync( + "../onwheels-data-prep/matches.tsv", + match_lengths.map((l) => l.join("\t")).join("\n") + ) + + fs.writeFileSync( + "../onwheels-data-prep/unmatched.geojson", + JSON.stringify( + { + type: "FeatureCollection", + features: unmatched, + }, + + null, + " " + ) + ) + } + private async findTimeFork( externalName: string, osmName: string, - osmId: string + osmId: OsmId ): Promise<{ earliestDateOfImport; latestDateOfImport }> { const history = await OsmObject.DownloadHistory(osmId).AsPromise((h) => h.length > 0) let earliest: Date = undefined @@ -60,106 +161,204 @@ export class Conflate extends Script { return { earliestDateOfImport: earliest, latestDateOfImport: latest } } - private earliestDate: Date = undefined - private latestDate: Date = undefined + private findPossibleMatchesFor( + osm_features: Feature[], + externalFeature: Feature, + max_range: number + ): PossibleMatch[] { + const possibleMatches: PossibleMatch[] = [] + for (const osmFeature of osm_features) { + const d = GeoOperations.distanceBetween( + GeoOperations.centerpointCoordinates(externalFeature), + GeoOperations.centerpointCoordinates(osmFeature) + ) - async main(args: string[]): Promise { - const [osm_file_path, external_file_path] = args - let max_range = 50 - if (args.length === 3) { - max_range = Number(args[2]) - } - if ( - osm_file_path.toLowerCase().indexOf("osm") < 0 && - osm_file_path.toLowerCase().indexOf("openstreetmap") < 0 - ) { - throw "OSM File path must contain 'osm' or 'openStreetMap'" - } - - if ( - external_file_path.toLowerCase().indexOf("osm") >= 0 || - external_file_path.toLowerCase().indexOf("openstreetmap") >= 0 - ) { - throw "External File path may not contain 'osm' or 'openStreetMap'" - } - - const external_features: Feature[] = JSON.parse( - fs.readFileSync(external_file_path, { encoding: "utf-8" }) - ).features - const osm_features: Feature[] = JSON.parse( - fs.readFileSync(osm_file_path, { encoding: "utf-8" }) - ).features - - const match_lengths: (string | number)[][] = [ - [ - "osm_id", - "external_index", - "match_distance", - "name_levenshtein_distance", - "osm_data", - "external_data", - "status", - ], - ] - for (let i = 0; i < external_features.length; i++) { - // console.log("Inspecting " + (i + 1) + "/" + external_features.length) - const externalFeature = external_features[i] - const possibleMatches: number[] = [] - for (const osmFeature of osm_features) { - const d = GeoOperations.distanceBetween( - GeoOperations.centerpointCoordinates(externalFeature), - GeoOperations.centerpointCoordinates(osmFeature) - ) - - if (d === 0) { - console.log( - "Found an exact match (name match: ", - osmFeature.properties.name === externalFeature.properties.name, - osmFeature.properties.name, - externalFeature.properties.name - ) - continue - } - continue - if (d < max_range) { - console.log("Found a match") - match_lengths.push([ - osmFeature.properties["@id"], - (i + " " + possibleMatches.join(",")).trim(), - d, - this.levenshteinDistancePharmacy( - externalFeature.properties.name, - osmFeature.properties.name - ), - externalFeature.properties.status, - ...this.conflate(osmFeature.properties, externalFeature.properties), - ]) - possibleMatches.push(osmFeature.properties["@id"]) - /* - possibleMatches.push({ - osmFeature, - d, - nameDist: Utils.levenshteinDistance( - osmFeature.properties.name, - externalFeature.properties.name - ), - })//*/ - } - // possibleMatches.sort((a, b) => b.d - a.d) + if (d < max_range) { + possibleMatches.push({ + external_feature: externalFeature, + osm_feature: osmFeature, + d, + }) } } - match_lengths.sort((a, b) => b[1] - a[1]) - console.log( - "The import probably happened between ", - this.earliestDate?.toISOString(), - "and", - this.latestDate?.toISOString() + return possibleMatches + } + + private async stillOnline(url: string): Promise { + // return true + if (url.indexOf("facebook.com") > 0) { + return true + } + const cachePath = this.historyCacheDir + "/urls/ " + url.replace(/[/\\:]/g, "_") + if (fs.existsSync(cachePath)) { + const online = JSON.parse(fs.readFileSync(cachePath, { encoding: "utf-8" })) + return online + } + let online: boolean | string = false + try { + online = await this.stillOnlineUncached(url) + } catch (e) { + console.log(e) + const urlObj = new URL(url) + if (e === "NOT_FOUND" && urlObj.pathname.length > 0) { + console.log("Maybe trying the homepage will help?") + } + } + fs.writeFileSync(cachePath, JSON.stringify(online, null, " "), { encoding: "utf-8" }) + return online + } + + private async stillOnlineUncached(url: string): Promise { + if (!url.startsWith("http")) { + url = "https://" + url + } + url = url.replace("http://", "https://") + try { + const result = await ScriptUtils.Download(url) + if (result["redirect"]) { + if (result["redirect"].startsWith("/")) { + return true + } + return result["redirect"] + } + if (result["content"]) { + return true + } + console.error("Got a result, but no content?", url, result) + } catch (e) { + console.log("Offline (error):", url, e.message) + return false + } + } + + private async historyCached(id): Promise { + const cachePath = this.historyCacheDir + "/" + id.replace("/", "_") + if (fs.existsSync(cachePath)) { + return JSON.parse(fs.readFileSync(cachePath, { encoding: "utf-8" })) + } + const history = await OsmObject.DownloadHistory(id).AsPromise((l) => l.length > 0) + fs.writeFileSync(cachePath, JSON.stringify(history, null, " "), { encoding: "utf-8" }) + return history + } + + private async normalize(properties: Record) { + if (properties["phone"]) { + properties["phone"] = new PhoneTextField().reformat(properties["phone"], () => "be") + } + if (properties["website"]) { + let website = properties.website.toLowerCase() + website + .replace("http://http://", "http://") + .replace("https//", "https://") + .replace("http://", "https://") + const validator = new UrlTextfieldDef() + if (validator.isValid(website)) { + properties.website = new UrlTextfieldDef().reformat(website) + const stillOnline = await this.stillOnline(website) + if (stillOnline === false) { + delete properties.website + } + if (typeof stillOnline === "string") { + properties.website = stillOnline + } + } else { + console.log("Invalid url:", website) + } + } + + if (properties["healthcare"] === "pharmacy") { + // we don't care about this tag + delete properties["healthcare"] + } + } + + private async replay(match: PossibleMatch): Promise { + const history = await this.historyCached(match.osm_feature.properties["@id"]) + + let certainly_imported = match.d < 0.0001 + let possibly_imported = false + + const resting_properties = { ...match.external_feature.properties } + await this.normalize(resting_properties) + + for (const historyElement of history) { + await this.normalize(historyElement.tags) + + if (historyElement.tags.name === resting_properties.name) { + possibly_imported = true + } + + for (const key in resting_properties) { + if (this.str_compare(historyElement.tags[key], resting_properties[key])) { + delete resting_properties[key] + } + } + } + + return { + certainly_imported, + possibly_imported, + resting_properties, + } + } + + private str_compare(a, b): boolean { + if (a === undefined || b === undefined) { + return false + } + a = a.toLowerCase().replaceAll(/[éèáàüë].*$/g, "") + b = b.toLowerCase().replaceAll(/[éèáàüë].*$/g, "") + + return a === b + } + + private async calculateMatches( + external_features: Feature[], + osm_features: Feature[], + max_range: number + ): Promise<{ match: PossibleMatch; replayed: ReplayResult }[]> { + const matches: { match: PossibleMatch; replayed: ReplayResult }[] = [] + for (const f of external_features) { + const match = await this.calculateMatch(osm_features, f, max_range) + if (match) { + matches.push(match) + } + } + return matches + } + + private async calculateMatch( + osm_features: Feature[], + externalFeature: Feature, + max_range: number + ): Promise<{ match: PossibleMatch; replayed: ReplayResult }> { + const possibleMatches = this.findPossibleMatchesFor( + osm_features, + externalFeature, + max_range ) - fs.writeFileSync( - "../onwheels-data-prep/match_lengths.tsv", - match_lengths.map((l) => l.join("\t")).join("\n") - ) - console.log(match_lengths) + let bestMatch: PossibleMatch = undefined + let bestMatchReplayed: ReplayResult = undefined + for (const possibleMatch of possibleMatches) { + const replayed = await this.replay(possibleMatch) + if ( + bestMatch === undefined || + (replayed.certainly_imported && !bestMatchReplayed.possibly_imported) || + (!bestMatchReplayed.certainly_imported && + replayed.possibly_imported && + !bestMatchReplayed.possibly_imported) + ) { + bestMatch = possibleMatch + bestMatchReplayed = replayed + } + } + if (bestMatch === undefined) { + return undefined + } + return { + replayed: bestMatchReplayed, + match: bestMatch, + } } private levenshteinDistancePharmacy(a?: string, b?: string) { From dbf47ec1216bd97f41ce687341f70753c4a2ff40 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Tue, 21 Mar 2023 20:02:06 +0100 Subject: [PATCH 6/9] Expose some validatedTextFields --- UI/Input/ValidatedTextField.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/UI/Input/ValidatedTextField.ts b/UI/Input/ValidatedTextField.ts index f24d8548b..468e2b27a 100644 --- a/UI/Input/ValidatedTextField.ts +++ b/UI/Input/ValidatedTextField.ts @@ -481,7 +481,7 @@ class OpeningHoursTextField extends TextFieldDef { } } -class UrlTextfieldDef extends TextFieldDef { +export class UrlTextfieldDef extends TextFieldDef { declare inputmode: "url" constructor() { @@ -852,7 +852,7 @@ class EmailTextField extends TextFieldDef { } } -class PhoneTextField extends TextFieldDef { +export class PhoneTextField extends TextFieldDef { inputmode = "tel" constructor() { From d4cef78325092155be72700d0160a299422580e3 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Sun, 11 Jun 2023 19:04:40 +0200 Subject: [PATCH 7/9] Finish conflate script --- Logic/Osm/OsmObject.ts | 29 +++++++------ scripts/Script.ts | 4 ++ scripts/conflate.ts | 95 +++++++++++++++++++++++++++++++----------- 3 files changed, 88 insertions(+), 40 deletions(-) diff --git a/Logic/Osm/OsmObject.ts b/Logic/Osm/OsmObject.ts index b9bff14ec..0589e647c 100644 --- a/Logic/Osm/OsmObject.ts +++ b/Logic/Osm/OsmObject.ts @@ -1,10 +1,10 @@ -import { Utils } from "../../Utils" +import {Utils} from "../../Utils" import polygon_features from "../../assets/polygon-features.json" -import { Store, UIEventSource } from "../UIEventSource" -import { BBox } from "../BBox" +import {Store, UIEventSource} from "../UIEventSource" +import {BBox} from "../BBox" import OsmToGeoJson from "osmtogeojson" -import { NodeId, OsmFeature, OsmId, OsmTags, RelationId, WayId } from "../../Models/OsmFeature" -import { Feature, LineString, Polygon } from "geojson" +import {NodeId, OsmFeature, OsmId, OsmTags, RelationId, WayId} from "../../Models/OsmFeature" +import {Feature, LineString, Polygon} from "geojson" export abstract class OsmObject { private static defaultBackend = "https://www.openstreetmap.org/" @@ -346,8 +346,7 @@ export abstract class OsmObject { } return tags } - - abstract ChangesetXML(changesetId: string): string + abstract ChangesetXML(changesetId: string, header?: string): string protected VersionXML() { if (this.version === undefined) { @@ -382,15 +381,15 @@ export class OsmNode extends OsmObject { super("node", id) } - ChangesetXML(changesetId: string): string { + ChangesetXML(changesetId: string, header?: string): string { let tags = this.TagsXML() return ( ' \n" + @@ -542,7 +541,7 @@ export class OsmRelation extends OsmObject { return [0, 0] // TODO } - ChangesetXML(changesetId: string): string { + ChangesetXML(changesetId: string, header?: string): string { let members = "" for (const member of this.members) { members += @@ -560,7 +559,7 @@ export class OsmRelation extends OsmObject { if (changesetId !== undefined) { cs = `changeset="${changesetId}"` } - return ` + return ` ${members}${tags} ` } diff --git a/scripts/Script.ts b/scripts/Script.ts index f76785161..2d47688c5 100644 --- a/scripts/Script.ts +++ b/scripts/Script.ts @@ -15,4 +15,8 @@ export default abstract class Script { args.splice(0, 2) this.main(args).then((_) => console.log("All done")) } + + public printHelp(){ + console.log(this._docs) + } } diff --git a/scripts/conflate.ts b/scripts/conflate.ts index 175072227..bdefebce5 100644 --- a/scripts/conflate.ts +++ b/scripts/conflate.ts @@ -1,11 +1,11 @@ import Script from "./Script" import fs from "fs" -import { Feature } from "geojson" -import { GeoOperations } from "../Logic/GeoOperations" -import { Utils } from "../Utils" -import { OsmObject } from "../Logic/Osm/OsmObject" -import { PhoneTextField, UrlTextfieldDef } from "../UI/Input/ValidatedTextField" -import { OsmId } from "../Models/OsmFeature" +import {Feature} from "geojson" +import {GeoOperations} from "../Logic/GeoOperations" +import {Utils} from "../Utils" +import {OsmObject} from "../Logic/Osm/OsmObject" +import {PhoneTextField, UrlTextfieldDef} from "../UI/Input/ValidatedTextField" +import {OsmId} from "../Models/OsmFeature" import ScriptUtils from "./ScriptUtils" interface PossibleMatch { @@ -45,9 +45,25 @@ export class Conflate extends Script { ) } + private static toXml(changedObjects: OsmObject[]): string { + + return [ + "", + "", + ...changedObjects.map(obj => + obj.ChangesetXML(undefined, ' action="modify" ') + ), + "" + ].join("\n"); + } + async main(args: string[]): Promise { + if (args.length < 2) { + super.printHelp() + return + } const [osm_file_path, external_file_path] = args - let max_range = 50 + let max_range = 25 if (args.length === 3) { max_range = Number(args[2]) } @@ -66,10 +82,10 @@ export class Conflate extends Script { } const external_features: Feature[] = JSON.parse( - fs.readFileSync(external_file_path, { encoding: "utf-8" }) + fs.readFileSync(external_file_path, {encoding: "utf-8"}) ).features const osm_features: Feature[] = JSON.parse( - fs.readFileSync(osm_file_path, { encoding: "utf-8" }) + fs.readFileSync(osm_file_path, {encoding: "utf-8"}) ).features const bestMatches = await this.calculateMatches(external_features, osm_features, max_range) @@ -86,31 +102,48 @@ export class Conflate extends Script { "...properties_differences", ], ] - for (const { match, replayed } of bestMatches) { - const { external_feature, d, osm_feature } = match - const { possibly_imported, certainly_imported, resting_properties } = replayed + + const changedObjects: OsmObject[] = [] + for (const {match, replayed} of bestMatches) { + const {external_feature, d, osm_feature} = match + const {possibly_imported, certainly_imported, resting_properties} = replayed const status = resting_properties["status"] delete resting_properties["status"] if (Object.keys(resting_properties).length === 0) { continue } + const id = osm_feature.properties["@id"] match_lengths.push([ - osm_feature.properties["@id"], + id, d, osm_feature.properties.name, certainly_imported ? "import" : possibly_imported ? "prob import" : "new", status, JSON.stringify(resting_properties), ]) + + const osmObj = await OsmObject.DownloadObjectAsync(id) + for (const key in resting_properties) { + osmObj.tags[key] = resting_properties[key] + } + changedObjects.push(osmObj) } + const targetDir = "../onwheels-data-prep/output" + console.log("Writing results to directory", targetDir) + fs.writeFileSync( - "../onwheels-data-prep/matches.tsv", + targetDir + "/matches.tsv", match_lengths.map((l) => l.join("\t")).join("\n") ) - fs.writeFileSync( - "../onwheels-data-prep/unmatched.geojson", + fs.writeFileSync(targetDir + "/changeset.xml", + Conflate.toXml(changedObjects) + ) + + + fs.writeFileSync(targetDir + + "/unmatched.geojson", JSON.stringify( { type: "FeatureCollection", @@ -158,7 +191,7 @@ export class Conflate extends Script { this.latestDate = latest } - return { earliestDateOfImport: earliest, latestDateOfImport: latest } + return {earliestDateOfImport: earliest, latestDateOfImport: latest} } private findPossibleMatchesFor( @@ -189,10 +222,12 @@ export class Conflate extends Script { if (url.indexOf("facebook.com") > 0) { return true } + if (!fs.existsSync(this.historyCacheDir + "urls/")) { + fs.mkdirSync(this.historyCacheDir + "urls/") + } const cachePath = this.historyCacheDir + "/urls/ " + url.replace(/[/\\:]/g, "_") if (fs.existsSync(cachePath)) { - const online = JSON.parse(fs.readFileSync(cachePath, { encoding: "utf-8" })) - return online + return JSON.parse(fs.readFileSync(cachePath, {encoding: "utf-8"})) } let online: boolean | string = false try { @@ -204,7 +239,7 @@ export class Conflate extends Script { console.log("Maybe trying the homepage will help?") } } - fs.writeFileSync(cachePath, JSON.stringify(online, null, " "), { encoding: "utf-8" }) + fs.writeFileSync(cachePath, JSON.stringify(online, null, " "), {encoding: "utf-8"}) return online } @@ -214,7 +249,9 @@ export class Conflate extends Script { } url = url.replace("http://", "https://") try { - const result = await ScriptUtils.Download(url) + const result = await ScriptUtils.Download(url, { + "User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0" + }) if (result["redirect"]) { if (result["redirect"].startsWith("/")) { return true @@ -225,6 +262,7 @@ export class Conflate extends Script { return true } console.error("Got a result, but no content?", url, result) + return false } catch (e) { console.log("Offline (error):", url, e.message) return false @@ -232,12 +270,15 @@ export class Conflate extends Script { } private async historyCached(id): Promise { - const cachePath = this.historyCacheDir + "/" + id.replace("/", "_") + const cachePath = this.historyCacheDir + id.replace("/", "_") + if (!fs.existsSync(this.historyCacheDir)) { + fs.mkdirSync(this.historyCacheDir) + } if (fs.existsSync(cachePath)) { - return JSON.parse(fs.readFileSync(cachePath, { encoding: "utf-8" })) + return JSON.parse(fs.readFileSync(cachePath, {encoding: "utf-8"})) } const history = await OsmObject.DownloadHistory(id).AsPromise((l) => l.length > 0) - fs.writeFileSync(cachePath, JSON.stringify(history, null, " "), { encoding: "utf-8" }) + fs.writeFileSync(cachePath, JSON.stringify(history, null, " "), {encoding: "utf-8"}) return history } @@ -249,8 +290,12 @@ export class Conflate extends Script { let website = properties.website.toLowerCase() website .replace("http://http://", "http://") + .replace("https://https://", "https://") .replace("https//", "https://") .replace("http://", "https://") + if (website.startsWith("https://")) { + website = "https://" + website + } const validator = new UrlTextfieldDef() if (validator.isValid(website)) { properties.website = new UrlTextfieldDef().reformat(website) @@ -278,7 +323,7 @@ export class Conflate extends Script { let certainly_imported = match.d < 0.0001 let possibly_imported = false - const resting_properties = { ...match.external_feature.properties } + const resting_properties = {...match.external_feature.properties} await this.normalize(resting_properties) for (const historyElement of history) { From c5300a3fd5b46e611a4604ad9c0483f823396dda Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Sun, 11 Jun 2023 19:08:06 +0200 Subject: [PATCH 8/9] Fix: add missing tag --- assets/layers/rainbow_crossings/rainbow_crossings.json | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/layers/rainbow_crossings/rainbow_crossings.json b/assets/layers/rainbow_crossings/rainbow_crossings.json index 3586a0d42..459169781 100644 --- a/assets/layers/rainbow_crossings/rainbow_crossings.json +++ b/assets/layers/rainbow_crossings/rainbow_crossings.json @@ -39,6 +39,7 @@ "fr": "un passage" }, "tags": [ + "highway=footway", "footway=crossing" ], "description": { From 5b291777be5a1aa4826ba894af76dbf4c09b9a7c Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Sun, 11 Jun 2023 22:52:22 +0200 Subject: [PATCH 9/9] Fix: fix rainbow crossings theme --- assets/layers/climbing_gym/climbing_gym.json | 2 +- assets/layers/etymology/etymology.json | 2 +- .../rainbow_crossings/rainbow_crossings.json | 18 +++++++++++++++--- assets/layers/sport_pitch/sport_pitch.json | 2 +- .../rainbow_crossings/rainbow_crossings.json | 2 +- langs/layers/en.json | 14 ++++++++++++++ langs/layers/nl.json | 14 ++++++++++++++ 7 files changed, 47 insertions(+), 7 deletions(-) diff --git a/assets/layers/climbing_gym/climbing_gym.json b/assets/layers/climbing_gym/climbing_gym.json index 7d57b0349..654ce5d02 100644 --- a/assets/layers/climbing_gym/climbing_gym.json +++ b/assets/layers/climbing_gym/climbing_gym.json @@ -275,4 +275,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/assets/layers/etymology/etymology.json b/assets/layers/etymology/etymology.json index ed558c2db..2a74ae97b 100644 --- a/assets/layers/etymology/etymology.json +++ b/assets/layers/etymology/etymology.json @@ -307,4 +307,4 @@ } } ] -} +} \ No newline at end of file diff --git a/assets/layers/rainbow_crossings/rainbow_crossings.json b/assets/layers/rainbow_crossings/rainbow_crossings.json index 459169781..a1ed6a389 100644 --- a/assets/layers/rainbow_crossings/rainbow_crossings.json +++ b/assets/layers/rainbow_crossings/rainbow_crossings.json @@ -14,9 +14,15 @@ }, "source": { "osmTags": { - "and": [ - "highway=footway", - "footway=crossing" + "or": [ + "surface:colour=rainbow", + "highway=crossing", + { + "and": [ + "highway=footway", + "footway=crossing" + ] + } ] } }, @@ -123,6 +129,12 @@ "point", "centroid" ] + }, + { + "color": "red", + "width": "5", + "dashArray": "10 2", + "lineCap": "square" } ] } diff --git a/assets/layers/sport_pitch/sport_pitch.json b/assets/layers/sport_pitch/sport_pitch.json index e7733347f..05d71c0d4 100644 --- a/assets/layers/sport_pitch/sport_pitch.json +++ b/assets/layers/sport_pitch/sport_pitch.json @@ -684,4 +684,4 @@ }, "open_now" ] -} +} \ No newline at end of file diff --git a/assets/themes/rainbow_crossings/rainbow_crossings.json b/assets/themes/rainbow_crossings/rainbow_crossings.json index 85aedd23b..46b4810b6 100644 --- a/assets/themes/rainbow_crossings/rainbow_crossings.json +++ b/assets/themes/rainbow_crossings/rainbow_crossings.json @@ -53,4 +53,4 @@ } } ] -} +} \ No newline at end of file diff --git a/langs/layers/en.json b/langs/layers/en.json index 46d27c10e..363066038 100644 --- a/langs/layers/en.json +++ b/langs/layers/en.json @@ -3023,6 +3023,17 @@ "name": { "question": "What is the name of this climbing gym?" }, + "rope_rental": { + "mappings": { + "0": { + "then": "A climbing rope can be rented here" + }, + "1": { + "then": "A climbing rope can not be rented here" + } + }, + "question": "Can one rent a climbing rope here?" + }, "shoe_rental": { "mappings": { "0": { @@ -7654,6 +7665,9 @@ }, "5": { "then": "Basketball is played here" + }, + "6": { + "then": "This is a skatepark" } }, "question": "Which sport can be played here?", diff --git a/langs/layers/nl.json b/langs/layers/nl.json index 0404bec22..c022f81c3 100644 --- a/langs/layers/nl.json +++ b/langs/layers/nl.json @@ -2913,6 +2913,17 @@ "name": { "question": "Wat is de naam van dit Klimzaal?" }, + "rope_rental": { + "mappings": { + "0": { + "then": "Een klimtouw kan hier gehuurd worden" + }, + "1": { + "then": "Hier kunnen geen klimtouwen gehuurd worden" + } + }, + "question": "Kan een klimtouw hier gehuurd worden?" + }, "shoe_rental": { "mappings": { "0": { @@ -7332,6 +7343,9 @@ }, "5": { "then": "Hier kan men basketbal beoefenen" + }, + "6": { + "then": "Dit is een skatepark" } }, "question": "Welke sporten kan men hier beoefenen?",