Scripts: add script to fix panoramax-hashes; see #2372

scripts/repairPanoramax.ts

@@ -0,0 +1,117 @@
+import Script from "./Script"
+import { Overpass } from "../src/Logic/Osm/Overpass"
+import { Or } from "../src/Logic/Tags/Or"
+import { RegexTag } from "../src/Logic/Tags/RegexTag"
+import { Utils } from "../src/Utils"
+import Constants from "../src/Models/Constants"
+import { ImmutableStore } from "../src/Logic/UIEventSource"
+import { BBox } from "../src/Logic/BBox"
+import ChangeTagAction from "../src/Logic/Osm/Actions/ChangeTagAction"
+import { Tag } from "../src/Logic/Tags/Tag"
+import { Panoramax } from "panoramax-js/dist"
+import { Changes } from "../src/Logic/Osm/Changes"
+import OsmChangeAction from "../src/Logic/Osm/Actions/OsmChangeAction"
+import { writeFileSync } from "fs"
+import { Feature } from "geojson"
+
+class RepairPanoramax extends Script {
+
+    private static readonly europe: Feature  =    {
+        "type": "Feature",
+        "properties": {},
+        "geometry": {
+            "coordinates": [
+                [
+                    [
+                        -20.091159690050006,
+                        25.773375277790038
+                    ],
+                    [
+                        46.12276429398841,
+                        25.773375277790038
+                    ],
+                    [
+                        46.12276429398841,
+                        65.41389761819318
+                    ],
+                    [
+                        -20.091159690050006,
+                        65.41389761819318
+                    ],
+                    [
+                        -20.091159690050006,
+                        25.773375277790038
+                    ]
+                ]
+            ],
+            "type": "Polygon"
+        }
+    }
+
+    constructor() {
+        super("See https://source.mapcomplete.org/MapComplete/MapComplete/issues/2372\n" +
+            "We accidentally added the full image URL instead of the hash due to a bug. This scripts rewrites all")
+    }
+
+    async main(args: string[]): Promise<void> {
+        const keys = ["panoramax", ...Utils.TimesT(10, i => "panoramax:" + i)]
+        const overpass = new Overpass(
+            new Or(
+                keys.map(k => new RegexTag(k, /^https:\/\/panoramax.mapcomplete.org\/.*/))
+            ),
+            [],
+            Constants.defaultOverpassUrls[0],
+            new ImmutableStore(500)
+        )
+
+        const [wrongFeatures] = await overpass.queryGeoJson(BBox.global)
+        const allChanges: OsmChangeAction[] = []
+        for (const f of wrongFeatures.features) {
+            for (const key of keys) {
+                const v = f.properties[key]
+                if (!v) {
+                    continue
+                }
+                const pre = "https://panoramax.mapcomplete.org/api/pictures/"
+                const pre1 = "https://panoramax.mapcomplete.org/permanent/"
+                let correct: string
+                if (v.startsWith(pre)) {
+                    correct = v.substring(pre.length)
+                    correct = correct.substring(0, correct.indexOf("/"))
+                } else if (v.startsWith(pre1)) {
+                    const stripped = v.substring(pre1.length)
+                    correct = stripped.replace(/\//g, "").replace(/\.jpg$/, "")
+                    correct = correct.substring(0, 8) + "-" + correct.substring(8)
+                } else if (Panoramax.isId(v)) {
+                    // This is a valid ID that came on an object also having an _invalid_ id
+                    // We can safely skip this
+                    continue
+                } else {
+                    throw "Unknown url " + v
+                }
+                console.log(key, correct, "    old: ", v)
+                if (!Panoramax.isId(correct)) {
+                    throw "Constructed an invalid id:" + correct
+                }
+                const change = new ChangeTagAction(
+                    f.properties.id,
+                    new Tag(key, correct),
+                    f.properties,
+                    {
+                        theme: "fix",
+                        changeType: "fix"
+                    }
+                )
+                allChanges.push(change)
+            }
+        }
+
+        const xml = await Changes.createChangesetXMLForJosm(allChanges)
+        console.log(xml)
+        const path = "repairPanoramax.osc"
+        writeFileSync(path, xml)
+        console.log("Written XML to", path)
+    }
+}
+
+new RepairPanoramax().run()

src/Logic/Osm/Changes.ts

@@ -95,8 +95,34 @@ export class Changes {
         })
     }
 
+    public static async createChangesetXMLForJosm(actions: OsmChangeAction[], osmConnection?: OsmConnection): Promise<string> {
+        osmConnection ??= new OsmConnection()
+        const changes = new Changes({
+            osmConnection
+        })
+        const descriptions: ChangeDescription[] = []
+        for (const action of actions) {
+            descriptions.push(...await action.Perform(changes))
+        }
+        const downloader = new OsmObjectDownloader(osmConnection.Backend(), undefined)
+        const downloaded: OsmObject[] = []
+        for (const action of actions) {
+            const osmObj = await downloader.DownloadObjectAsync(action.mainObjectId)
+            if (osmObj === "deleted") {
+                continue
+            }
+            downloaded.push(osmObj)
+        }
+        return Changes.buildChangesetXML("", changes.CreateChangesetObjects(descriptions, downloaded))
+    }
+
+    /**
+     * Creates a changeset
+     * @param csId either the ID-number of the changeset, or an empty string (e.g. when uploading with JOSM)
+     * @param allChanges use 'new Changes()
+     */
     static buildChangesetXML(
-        csId: string,
+        csId: number | string,
         allChanges: {
             modifiedObjects: OsmObject[]
             newObjects: OsmObject[]
@@ -111,20 +137,20 @@ export class Changes {
         if (newElements.length > 0) {
             changes +=
                 "\n<create>\n" +
-                newElements.map((e) => e.ChangesetXML(csId)).join("\n") +
+                newElements.map((e) => e.ChangesetXML("" + csId)).join("\n") +
                 "</create>"
         }
         if (changedElements.length > 0) {
             changes +=
                 "\n<modify>\n" +
-                changedElements.map((e) => e.ChangesetXML(csId)).join("\n") +
+                changedElements.map((e) => e.ChangesetXML("" + csId)).join("\n") +
                 "\n</modify>"
         }
 
         if (deletedElements.length > 0) {
             changes +=
                 "\n<delete>\n" +
-                deletedElements.map((e) => e.ChangesetXML(csId)).join("\n") +
+                deletedElements.map((e) => e.ChangesetXML("" + csId)).join("\n") +
                 "\n</delete>"
         }
 
@@ -722,7 +748,7 @@ export class Changes {
                     deletedObjects: OsmObject[]
                 } = this.CreateChangesetObjects(toUpload, objects)
 
-                return Changes.buildChangesetXML("" + csId, changes)
+                return Changes.buildChangesetXML(csId, changes)
             },
             metatags,
             openChangeset

src/Logic/Osm/Overpass.ts

@@ -3,10 +3,10 @@ import { Utils } from "../../Utils"
 import { ImmutableStore, Store } from "../UIEventSource"
 import { BBox } from "../BBox"
 import osmtogeojson from "osmtogeojson"
-import { FeatureCollection } from "@turf/turf"
-import { Geometry } from "geojson"
+import { FeatureCollection, Geometry } from "geojson"
 import { OsmTags } from "../../Models/OsmFeature"
-;("use strict")
+
+("use strict")
 /**
  * Interfaces overpass to get all the latest data
  */
@@ -64,14 +64,14 @@ export class Overpass {
             elements: []
             remark
             osm3s: { timestamp_osm_base: string }
-        }>(this.buildUrl(query))
+        }>(this.buildUrl(query), {})
 
         if (json.elements.length === 0 && json.remark !== undefined) {
             console.warn("Timeout or other runtime error while querying overpass", json.remark)
             throw `Runtime error (timeout or similar)${json.remark}`
         }
         if (json.elements.length === 0) {
-            console.warn("No features for", json)
+            console.warn("No features for", this.buildUrl(query))
         }
 
         const geojson = <FeatureCollection<Geometry, OsmTags>>osmtogeojson(json)