Finish conflate script

Logic/Osm/OsmObject.ts

@@ -346,8 +346,7 @@ export abstract class OsmObject {
         }
         return tags
     }
-
+    abstract ChangesetXML(changesetId: string, header?: string): string
-    abstract ChangesetXML(changesetId: string): string
 
     protected VersionXML() {
         if (this.version === undefined) {
@@ -382,15 +381,15 @@ export class OsmNode extends OsmObject {
         super("node", id)
     }
 
-    ChangesetXML(changesetId: string): string {
+    ChangesetXML(changesetId: string, header?: string): string {
         let tags = this.TagsXML()
 
         return (
             '    <node id="' +
             this.id +
-            '" changeset="' +
-            changesetId +
             '" ' +
+                (header ?? "") +
+            (changesetId ? ('" changeset="' + changesetId) : "" ) +
             this.VersionXML() +
             ' lat="' +
             this.lat +
@@ -438,7 +437,7 @@ export class OsmWay extends OsmObject {
         return [this.lat, this.lon]
     }
 
-    ChangesetXML(changesetId: string): string {
+    ChangesetXML(changesetId: string, header?: string): string {
         let tags = this.TagsXML()
         let nds = ""
         for (const node in this.nodes) {
@@ -448,8 +447,8 @@ export class OsmWay extends OsmObject {
         return (
             '    <way id="' +
             this.id +
-            '" changeset="' +
+            (header ?? "")+
-            changesetId +
+            (changesetId ? ('" changeset="' + changesetId) : "" ) +
             '" ' +
             this.VersionXML() +
             ">\n" +
@@ -542,7 +541,7 @@ export class OsmRelation extends OsmObject {
         return [0, 0] // TODO
     }
 
-    ChangesetXML(changesetId: string): string {
+    ChangesetXML(changesetId: string, header?: string): string {
         let members = ""
         for (const member of this.members) {
             members +=
@@ -560,7 +559,7 @@ export class OsmRelation extends OsmObject {
         if (changesetId !== undefined) {
             cs = `changeset="${changesetId}"`
         }
-        return `    <relation id="${this.id}" ${cs} ${this.VersionXML()}>
+        return `    <relation id="${this.id}" ${header ?? ""} ${cs} ${this.VersionXML()}>
 ${members}${tags}        </relation>
 `
     }

scripts/Script.ts

@@ -15,4 +15,8 @@ export default abstract class Script {
         args.splice(0, 2)
         this.main(args).then((_) => console.log("All done"))
     }
+
+    public printHelp(){
+        console.log(this._docs)
+    }
 }

scripts/conflate.ts

@@ -45,9 +45,25 @@ export class Conflate extends Script {
         )
     }
 
+    private static toXml(changedObjects: OsmObject[]): string {
+
+        return [
+            "<?xml version='1.0' encoding='UTF-8'?>",
+            "<osm version=\"0.6\" generator='mapcomplete-conflate-script'>",
+            ...changedObjects.map(obj =>
+                obj.ChangesetXML(undefined, ' action="modify" ')
+            ),
+            "</osm>"
+        ].join("\n");
+    }
+
     async main(args: string[]): Promise<void> {
+        if (args.length < 2) {
+            super.printHelp()
+            return
+        }
         const [osm_file_path, external_file_path] = args
-        let max_range = 50
+        let max_range = 25
         if (args.length === 3) {
             max_range = Number(args[2])
         }
@@ -86,6 +102,8 @@ export class Conflate extends Script {
                 "...properties_differences",
             ],
         ]
+
+        const changedObjects: OsmObject[] = []
         for (const {match, replayed} of bestMatches) {
             const {external_feature, d, osm_feature} = match
             const {possibly_imported, certainly_imported, resting_properties} = replayed
@@ -94,23 +112,38 @@ export class Conflate extends Script {
             if (Object.keys(resting_properties).length === 0) {
                 continue
             }
+            const id = osm_feature.properties["@id"]
             match_lengths.push([
-                osm_feature.properties["@id"],
+                id,
                 d,
                 osm_feature.properties.name,
                 certainly_imported ? "import" : possibly_imported ? "prob import" : "new",
                 status,
                 JSON.stringify(resting_properties),
             ])
+
+            const osmObj = await OsmObject.DownloadObjectAsync(id)
+            for (const key in resting_properties) {
+                osmObj.tags[key] = resting_properties[key]
+            }
+            changedObjects.push(osmObj)
         }
 
+        const targetDir = "../onwheels-data-prep/output"
+        console.log("Writing results to directory", targetDir)
+
         fs.writeFileSync(
-            "../onwheels-data-prep/matches.tsv",
+            targetDir + "/matches.tsv",
             match_lengths.map((l) => l.join("\t")).join("\n")
         )
 
-        fs.writeFileSync(
+        fs.writeFileSync(targetDir + "/changeset.xml",
-            "../onwheels-data-prep/unmatched.geojson",
+            Conflate.toXml(changedObjects)
+        )
+
+
+        fs.writeFileSync(targetDir +
+            "/unmatched.geojson",
             JSON.stringify(
                 {
                     type: "FeatureCollection",
@@ -189,10 +222,12 @@ export class Conflate extends Script {
         if (url.indexOf("facebook.com") > 0) {
             return true
         }
+        if (!fs.existsSync(this.historyCacheDir + "urls/")) {
+            fs.mkdirSync(this.historyCacheDir + "urls/")
+        }
         const cachePath = this.historyCacheDir + "/urls/    " + url.replace(/[/\\:]/g, "_")
         if (fs.existsSync(cachePath)) {
-            const online = JSON.parse(fs.readFileSync(cachePath, { encoding: "utf-8" }))
+            return JSON.parse(fs.readFileSync(cachePath, {encoding: "utf-8"}))
-            return online
         }
         let online: boolean | string = false
         try {
@@ -214,7 +249,9 @@ export class Conflate extends Script {
         }
         url = url.replace("http://", "https://")
         try {
-            const result = await ScriptUtils.Download(url)
+            const result = await ScriptUtils.Download(url, {
+                "User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0"
+            })
             if (result["redirect"]) {
                 if (result["redirect"].startsWith("/")) {
                     return true
@@ -225,6 +262,7 @@ export class Conflate extends Script {
                 return true
             }
             console.error("Got a result, but no content?", url, result)
+            return false
         } catch (e) {
             console.log("Offline (error):", url, e.message)
             return false
@@ -232,7 +270,10 @@ export class Conflate extends Script {
     }
 
     private async historyCached(id): Promise<OsmObject[]> {
-        const cachePath = this.historyCacheDir + "/" + id.replace("/", "_")
+        const cachePath = this.historyCacheDir + id.replace("/", "_")
+        if (!fs.existsSync(this.historyCacheDir)) {
+            fs.mkdirSync(this.historyCacheDir)
+        }
         if (fs.existsSync(cachePath)) {
             return JSON.parse(fs.readFileSync(cachePath, {encoding: "utf-8"}))
         }
@@ -249,8 +290,12 @@ export class Conflate extends Script {
             let website = properties.website.toLowerCase()
             website
                 .replace("http://http://", "http://")
+                .replace("https://https://", "https://")
                 .replace("https//", "https://")
                 .replace("http://", "https://")
+            if (website.startsWith("https://")) {
+                website = "https://" + website
+            }
             const validator = new UrlTextfieldDef()
             if (validator.isValid(website)) {
                 properties.website = new UrlTextfieldDef().reformat(website)