Improve script to handle errors

scripts/handleErrors.ts

@@ -1,5 +1,5 @@
 import Script from "./Script"
-import { readFileSync, writeFileSync } from "fs"
+import { appendFileSync, readFileSync, writeFile, writeFileSync } from "fs"
 import { ChangeDescription } from "../src/Logic/Osm/Actions/ChangeDescription"
 import { Changes } from "../src/Logic/Osm/Changes"
 import { OsmObject } from "../src/Logic/Osm/OsmObject"
@@ -28,6 +28,91 @@ class HandleErrors extends Script {
     constructor() {
         super("Inspects the errors made on a given day. Argument: path to errors")
     }
+
+    private readonly ignoreUsers = new Set<string>([])
+
+    private async handleError(parsed: ErrorMessage, changesObj: Changes, downloader: OsmObjectDownloader, createdChangesets: Set<string>, refusedFiles: Set<string>) {
+        console.log(
+            parsed.message.username,
+            parsed.message.layout,
+            parsed.message.message,
+            parsed.date,
+        )
+
+        const e = parsed.message
+        const neededIds = Changes.GetNeededIds(e.pendingChanges)
+        // We _do not_ pass in the Changes object itself - we want the data from OSM directly in order to apply the changes
+        const osmObjects: { id: string; osmObj: OsmObject | "deleted" }[] = await Promise.all<{
+            id: string
+            osmObj: OsmObject | "deleted"
+        }>(
+            neededIds.map(async (id) => {
+                try {
+
+                    const osmObj = await downloader.DownloadObjectAsync(id)
+                    return ({
+                        id,
+                        osmObj,
+                    })
+                } catch (e) {
+                    console.error("COULD NOT DOWNLOAD OBJECT", id)
+                    return {
+                        id,
+                        osmObj: "deleted",
+                    }
+                }
+            }),
+        )
+
+        const objects = osmObjects
+            .filter((obj) => obj.osmObj !== "deleted")
+            .map((obj) => <OsmObject>obj.osmObj)
+
+        const { toUpload, refused } = changesObj.fragmentChanges(e.pendingChanges, objects)
+
+        const changes: {
+            newObjects: OsmObject[]
+            modifiedObjects: OsmObject[]
+            deletedObjects: OsmObject[]
+        } = changesObj.CreateChangesetObjects(toUpload, objects, true)
+
+        const changeset = Changes.createChangesetFor("", changes)
+        const path =
+            "error_changeset_" + parsed.index + "_" + e.layout + "_" + e.username + ".osc"
+        if (
+            changeset ===
+            `<osmChange version='0.6' generator='Mapcomplete ${Constants.vNumber}'></osmChange>`
+        ) {
+            /*console.log(
+                "Changes for " + parsed.index + ": empty changeset, not creating a file for it"
+            )*/
+        } else if (createdChangesets.has(changeset)) {
+            /* console.log(
+                "Changeset " +
+                    parsed.index +
+                    " is identical to previously seen changeset, not writing to file"
+            )*/
+        } else {
+            const changesetWithMsg = `<!-- User: ${parsed.message.username} (${parsed.message.userid}) ${parsed.message.layout}; Version ${parsed.message.version}; Not uploaded due to ${parsed.message.message} -->
+${changeset}`
+            writeFileSync(path, changesetWithMsg, "utf8")
+            createdChangesets.add(changeset)
+            console.log("Written", path, "with " + e.pendingChanges.length + " changes")
+        }
+        const refusedContent = JSON.stringify(refused)
+        if (refusedFiles.has(refusedContent)) {
+            /* console.log(
+                "Refused changes for " +
+                    parsed.index +
+                    " is identical to previously seen changeset, not writing to file"
+            )*/
+        } else {
+            writeFileSync(path + ".refused.json", refusedContent, "utf8")
+            refusedFiles.add(refusedContent)
+            console.log("Written refused", path)
+        }
+    }
+
     async main(args: string[]): Promise<void> {
         const osmConnection = new OsmConnection()
         const downloader = new OsmObjectDownloader(osmConnection.Backend(), undefined)
@@ -45,7 +130,7 @@ class HandleErrors extends Script {
                 osmConnection,
             },
             false,
-            (err) => console.error(err)
+            (err) => console.error(err),
         )
 
         const all: ErrorMessage[] = []
@@ -60,11 +145,14 @@ class HandleErrors extends Script {
                     console.log("Skipping GRB ")
                     continue
                 }
+                if (this.ignoreUsers.has(e.username)) {
+                    continue
+                }
                 for (const pendingChange of e.pendingChanges) {
                     console.log(
                         "\t https://osm.org/" + pendingChange.type + "/" + pendingChange.id,
                         pendingChange.meta.changeType,
-                        pendingChange.doDelete ? "DELETE" : ""
+                        pendingChange.doDelete ? "DELETE" : "",
                     )
                 }
                 all.push(parsed)
@@ -74,72 +162,12 @@ class HandleErrors extends Script {
         }
 
         for (const parsed of all) {
-            console.log(
+            try {
-                parsed.message.username,
-                parsed.message.layout,
-                parsed.message.message,
-                parsed.date
-            )
 
-            const e = parsed.message
+                await this.handleError(parsed, changesObj, downloader, createdChangesets, refusedFiles)
-            const neededIds = Changes.GetNeededIds(e.pendingChanges)
+            } catch (e) {
-            // We _do not_ pass in the Changes object itself - we want the data from OSM directly in order to apply the changes
+                console.error("ERROR: could not handle ", parsed, " due to", e)
-            const osmObjects: { id: string; osmObj: OsmObject | "deleted" }[] = await Promise.all<{
+                writeFileSync("ERRORS."+parsed.index, "ERROR: due to " + e + ": could not handle\n" + JSON.stringify(parsed), "utf8")
-                id: string
-                osmObj: OsmObject | "deleted"
-            }>(
-                neededIds.map(async (id) => ({
-                    id,
-                    osmObj: await downloader.DownloadObjectAsync(id),
-                }))
-            )
-
-            const objects = osmObjects
-                .filter((obj) => obj.osmObj !== "deleted")
-                .map((obj) => <OsmObject>obj.osmObj)
-
-            const { toUpload, refused } = changesObj.fragmentChanges(e.pendingChanges, objects)
-
-            const changes: {
-                newObjects: OsmObject[]
-                modifiedObjects: OsmObject[]
-                deletedObjects: OsmObject[]
-            } = changesObj.CreateChangesetObjects(toUpload, objects)
-
-            const changeset = Changes.createChangesetFor("", changes)
-            const path =
-                "error_changeset_" + parsed.index + "_" + e.layout + "_" + e.username + ".osc"
-            if (
-                changeset ===
-                `<osmChange version='0.6' generator='Mapcomplete ${Constants.vNumber}'></osmChange>`
-            ) {
-                /*console.log(
-                    "Changes for " + parsed.index + ": empty changeset, not creating a file for it"
-                )*/
-            } else if (createdChangesets.has(changeset)) {
-                /* console.log(
-                    "Changeset " +
-                        parsed.index +
-                        " is identical to previously seen changeset, not writing to file"
-                )*/
-            } else {
-                const changesetWithMsg = `<!-- User: ${parsed.message.username} (${parsed.message.userid}) ${parsed.message.layout}; Version ${parsed.message.version}; Not uploaded due to ${parsed.message.message} -->
-${changeset}`
-                writeFileSync(path, changesetWithMsg, "utf8")
-                createdChangesets.add(changeset)
-                console.log("Written", path, "with " + e.pendingChanges.length + " changes")
-            }
-            const refusedContent = JSON.stringify(refused)
-            if (refusedFiles.has(refusedContent)) {
-                /* console.log(
-                    "Refused changes for " +
-                        parsed.index +
-                        " is identical to previously seen changeset, not writing to file"
-                )*/
-            } else {
-                writeFileSync(path + ".refused.json", refusedContent, "utf8")
-                refusedFiles.add(refusedContent)
-                console.log("Written refused", path)
             }
         }
     }

src/Logic/Osm/Changes.ts

@@ -266,7 +266,8 @@ export class Changes {
 
     public CreateChangesetObjects(
         changes: ChangeDescription[],
-        downloadedOsmObjects: OsmObject[]
+        downloadedOsmObjects: OsmObject[],
+        ignoreNoCreate: boolean = false
     ): {
         newObjects: OsmObject[]
         modifiedObjects: OsmObject[]
@@ -314,6 +315,9 @@ export class Changes {
                 }
                 if (change.changes === undefined) {
                     // This object is a change to a newly created object. However, we have not seen the creation changedescription yet!
+                    if(ignoreNoCreate){
+                        continue
+                    }
                     throw "Not a creation of the object: " + JSON.stringify(change)
                 }
                 // This is a new object that should be created