Fetch changeset info from OSM api to detect all images (workaround for https://github.com/mapbox/osmcha-frontend/issues/641); send a direct message to bot owner when running; improve usernames of image authors

package-lock.json

@@ -11,6 +11,7 @@
       "dependencies": {
         "@types/node-fetch": "^2.6.2",
         "@types/showdown": "^2.0.0",
+        "@xmldom/xmldom": "^0.8.6",
         "doctest-ts-improved": "^0.8.7",
         "escape-html": "^1.0.3",
         "fake-dom": "^1.0.4",
@@ -294,6 +295,14 @@
       "resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz",
       "integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q=="
     },
+    "node_modules/@xmldom/xmldom": {
+      "version": "0.8.6",
+      "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.6.tgz",
+      "integrity": "sha512-uRjjusqpoqfmRkTaNuLJ2VohVr67Q5YwDATW3VU7PfzTj6IRaihGrYI7zckGZjxQPBIp63nfvJbM+Yu5ICh0Bg==",
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
     "node_modules/abab": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/abab/-/abab-1.0.4.tgz",
@@ -3099,6 +3108,11 @@
       "resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz",
       "integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q=="
     },
+    "@xmldom/xmldom": {
+      "version": "0.8.6",
+      "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.6.tgz",
+      "integrity": "sha512-uRjjusqpoqfmRkTaNuLJ2VohVr67Q5YwDATW3VU7PfzTj6IRaihGrYI7zckGZjxQPBIp63nfvJbM+Yu5ICh0Bg=="
+    },
     "abab": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/abab/-/abab-1.0.4.tgz",

package.json

@@ -23,6 +23,7 @@
   "dependencies": {
     "@types/node-fetch": "^2.6.2",
     "@types/showdown": "^2.0.0",
+    "@xmldom/xmldom": "^0.8.6",
     "doctest-ts-improved": "^0.8.7",
     "escape-html": "^1.0.3",
     "fake-dom": "^1.0.4",

src/Mastodon.ts

@@ -39,7 +39,7 @@ export default class MastodonPoster {
     
     public async writeMessage(text: string, options?: CreateStatusParamsBase): Promise<{id: string}>{
         if(this._dryrun){  
-            console.log("Dryrun enabled; not posting:\n", text.split("\n").map(txt => "  > "+txt).join("\n"))
+            console.log("Dryrun enabled - not posting",options?.visibility??"public","message: \n" + text.split("\n").map(txt => "  > "+txt).join("\n"))
             return {id: "some_id"}
         }
         const statusUpate = await this.instance.v1.statuses.create({

src/OsmUserInfo.ts

@@ -40,6 +40,17 @@ export default class OsmUserInfo {
 
     }
     
+    public async GetMastodonLink(): Promise<string | undefined> {
+        const mastodonLinks = await this.getMeLinks()
+
+        if (mastodonLinks.length <= 0) {
+            return undefined
+        }
+        
+        const url = new URL(mastodonLinks[0])
+        return url.pathname.substring(1) + "@" + url.host
+    } 
+
     public async getMeLinks(): Promise<string[]> {
         const userdata = await this.getUserInfo()
         const div = document.createElement("div")

src/Postbuilder.ts

@@ -6,7 +6,7 @@ import Config from "./Config";
 import MastodonPoster from "./Mastodon";
 import ImgurAttribution from "./ImgurAttribution";
 
-type ImageInfo = { image: string, changeset: ChangeSetData, tags: Record<string, string[]> }
+type ImageInfo = { image: string, changeset: ChangeSetData }
 
 export class Postbuilder {
     private static readonly metakeys = [
@@ -114,15 +114,11 @@ export class Postbuilder {
     async createOverviewForContributor(uid: string, changesetsMade: ChangeSetData[]): Promise<string> {
         const userinfo = new OsmUserInfo(Number(uid), this._config)
         const inf = await userinfo.getUserInfo()
-        const mastodonLinks = await userinfo.getMeLinks()
 
         const themes = new Histogram(changesetsMade, cs => cs.properties.theme)
 
-        let username = inf.display_name
+        let username = await userinfo.GetMastodonLink() ?? inf.display_name
-        if (mastodonLinks.length > 0) {
+
-            const url = new URL(mastodonLinks[0])
-            username = url.pathname.substring(1) + "@" + url.host
-        }
         const statistics = this.getStatisticsFor(changesetsMade)
 
         let thematicMaps = "maps " + Utils.commasAnd(themes.keys())
@@ -172,7 +168,7 @@ export class Postbuilder {
 
             for (const image of images) {
                 const props = image.changeset.properties
-                const uid = ""+props.uid
+                const uid = "" + props.uid
 
                 if (result.indexOf(image) >= 0) {
                     continue
@@ -199,13 +195,13 @@ export class Postbuilder {
             themeBonus[theme] = (themeBonus[theme] ?? 0) - 1
             const uid = randomBestImage.changeset.properties.uid
             alreadyEncounteredUid.set(uid, (alreadyEncounteredUid.get(uid) ?? 0) + 1)
-            console.log("Selecting image",randomBestImage.image," by ", randomBestImage.changeset.properties.user+" with score "+bestImageScore)
+            console.log("Selecting image", randomBestImage.image, " by ", randomBestImage.changeset.properties.user + " with score " + bestImageScore)
         }
 
         return result
     }
 
-    public async buildMessage(date:string): Promise<void> {
+    public async buildMessage(date: string): Promise<void> {
         const changesets = this._changesetsMade
         const perContributor = new Histogram(changesets, cs => cs.properties.uid)
 
@@ -223,7 +219,12 @@ export class Postbuilder {
 
 
         const totalStats = this.getStatisticsFor()
-        const {totalImagesCreated, attachmentIds, imgAuthors, totalImageContributorCount} = await this.prepareImages(changesets, 12)
+        const {
+            totalImagesCreated,
+            attachmentIds,
+            imgAuthors,
+            totalImageContributorCount
+        } = await this.prepareImages(changesets, 12)
 
         let toSend: string[] = [
             "Yesterday, " + perContributor.keys().length + " different persons made " + totalStats.total + " changes to #OpenStreetMap using https://mapcomplete.osm.be .\n",
@@ -267,17 +268,20 @@ export class Postbuilder {
             mediaIds: attachmentIds.slice(4, 8)
         })
 
-        await this._poster.writeMessage([
-            "In total, "+totalImageContributorCount+" different contributors uploaded "+totalImagesCreated+" images.\n",
-            "Images in this thread are randomly selected from them and were made by: ",
-            ...Array.from(new Set<string>(imgAuthors)).map(auth => "- "+auth ),
-            "",
-            "Changeset of "+date
 
-        ].join("\n"),{
+        const authorNames = Array.from(new Set<string>(imgAuthors))
-            inReplyToId: secondPost["id"],
+        await this._poster.writeMessage([
-            mediaIds: attachmentIds.slice(8,12)
+                "In total, " + totalImageContributorCount + " different contributors uploaded " + totalImagesCreated + " images.\n",
-        })
+                "Images in this thread are randomly selected from them and were made by: ",
+                ...authorNames.map(auth => "- " + auth),
+                "",
+                "All changes were made on " + date
+
+            ].join("\n"), {
+                inReplyToId: secondPost["id"],
+                mediaIds: attachmentIds.slice(8, 12)
+            }
+        )
 
 
     }
@@ -288,15 +292,15 @@ export class Postbuilder {
 
         const images: ImageInfo[] = []
         for (const changeset of withImage) {
-            const tags = changeset.properties.tag_changes
+
-            for (const key in tags) {
+            const url = this._config.osmBackend + "/api/0.6/changeset/" + changeset.id + "/download"
-                if (!key.startsWith("image")) {
+            const osmChangeset = await Utils.DownloadXml(url)
-                    continue
+            const osmChangesetTags: { k: string, v: string }[] = Array.from(osmChangeset.getElementsByTagName("tag"))
-                }
+                .map(tag => ({k: tag.getAttribute("k"), v: tag.getAttribute("v")}))
-                const values: string[] = tags[key]
+                .filter(kv => kv.k.startsWith("image"))
-                for (const image of values) {
+
-                    images.push({image, changeset, tags})
+            for (const kv of osmChangesetTags) {
-                }
+                images.push({image: kv.v, changeset})
             }
         }
 
@@ -304,18 +308,33 @@ export class Postbuilder {
         const attachmentIds: string[] = []
         const imgAuthors: string[] = []
         for (const randomImage of randomImages) {
-            if(this._config.mastodonAuth.dryrun){
+          
-                console.log("Not uploading/downloading image:"+randomImage.image+" dryrun")
+            const cs = randomImage.changeset.properties
+            let authorName = cs.user
+            try {
+                const authorInfo = new OsmUserInfo(Number(cs.uid), this._config)
+                authorName = (await authorInfo.GetMastodonLink()) ?? cs.user
+            }catch (e) {
+                console.log("Could not fetch more info about contributor", authorName, cs.uid, "due to", e)
+            }
+            imgAuthors.push(authorName)
+            if (this._config.mastodonAuth.dryrun) {
+                console.log("Not uploading/downloading image:" + randomImage.image + " dryrun")
                 continue
             }
+            const attribution = await ImgurAttribution.DownloadAttribution(randomImage.image)
             const id = randomImage.image.substring(randomImage.image.lastIndexOf("/") + 1)
             const path = this._config.cacheDir + "/image_" + id
             await Utils.DownloadBlob(randomImage.image, path)
-            const attribution = await ImgurAttribution.DownloadAttribution(randomImage.image)
+            const mediaId = await this._poster.uploadImage(path, "Image taken by " + authorName + ", available under " + attribution.license + ". It is made with the thematic map " + randomImage.changeset.properties.theme + " in changeset https://openstreetmap.org/changeset/" + randomImage.changeset.id)
-            const mediaId = await this._poster.uploadImage(path, "Image taken by " + attribution.author + ", available under " + attribution.license + ". It is made with the thematic map "+randomImage.changeset.properties.theme+" in changeset https://openstreetmap.org/changeset/"+randomImage.changeset.id)
             attachmentIds.push(mediaId)
-            imgAuthors.push(attribution.author)
+           
+        }
+        return {
+            attachmentIds,
+            imgAuthors,
+            totalImagesCreated,
+            totalImageContributorCount: new Set(withImage.map(cs => cs.properties.uid)).size
         }
-        return {attachmentIds, imgAuthors, totalImagesCreated, totalImageContributorCount: new Set(changesets.map(cs => cs.properties.uid)).size}
     }
 }

src/Utils.ts

@@ -1,5 +1,6 @@
 import https from "https";
 import * as fs from "fs";
+import {DOMParser} from '@xmldom/xmldom'
 
 export default class Utils {
     public static async DownloadJson(url, headers?: any): Promise<any> {
@@ -25,11 +26,17 @@ export default class Utils {
         return "" + i
     }
 
+    public static async DownloadXml(url, headers?: any): Promise<Document> {
+        const content = await Utils.Download(url, {...headers, accept: "application/xml"})
+        const parser = new DOMParser();
+        return parser.parseFromString(content.content, "text/xml");
+    }
+
     public static Download(url, headers?: any): Promise<{ content: string }> {
         return new Promise((resolve, reject) => {
             try {
                 headers = headers ?? {}
-                headers.accept = "application/json"
+                headers.accept ??= "application/json"
                 const urlObj = new URL(url)
                 https.get(
                     {
@@ -65,15 +72,15 @@ export default class Utils {
      * Utils.commasAnd(["A"]) // => "A"
      * Utils.commasAnd([]) // => ""
      */
-    public static commasAnd(items: string[]){
+    public static commasAnd(items: string[]) {
-        if(items.length === 1){
+        if (items.length === 1) {
             return items[0]
         }
-        if(items.length === 0){
+        if (items.length === 0) {
             return ""
         }
-        const last = items[items.length - 1 ]
+        const last = items[items.length - 1]
-        return items.slice(0, items.length - 1).join(", ") + " and "+last
+        return items.slice(0, items.length - 1).join(", ") + " and " + last
     }
 
     public static DownloadBlob(url: string, filepath: string): Promise<string> {

src/index.ts

@@ -5,6 +5,8 @@ import OsmCha, {ChangeSetData} from "./OsmCha";
 import Config from "./Config";
 import * as configF from "../config/config.json"
 import {Postbuilder} from "./Postbuilder";
+import {Dir} from "fs";
+import Utils from "./Utils";
 
 export class Main {
 
@@ -12,6 +14,7 @@ export class Main {
 
     constructor(config: Config) {
         this._config = config;
+        this._config.osmBackend ??= "https://www.openstreetmap.org"
     }
 
     async main() {
@@ -27,13 +30,33 @@ export class Main {
 
         const poster = await MastodonPoster.construct(this._config.mastodonAuth)
 
-        console.log("Fetching recent changesets...")
+        const notice = await poster.writeMessage("@pietervdvn@en.osm.town Starting MapComplete bot...",{
-        const osmcha = new OsmCha(this._config)
+            visibility: "direct"
-        const today = new Date()
+        })
-        let changesets: ChangeSetData[] = await osmcha.DownloadStatsForDay(today.getUTCFullYear(), today.getUTCMonth() + 1,  today.getUTCDate() - 1)
+        const start = Date.now()
+        try {
 
-        console.log("Building post...")
+            console.log("Fetching recent changesets...")
-        await new Postbuilder(this._config, poster, changesets).buildMessage(today.getUTCFullYear()+"-"+ (today.getUTCMonth() + 1) +  (today.getUTCDate() - 1))
+            const osmcha = new OsmCha(this._config)
+            const today = new Date()
+            let changesets: ChangeSetData[] = await osmcha.DownloadStatsForDay(today.getUTCFullYear(), today.getUTCMonth() + 1, today.getUTCDate() - 1)
+
+            console.log("Building post...")
+            await new Postbuilder(this._config, poster, changesets).buildMessage(today.getUTCFullYear() + "-" + Utils.TwoDigits(today.getUTCMonth() + 1) +"-"+ Utils.TwoDigits (today.getUTCDate() - 1))
+            const end = Date.now()
+            const timeNeeded= Math.floor ((end - start) / 1000)
+            await poster.writeMessage("Finished running MapComplete bot, this took "+timeNeeded+"seconds",{
+                inReplyToId: notice.id,
+                visibility: "direct"
+            })
+        } catch (e) {
+            const end = Date.now()
+            const timeNeeded= Math.floor ((end - start) / 1000)
+            await poster.writeMessage("Running MapComplete bot failed in "+timeNeeded+"seconds, the error is "+e,{
+                inReplyToId: notice.id,
+                visibility: "direct"
+            })
+        }
 
     }