From e7e2d8609f9b573a3d64be9163b0b0ecd00053cb Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Mon, 16 Jan 2023 01:54:51 +0100 Subject: [PATCH] Fetch changeset info from OSM api to detect all images (workaround for https://github.com/mapbox/osmcha-frontend/issues/641); send a direct message to bot owner when running; improve usernames of image authors --- package-lock.json | 14 +++++++ package.json | 1 + src/Mastodon.ts | 2 +- src/OsmUserInfo.ts | 11 ++++++ src/Postbuilder.ts | 99 +++++++++++++++++++++++++++------------------- src/Utils.ts | 21 ++++++---- src/index.ts | 39 ++++++++++++++---- 7 files changed, 131 insertions(+), 56 deletions(-) diff --git a/package-lock.json b/package-lock.json index e731195..86928aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@types/node-fetch": "^2.6.2", "@types/showdown": "^2.0.0", + "@xmldom/xmldom": "^0.8.6", "doctest-ts-improved": "^0.8.7", "escape-html": "^1.0.3", "fake-dom": "^1.0.4", @@ -294,6 +295,14 @@ "resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz", "integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==" }, + "node_modules/@xmldom/xmldom": { + "version": "0.8.6", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.6.tgz", + "integrity": "sha512-uRjjusqpoqfmRkTaNuLJ2VohVr67Q5YwDATW3VU7PfzTj6IRaihGrYI7zckGZjxQPBIp63nfvJbM+Yu5ICh0Bg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/abab": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/abab/-/abab-1.0.4.tgz", @@ -3099,6 +3108,11 @@ "resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz", "integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==" }, + "@xmldom/xmldom": { + "version": "0.8.6", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.6.tgz", + "integrity": "sha512-uRjjusqpoqfmRkTaNuLJ2VohVr67Q5YwDATW3VU7PfzTj6IRaihGrYI7zckGZjxQPBIp63nfvJbM+Yu5ICh0Bg==" + }, "abab": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/abab/-/abab-1.0.4.tgz", diff --git a/package.json b/package.json index 8c77dfb..dc24b56 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "dependencies": { "@types/node-fetch": "^2.6.2", "@types/showdown": "^2.0.0", + "@xmldom/xmldom": "^0.8.6", "doctest-ts-improved": "^0.8.7", "escape-html": "^1.0.3", "fake-dom": "^1.0.4", diff --git a/src/Mastodon.ts b/src/Mastodon.ts index 491f223..3c6745d 100644 --- a/src/Mastodon.ts +++ b/src/Mastodon.ts @@ -39,7 +39,7 @@ export default class MastodonPoster { public async writeMessage(text: string, options?: CreateStatusParamsBase): Promise<{id: string}>{ if(this._dryrun){ - console.log("Dryrun enabled; not posting:\n", text.split("\n").map(txt => " > "+txt).join("\n")) + console.log("Dryrun enabled - not posting",options?.visibility??"public","message: \n" + text.split("\n").map(txt => " > "+txt).join("\n")) return {id: "some_id"} } const statusUpate = await this.instance.v1.statuses.create({ diff --git a/src/OsmUserInfo.ts b/src/OsmUserInfo.ts index dd6af49..62b80de 100644 --- a/src/OsmUserInfo.ts +++ b/src/OsmUserInfo.ts @@ -39,6 +39,17 @@ export default class OsmUserInfo { } } + + public async GetMastodonLink(): Promise { + const mastodonLinks = await this.getMeLinks() + + if (mastodonLinks.length <= 0) { + return undefined + } + + const url = new URL(mastodonLinks[0]) + return url.pathname.substring(1) + "@" + url.host + } public async getMeLinks(): Promise { const userdata = await this.getUserInfo() diff --git a/src/Postbuilder.ts b/src/Postbuilder.ts index bc23bbe..9abb4d6 100644 --- a/src/Postbuilder.ts +++ b/src/Postbuilder.ts @@ -6,7 +6,7 @@ import Config from "./Config"; import MastodonPoster from "./Mastodon"; import ImgurAttribution from "./ImgurAttribution"; -type ImageInfo = { image: string, changeset: ChangeSetData, tags: Record } +type ImageInfo = { image: string, changeset: ChangeSetData } export class Postbuilder { private static readonly metakeys = [ @@ -114,15 +114,11 @@ export class Postbuilder { async createOverviewForContributor(uid: string, changesetsMade: ChangeSetData[]): Promise { const userinfo = new OsmUserInfo(Number(uid), this._config) const inf = await userinfo.getUserInfo() - const mastodonLinks = await userinfo.getMeLinks() const themes = new Histogram(changesetsMade, cs => cs.properties.theme) - let username = inf.display_name - if (mastodonLinks.length > 0) { - const url = new URL(mastodonLinks[0]) - username = url.pathname.substring(1) + "@" + url.host - } + let username = await userinfo.GetMastodonLink() ?? inf.display_name + const statistics = this.getStatisticsFor(changesetsMade) let thematicMaps = "maps " + Utils.commasAnd(themes.keys()) @@ -164,7 +160,7 @@ export class Postbuilder { } const alreadyEncounteredUid = new Map() - + const result: ImageInfo[] = [] for (let i = 0; i < targetCount; i++) { let bestImageScore: number = -999999999 @@ -172,7 +168,7 @@ export class Postbuilder { for (const image of images) { const props = image.changeset.properties - const uid = ""+props.uid + const uid = "" + props.uid if (result.indexOf(image) >= 0) { continue @@ -199,13 +195,13 @@ export class Postbuilder { themeBonus[theme] = (themeBonus[theme] ?? 0) - 1 const uid = randomBestImage.changeset.properties.uid alreadyEncounteredUid.set(uid, (alreadyEncounteredUid.get(uid) ?? 0) + 1) - console.log("Selecting image",randomBestImage.image," by ", randomBestImage.changeset.properties.user+" with score "+bestImageScore) + console.log("Selecting image", randomBestImage.image, " by ", randomBestImage.changeset.properties.user + " with score " + bestImageScore) } return result } - public async buildMessage(date:string): Promise { + public async buildMessage(date: string): Promise { const changesets = this._changesetsMade const perContributor = new Histogram(changesets, cs => cs.properties.uid) @@ -223,7 +219,12 @@ export class Postbuilder { const totalStats = this.getStatisticsFor() - const {totalImagesCreated, attachmentIds, imgAuthors, totalImageContributorCount} = await this.prepareImages(changesets, 12) + const { + totalImagesCreated, + attachmentIds, + imgAuthors, + totalImageContributorCount + } = await this.prepareImages(changesets, 12) let toSend: string[] = [ "Yesterday, " + perContributor.keys().length + " different persons made " + totalStats.total + " changes to #OpenStreetMap using https://mapcomplete.osm.be .\n", @@ -266,20 +267,23 @@ export class Postbuilder { inReplyToId: firstPost["id"], mediaIds: attachmentIds.slice(4, 8) }) - + + + const authorNames = Array.from(new Set(imgAuthors)) await this._poster.writeMessage([ - "In total, "+totalImageContributorCount+" different contributors uploaded "+totalImagesCreated+" images.\n", - "Images in this thread are randomly selected from them and were made by: ", - ...Array.from(new Set(imgAuthors)).map(auth => "- "+auth ), - "", - "Changeset of "+date - - ].join("\n"),{ - inReplyToId: secondPost["id"], - mediaIds: attachmentIds.slice(8,12) - }) - - + "In total, " + totalImageContributorCount + " different contributors uploaded " + totalImagesCreated + " images.\n", + "Images in this thread are randomly selected from them and were made by: ", + ...authorNames.map(auth => "- " + auth), + "", + "All changes were made on " + date + + ].join("\n"), { + inReplyToId: secondPost["id"], + mediaIds: attachmentIds.slice(8, 12) + } + ) + + } private async prepareImages(changesets: ChangeSetData[], targetCount: number = 4): Promise<{ imgAuthors: string[], attachmentIds: string[], totalImagesCreated: number, totalImageContributorCount: number }> { @@ -288,15 +292,15 @@ export class Postbuilder { const images: ImageInfo[] = [] for (const changeset of withImage) { - const tags = changeset.properties.tag_changes - for (const key in tags) { - if (!key.startsWith("image")) { - continue - } - const values: string[] = tags[key] - for (const image of values) { - images.push({image, changeset, tags}) - } + + const url = this._config.osmBackend + "/api/0.6/changeset/" + changeset.id + "/download" + const osmChangeset = await Utils.DownloadXml(url) + const osmChangesetTags: { k: string, v: string }[] = Array.from(osmChangeset.getElementsByTagName("tag")) + .map(tag => ({k: tag.getAttribute("k"), v: tag.getAttribute("v")})) + .filter(kv => kv.k.startsWith("image")) + + for (const kv of osmChangesetTags) { + images.push({image: kv.v, changeset}) } } @@ -304,18 +308,33 @@ export class Postbuilder { const attachmentIds: string[] = [] const imgAuthors: string[] = [] for (const randomImage of randomImages) { - if(this._config.mastodonAuth.dryrun){ - console.log("Not uploading/downloading image:"+randomImage.image+" dryrun") + + const cs = randomImage.changeset.properties + let authorName = cs.user + try { + const authorInfo = new OsmUserInfo(Number(cs.uid), this._config) + authorName = (await authorInfo.GetMastodonLink()) ?? cs.user + }catch (e) { + console.log("Could not fetch more info about contributor", authorName, cs.uid, "due to", e) + } + imgAuthors.push(authorName) + if (this._config.mastodonAuth.dryrun) { + console.log("Not uploading/downloading image:" + randomImage.image + " dryrun") continue } + const attribution = await ImgurAttribution.DownloadAttribution(randomImage.image) const id = randomImage.image.substring(randomImage.image.lastIndexOf("/") + 1) const path = this._config.cacheDir + "/image_" + id await Utils.DownloadBlob(randomImage.image, path) - const attribution = await ImgurAttribution.DownloadAttribution(randomImage.image) - const mediaId = await this._poster.uploadImage(path, "Image taken by " + attribution.author + ", available under " + attribution.license + ". It is made with the thematic map "+randomImage.changeset.properties.theme+" in changeset https://openstreetmap.org/changeset/"+randomImage.changeset.id) + const mediaId = await this._poster.uploadImage(path, "Image taken by " + authorName + ", available under " + attribution.license + ". It is made with the thematic map " + randomImage.changeset.properties.theme + " in changeset https://openstreetmap.org/changeset/" + randomImage.changeset.id) attachmentIds.push(mediaId) - imgAuthors.push(attribution.author) + + } + return { + attachmentIds, + imgAuthors, + totalImagesCreated, + totalImageContributorCount: new Set(withImage.map(cs => cs.properties.uid)).size } - return {attachmentIds, imgAuthors, totalImagesCreated, totalImageContributorCount: new Set(changesets.map(cs => cs.properties.uid)).size} } } \ No newline at end of file diff --git a/src/Utils.ts b/src/Utils.ts index 067c83c..32688d6 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -1,5 +1,6 @@ import https from "https"; import * as fs from "fs"; +import {DOMParser} from '@xmldom/xmldom' export default class Utils { public static async DownloadJson(url, headers?: any): Promise { @@ -25,11 +26,17 @@ export default class Utils { return "" + i } + public static async DownloadXml(url, headers?: any): Promise { + const content = await Utils.Download(url, {...headers, accept: "application/xml"}) + const parser = new DOMParser(); + return parser.parseFromString(content.content, "text/xml"); + } + public static Download(url, headers?: any): Promise<{ content: string }> { return new Promise((resolve, reject) => { try { headers = headers ?? {} - headers.accept = "application/json" + headers.accept ??= "application/json" const urlObj = new URL(url) https.get( { @@ -60,20 +67,20 @@ export default class Utils { /** * Adds commas and a single 'and' between the given items - * + * * Utils.commasAnd(["A","B","C"]) // => "A, B and C" * Utils.commasAnd(["A"]) // => "A" * Utils.commasAnd([]) // => "" */ - public static commasAnd(items: string[]){ - if(items.length === 1){ + public static commasAnd(items: string[]) { + if (items.length === 1) { return items[0] } - if(items.length === 0){ + if (items.length === 0) { return "" } - const last = items[items.length - 1 ] - return items.slice(0, items.length - 1).join(", ") + " and "+last + const last = items[items.length - 1] + return items.slice(0, items.length - 1).join(", ") + " and " + last } public static DownloadBlob(url: string, filepath: string): Promise { diff --git a/src/index.ts b/src/index.ts index e8c9211..fee7199 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,13 +5,16 @@ import OsmCha, {ChangeSetData} from "./OsmCha"; import Config from "./Config"; import * as configF from "../config/config.json" import {Postbuilder} from "./Postbuilder"; +import {Dir} from "fs"; +import Utils from "./Utils"; export class Main { - + private readonly _config: Config; constructor(config: Config) { this._config = config; + this._config.osmBackend ??= "https://www.openstreetmap.org" } async main() { @@ -27,13 +30,33 @@ export class Main { const poster = await MastodonPoster.construct(this._config.mastodonAuth) - console.log("Fetching recent changesets...") - const osmcha = new OsmCha(this._config) - const today = new Date() - let changesets: ChangeSetData[] = await osmcha.DownloadStatsForDay(today.getUTCFullYear(), today.getUTCMonth() + 1, today.getUTCDate() - 1) - - console.log("Building post...") - await new Postbuilder(this._config, poster, changesets).buildMessage(today.getUTCFullYear()+"-"+ (today.getUTCMonth() + 1) + (today.getUTCDate() - 1)) + const notice = await poster.writeMessage("@pietervdvn@en.osm.town Starting MapComplete bot...",{ + visibility: "direct" + }) + const start = Date.now() + try { + + console.log("Fetching recent changesets...") + const osmcha = new OsmCha(this._config) + const today = new Date() + let changesets: ChangeSetData[] = await osmcha.DownloadStatsForDay(today.getUTCFullYear(), today.getUTCMonth() + 1, today.getUTCDate() - 1) + + console.log("Building post...") + await new Postbuilder(this._config, poster, changesets).buildMessage(today.getUTCFullYear() + "-" + Utils.TwoDigits(today.getUTCMonth() + 1) +"-"+ Utils.TwoDigits (today.getUTCDate() - 1)) + const end = Date.now() + const timeNeeded= Math.floor ((end - start) / 1000) + await poster.writeMessage("Finished running MapComplete bot, this took "+timeNeeded+"seconds",{ + inReplyToId: notice.id, + visibility: "direct" + }) + } catch (e) { + const end = Date.now() + const timeNeeded= Math.floor ((end - start) / 1000) + await poster.writeMessage("Running MapComplete bot failed in "+timeNeeded+"seconds, the error is "+e,{ + inReplyToId: notice.id, + visibility: "direct" + }) + } }