midgard/MastodonBot
1
0
Fork 0
forked from MapComplete/MastodonBot
Code Pull requests Activity

Fetch changeset info from OSM api to detect all images (workaround for https://github.com/mapbox/osmcha-frontend/issues/641); send a direct message to bot owner when running; improve usernames of image authors

Browse source
This commit is contained in:
Pieter Vander Vennet 2023-01-16 01:54:51 +01:00
parent 70a7224160
commit e7e2d8609f
7 changed files with 131 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
package-lock.json generated
View file

@ -11,6 +11,7 @@
"dependencies": {
"@types/node-fetch": "^2.6.2",
"@types/showdown": "^2.0.0",
"@xmldom/xmldom": "^0.8.6",
"doctest-ts-improved": "^0.8.7",
"escape-html": "^1.0.3",
"fake-dom": "^1.0.4",
@ -294,6 +295,14 @@
"resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz",
"integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q=="
},
"node_modules/@xmldom/xmldom": {
"version": "0.8.6",
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.6.tgz",
"integrity": "sha512-uRjjusqpoqfmRkTaNuLJ2VohVr67Q5YwDATW3VU7PfzTj6IRaihGrYI7zckGZjxQPBIp63nfvJbM+Yu5ICh0Bg==",
"engines": {
"node": ">=10.0.0"
}
},
"node_modules/abab": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/abab/-/abab-1.0.4.tgz",
@ -3099,6 +3108,11 @@
"resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz",
"integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q=="
},
"@xmldom/xmldom": {
"version": "0.8.6",
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.6.tgz",
"integrity": "sha512-uRjjusqpoqfmRkTaNuLJ2VohVr67Q5YwDATW3VU7PfzTj6IRaihGrYI7zckGZjxQPBIp63nfvJbM+Yu5ICh0Bg=="
},
"abab": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/abab/-/abab-1.0.4.tgz",

1
package.json
View file

@ -23,6 +23,7 @@
"dependencies": {
"@types/node-fetch": "^2.6.2",
"@types/showdown": "^2.0.0",
"@xmldom/xmldom": "^0.8.6",
"doctest-ts-improved": "^0.8.7",
"escape-html": "^1.0.3",
"fake-dom": "^1.0.4",

2
src/Mastodon.ts
View file

@ -39,7 +39,7 @@ export default class MastodonPoster {
public async writeMessage(text: string, options?: CreateStatusParamsBase): Promise<{id: string}>{
if(this._dryrun){
console.log("Dryrun enabled; not posting:\n", text.split("\n").map(txt => " > "+txt).join("\n"))
console.log("Dryrun enabled - not posting",options?.visibility??"public","message: \n" + text.split("\n").map(txt => " > "+txt).join("\n"))
return {id: "some_id"}
}
const statusUpate = await this.instance.v1.statuses.create({

11
src/OsmUserInfo.ts
View file

@ -39,6 +39,17 @@ export default class OsmUserInfo {
}
}
public async GetMastodonLink(): Promise<string | undefined> {
const mastodonLinks = await this.getMeLinks()
if (mastodonLinks.length <= 0) {
return undefined
}
const url = new URL(mastodonLinks[0])
return url.pathname.substring(1) + "@" + url.host
}
public async getMeLinks(): Promise<string[]> {
const userdata = await this.getUserInfo()

99
src/Postbuilder.ts
View file

@ -6,7 +6,7 @@ import Config from "./Config";
import MastodonPoster from "./Mastodon";
import ImgurAttribution from "./ImgurAttribution";
type ImageInfo = { image: string, changeset: ChangeSetData, tags: Record<string, string[]> }
type ImageInfo = { image: string, changeset: ChangeSetData }
export class Postbuilder {
private static readonly metakeys = [
@ -114,15 +114,11 @@ export class Postbuilder {
async createOverviewForContributor(uid: string, changesetsMade: ChangeSetData[]): Promise<string> {
const userinfo = new OsmUserInfo(Number(uid), this._config)
const inf = await userinfo.getUserInfo()
const mastodonLinks = await userinfo.getMeLinks()
const themes = new Histogram(changesetsMade, cs => cs.properties.theme)
let username = inf.display_name
if (mastodonLinks.length > 0) {
const url = new URL(mastodonLinks[0])
username = url.pathname.substring(1) + "@" + url.host
}
let username = await userinfo.GetMastodonLink() ?? inf.display_name
const statistics = this.getStatisticsFor(changesetsMade)
let thematicMaps = "maps " + Utils.commasAnd(themes.keys())
@ -164,7 +160,7 @@ export class Postbuilder {
}
const alreadyEncounteredUid = new Map<string, number>()
const result: ImageInfo[] = []
for (let i = 0; i < targetCount; i++) {
let bestImageScore: number = -999999999
@ -172,7 +168,7 @@ export class Postbuilder {
for (const image of images) {
const props = image.changeset.properties
const uid = ""+props.uid
const uid = "" + props.uid
if (result.indexOf(image) >= 0) {
continue
@ -199,13 +195,13 @@ export class Postbuilder {
themeBonus[theme] = (themeBonus[theme] ?? 0) - 1
const uid = randomBestImage.changeset.properties.uid
alreadyEncounteredUid.set(uid, (alreadyEncounteredUid.get(uid) ?? 0) + 1)
console.log("Selecting image",randomBestImage.image," by ", randomBestImage.changeset.properties.user+" with score "+bestImageScore)
console.log("Selecting image", randomBestImage.image, " by ", randomBestImage.changeset.properties.user + " with score " + bestImageScore)
}
return result
}
public async buildMessage(date:string): Promise<void> {
public async buildMessage(date: string): Promise<void> {
const changesets = this._changesetsMade
const perContributor = new Histogram(changesets, cs => cs.properties.uid)
@ -223,7 +219,12 @@ export class Postbuilder {
const totalStats = this.getStatisticsFor()
const {totalImagesCreated, attachmentIds, imgAuthors, totalImageContributorCount} = await this.prepareImages(changesets, 12)
const {
totalImagesCreated,
attachmentIds,
imgAuthors,
totalImageContributorCount
} = await this.prepareImages(changesets, 12)
let toSend: string[] = [
"Yesterday, " + perContributor.keys().length + " different persons made " + totalStats.total + " changes to #OpenStreetMap using https://mapcomplete.osm.be .\n",
@ -266,20 +267,23 @@ export class Postbuilder {
inReplyToId: firstPost["id"],
mediaIds: attachmentIds.slice(4, 8)
})
const authorNames = Array.from(new Set<string>(imgAuthors))
await this._poster.writeMessage([
"In total, "+totalImageContributorCount+" different contributors uploaded "+totalImagesCreated+" images.\n",
"Images in this thread are randomly selected from them and were made by: ",
...Array.from(new Set<string>(imgAuthors)).map(auth => "- "+auth ),
"",
"Changeset of "+date
].join("\n"),{
inReplyToId: secondPost["id"],
mediaIds: attachmentIds.slice(8,12)
})
"In total, " + totalImageContributorCount + " different contributors uploaded " + totalImagesCreated + " images.\n",
"Images in this thread are randomly selected from them and were made by: ",
...authorNames.map(auth => "- " + auth),
"",
"All changes were made on " + date
].join("\n"), {
inReplyToId: secondPost["id"],
mediaIds: attachmentIds.slice(8, 12)
}
)
}
private async prepareImages(changesets: ChangeSetData[], targetCount: number = 4): Promise<{ imgAuthors: string[], attachmentIds: string[], totalImagesCreated: number, totalImageContributorCount: number }> {
@ -288,15 +292,15 @@ export class Postbuilder {
const images: ImageInfo[] = []
for (const changeset of withImage) {
const tags = changeset.properties.tag_changes
for (const key in tags) {
if (!key.startsWith("image")) {
continue
}
const values: string[] = tags[key]
for (const image of values) {
images.push({image, changeset, tags})
}
const url = this._config.osmBackend + "/api/0.6/changeset/" + changeset.id + "/download"
const osmChangeset = await Utils.DownloadXml(url)
const osmChangesetTags: { k: string, v: string }[] = Array.from(osmChangeset.getElementsByTagName("tag"))
.map(tag => ({k: tag.getAttribute("k"), v: tag.getAttribute("v")}))
.filter(kv => kv.k.startsWith("image"))
for (const kv of osmChangesetTags) {
images.push({image: kv.v, changeset})
}
}
@ -304,18 +308,33 @@ export class Postbuilder {
const attachmentIds: string[] = []
const imgAuthors: string[] = []
for (const randomImage of randomImages) {
if(this._config.mastodonAuth.dryrun){
console.log("Not uploading/downloading image:"+randomImage.image+" dryrun")
const cs = randomImage.changeset.properties
let authorName = cs.user
try {
const authorInfo = new OsmUserInfo(Number(cs.uid), this._config)
authorName = (await authorInfo.GetMastodonLink()) ?? cs.user
}catch (e) {
console.log("Could not fetch more info about contributor", authorName, cs.uid, "due to", e)
}
imgAuthors.push(authorName)
if (this._config.mastodonAuth.dryrun) {
console.log("Not uploading/downloading image:" + randomImage.image + " dryrun")
continue
}
const attribution = await ImgurAttribution.DownloadAttribution(randomImage.image)
const id = randomImage.image.substring(randomImage.image.lastIndexOf("/") + 1)
const path = this._config.cacheDir + "/image_" + id
await Utils.DownloadBlob(randomImage.image, path)
const attribution = await ImgurAttribution.DownloadAttribution(randomImage.image)
const mediaId = await this._poster.uploadImage(path, "Image taken by " + attribution.author + ", available under " + attribution.license + ". It is made with the thematic map "+randomImage.changeset.properties.theme+" in changeset https://openstreetmap.org/changeset/"+randomImage.changeset.id)
const mediaId = await this._poster.uploadImage(path, "Image taken by " + authorName + ", available under " + attribution.license + ". It is made with the thematic map " + randomImage.changeset.properties.theme + " in changeset https://openstreetmap.org/changeset/" + randomImage.changeset.id)
attachmentIds.push(mediaId)
imgAuthors.push(attribution.author)
}
return {
attachmentIds,
imgAuthors,
totalImagesCreated,
totalImageContributorCount: new Set(withImage.map(cs => cs.properties.uid)).size
}
return {attachmentIds, imgAuthors, totalImagesCreated, totalImageContributorCount: new Set(changesets.map(cs => cs.properties.uid)).size}
}
}

21
src/Utils.ts
View file

@ -1,5 +1,6 @@
import https from "https";
import * as fs from "fs";
import {DOMParser} from '@xmldom/xmldom'
export default class Utils {
public static async DownloadJson(url, headers?: any): Promise<any> {
@ -25,11 +26,17 @@ export default class Utils {
return "" + i
}
public static async DownloadXml(url, headers?: any): Promise<Document> {
const content = await Utils.Download(url, {...headers, accept: "application/xml"})
const parser = new DOMParser();
return parser.parseFromString(content.content, "text/xml");
}
public static Download(url, headers?: any): Promise<{ content: string }> {
return new Promise((resolve, reject) => {
try {
headers = headers ?? {}
headers.accept = "application/json"
headers.accept ??= "application/json"
const urlObj = new URL(url)
https.get(
{
@ -60,20 +67,20 @@ export default class Utils {
/**
* Adds commas and a single 'and' between the given items
*
*
* Utils.commasAnd(["A","B","C"]) // => "A, B and C"
* Utils.commasAnd(["A"]) // => "A"
* Utils.commasAnd([]) // => ""
*/
public static commasAnd(items: string[]){
if(items.length === 1){
public static commasAnd(items: string[]) {
if (items.length === 1) {
return items[0]
}
if(items.length === 0){
if (items.length === 0) {
return ""
}
const last = items[items.length - 1 ]
return items.slice(0, items.length - 1).join(", ") + " and "+last
const last = items[items.length - 1]
return items.slice(0, items.length - 1).join(", ") + " and " + last
}
public static DownloadBlob(url: string, filepath: string): Promise<string> {

39
src/index.ts
View file

@ -5,13 +5,16 @@ import OsmCha, {ChangeSetData} from "./OsmCha";
import Config from "./Config";
import * as configF from "../config/config.json"
import {Postbuilder} from "./Postbuilder";
import {Dir} from "fs";
import Utils from "./Utils";
export class Main {
private readonly _config: Config;
constructor(config: Config) {
this._config = config;
this._config.osmBackend ??= "https://www.openstreetmap.org"
}
async main() {
@ -27,13 +30,33 @@ export class Main {
const poster = await MastodonPoster.construct(this._config.mastodonAuth)
console.log("Fetching recent changesets...")
const osmcha = new OsmCha(this._config)
const today = new Date()
let changesets: ChangeSetData[] = await osmcha.DownloadStatsForDay(today.getUTCFullYear(), today.getUTCMonth() + 1, today.getUTCDate() - 1)
console.log("Building post...")
await new Postbuilder(this._config, poster, changesets).buildMessage(today.getUTCFullYear()+"-"+ (today.getUTCMonth() + 1) + (today.getUTCDate() - 1))
const notice = await poster.writeMessage("@pietervdvn@en.osm.town Starting MapComplete bot...",{
visibility: "direct"
})
const start = Date.now()
try {
console.log("Fetching recent changesets...")
const osmcha = new OsmCha(this._config)
const today = new Date()
let changesets: ChangeSetData[] = await osmcha.DownloadStatsForDay(today.getUTCFullYear(), today.getUTCMonth() + 1, today.getUTCDate() - 1)
console.log("Building post...")
await new Postbuilder(this._config, poster, changesets).buildMessage(today.getUTCFullYear() + "-" + Utils.TwoDigits(today.getUTCMonth() + 1) +"-"+ Utils.TwoDigits (today.getUTCDate() - 1))
const end = Date.now()
const timeNeeded= Math.floor ((end - start) / 1000)
await poster.writeMessage("Finished running MapComplete bot, this took "+timeNeeded+"seconds",{
inReplyToId: notice.id,
visibility: "direct"
})
} catch (e) {
const end = Date.now()
const timeNeeded= Math.floor ((end - start) / 1000)
await poster.writeMessage("Running MapComplete bot failed in "+timeNeeded+"seconds, the error is "+e,{
inReplyToId: notice.id,
visibility: "direct"
})
}
}