From edf6301cbff6d23c8d166b7035f5145531f8ece5 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Tue, 28 Nov 2023 13:06:44 +0100 Subject: [PATCH] Formatting, attempt to debug (without success) --- package.json | 2 +- src/Mastodon.ts | 38 +++++- src/Postbuilder.ts | 312 +++++++++++++++++++++++---------------------- src/index.ts | 30 ++--- 4 files changed, 207 insertions(+), 175 deletions(-) diff --git a/package.json b/package.json index 144379a..df75d2b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mastodon-bot", - "version": "0.0.2", + "version": "0.0.3", "author": "Pietervdvn", "license": "GPL", "description": "Experimenting with mastodon-bot", diff --git a/src/Mastodon.ts b/src/Mastodon.ts index d467997..6c215a1 100644 --- a/src/Mastodon.ts +++ b/src/Mastodon.ts @@ -44,12 +44,36 @@ export default class MastodonPoster { ) } + /** + * Returns the length, counting a link as 23 characters + * @param text + */ + public static length23(text: string): number{ + const splitted = text.split(" ") + + let total = 0; + for (const piece of splitted) { + try{ + // This is a link, it counts for 23 characters + // https://docs.joinmastodon.org/user/posting/#links + new URL(piece) + total += 23 + }catch(e){ + total += piece.length + } + } + // add the spaces + total += splitted.length - 1 + return total + + } + public async writeMessage(text: string, options?: CreateStatusParamsBase): Promise<{ id: string }> { if (options?.visibility === "direct" && text.indexOf("@") < 0) { throw ("Error: you try to send a direct message, but it has no username...") } - if (text.length > 500) { + if (MastodonPoster.length23(text) > 500) { console.log(text.split("\n").map(txt => " > " + txt).join("\n")) throw "Error: text is too long:" + text.length @@ -61,17 +85,19 @@ export default class MastodonPoster { } if (this._dryrun) { - console.log("Dryrun enabled - not posting", options?.visibility ?? "public", "message: \n" + text.split("\n").map(txt => " > " + txt).join("\n")) + console.log("Dryrun enabled - not posting", options?.visibility ?? "public", `message (length ${text.length}, link23: ${MastodonPoster.length23(text)}): +${text.split("\n").map(txt => " > " + txt).join("\n")}`) return {id: "some_id"} } - const statusUpate = await this.instance.v1.statuses.create({ + console.log("Uploading message", text.substring(0, 25)+"...", `(length ${text.length}, link23: ${MastodonPoster.length23(text)})`) + console.log(text.split("\n").map(txt => " > " + txt).join("\n")) + const statusUpdate = await this.instance.v1.statuses.create({ visibility: 'public', ...(options ?? {}), status: text }) - console.log("Posted to", statusUpate.url) - console.log(text.split("\n").map(txt => " > " + txt).join("\n")) - return statusUpate + console.log("Posted successfully to", statusUpdate.url) + return statusUpdate } public async hasNoBot(username: string): Promise { diff --git a/src/Postbuilder.ts b/src/Postbuilder.ts index 34374db..658fa4e 100644 --- a/src/Postbuilder.ts +++ b/src/Postbuilder.ts @@ -34,8 +34,150 @@ export class Postbuilder { ; } + public async buildMessage(date: string): Promise { + const changesets = this._changesetsMade + let lastPostId: string = undefined - getStatisticsFor(changesetsMade?: ChangeSetData[]): { + + if (this._config.report) { + const report = this._config.report + const overpass = new Overpass(report) + const data = await overpass.query() + const ids = data.elements.map(e => e.type + "/" + e.id) + const total = new Set(ids).size + const date = data.osm3s.timestamp_osm_base.substring(0, 10) + lastPostId = (await this._poster.writeMessage( + report.post.replace(/{total}/g, "" + total).replace(/{date}/g, date), + {spoilerText: this._config.contentWarning} + )).id + } + + const perContributor = new Histogram(changesets, cs => cs.properties.uid) + const topContributors = perContributor.sortedByCount({ + countMethod: cs => { + let sum = 0 + for (const metakey of Postbuilder.metakeys) { + if (cs.properties[metakey]) { + sum += cs.properties[metakey] + } + } + return sum + } + }); + + + const totalStats = this.getStatisticsFor() + const { + totalImagesCreated, + randomImages, + totalImageContributorCount + } = await this.prepareImages(changesets) + const imageUploader = new ImageUploader(randomImages, this._poster, this._globalConfig) + + let timePeriod = "yesterday" + if (this._config.numberOfDays > 1) { + timePeriod = "in the past " + this._config.numberOfDays + " days" + } + + + if (this._config.showTopContributors && topContributors.length > 0) { + const singleTheme = this._config?.themeWhitelist?.length === 1 ? "/" + this._config.themeWhitelist[0] : "" + const toSend: string[] = [ + `${perContributor.keys().length} people made ${totalStats.total} changes ${timePeriod} to #OpenStreetMap using https://mapcomplete.org${singleTheme}`, + "" + ] + + // We group contributors that only contributed to 'etymology' as they otherwise spam the first entry + + const { + alreadyMentioned, + message + } = await this.GroupTopcontributorsForTheme("etymology", topContributors, perContributor) + if (message?.length > 0) { + toSend.push(message) + } + + for (const topContributor of topContributors) { + const uid = topContributor.key + if (alreadyMentioned.has(uid)) { + continue + } + const changesetsMade = perContributor.get(uid) + try { + const userInfo = new OsmUserInfo(Number(uid), this._globalConfig) + const {nobot} = await userInfo.hasNoBotTag() + if (nobot) { + continue + } + const overview = await this.createOverviewForContributor(uid, changesetsMade) + if (MastodonPoster.length23(overview) + MastodonPoster.length23(toSend.join("\n")) + 1 /*+1 for the separating \n*/ > 500) { + break + } + toSend.push(overview) + } catch (e) { + console.error("Could not add contributor " + uid, e) + } + } + lastPostId = (await this._poster.writeMessage(toSend.join("\n"), + { + inReplyToId: lastPostId, + mediaIds: await imageUploader.attemptToUpload(4), + spoilerText: this._config.contentWarning + })).id + } + + const perTheme = new Histogram(changesets, cs => { + return cs.properties.theme; + }) + + const mostPopularThemes = perTheme.sortedByCount({ + countMethod: cs => this.getStatisticsFor([cs]).total, + dropZeroValues: true + }) + if (this._config.showTopThemes && mostPopularThemes.length > 0) { + const toSend = [] + for (const theme of mostPopularThemes) { + const themeId = theme.key + const changesetsMade = perTheme.get(themeId) + const overview = await this.createOverviewForTheme(themeId, changesetsMade) + if (MastodonPoster.length23(overview) + MastodonPoster.length23(toSend.join("\n")) + 1 > 500) { + break + } + toSend.push(overview) + } + + lastPostId = (await this._poster.writeMessage(toSend.join("\n"), { + inReplyToId: lastPostId, + // mediaIds: await imageUploader.attemptToUpload(4), + spoilerText: this._config.contentWarning + })).id + } + + + const images = await imageUploader.attemptToUpload(4) + const authors = Array.from(new Set(imageUploader.getCurrentAuthors())) + if (authors.length > 0) { + await this._poster.writeMessage([ + "In total, " + totalImageContributorCount + " different contributors uploaded " + totalImagesCreated + " images.\n", + "Images in this thread are randomly selected from them and were made by: ", + ...authors, + "", + "A big thanks to everyone who is contributing!", + "", + "All changes were made on " + date + (this._config.numberOfDays > 1 ? ` or at most ${this._config.numberOfDays} days before` : "") + + ].join("\n"), { + inReplyToId: lastPostId, + mediaIds: images, + spoilerText: this._config.contentWarning + } + ) + } + + + } + + private getStatisticsFor(changesetsMade?: ChangeSetData[]): { total: number, answered?: number, created?: number, @@ -132,8 +274,7 @@ export class Postbuilder { } - - async createOverviewForContributor(uid: string, changesetsMade: ChangeSetData[]): Promise { + private async createOverviewForContributor(uid: string, changesetsMade: ChangeSetData[]): Promise { const userinfo = new OsmUserInfo(Number(uid), this._globalConfig) const inf = await userinfo.getUserInfo() @@ -153,7 +294,7 @@ export class Postbuilder { return username + " " + statistics.summaryText + thematicMaps } - async createOverviewForTheme(theme: string, changesetsMade: ChangeSetData[]): Promise { + private async createOverviewForTheme(theme: string, changesetsMade: ChangeSetData[]): Promise { const statistics = this.getStatisticsFor(changesetsMade) const contributorCount = new Set(changesetsMade.map(cs => cs.properties.uid)).size @@ -170,7 +311,7 @@ export class Postbuilder { * However, it is biased to select pictures from certain themes too * @param images */ - public selectImages(images: ImageInfo[]): + private selectImages(images: ImageInfo[]): ImageInfo[] { const themeBonus = { @@ -229,7 +370,13 @@ export class Postbuilder { return result } - public async GroupTopcontributorsForTheme(theme: string, topContributors: { key: string; count: number }[], perContributor: Histogram, maxCount = 3): Promise<{ alreadyMentioned: Set; message: string }> { + private async GroupTopcontributorsForTheme(theme: string, topContributors: { + key: string; + count: number + }[], perContributor: Histogram, maxCount = 3): Promise<{ + alreadyMentioned: Set; + message: string + }> { const alreadyMentioned = new Set() const etymologyContributors: { username: string }[] = [] for (const topContributor of topContributors) { @@ -250,7 +397,7 @@ export class Postbuilder { etymologyContributors.push({username}) } let message: string - if(etymologyContributors.length <= 1){ + if (etymologyContributors.length <= 1) { return { alreadyMentioned: new Set(), message: undefined @@ -269,152 +416,11 @@ export class Postbuilder { } } - public async buildMessage(date: string): Promise { - const changesets = this._changesetsMade - let lastPostId: string = undefined - - - if (this._config.report) { - const report = this._config.report - const overpass = new Overpass(report) - const data = await overpass.query() - const ids = data.elements.map(e => e.type+"/"+e.id) - const total = new Set(ids).size - const date = data.osm3s.timestamp_osm_base.substring(0, 10) - lastPostId = (await this._poster.writeMessage( - report.post.replace(/{total}/g, "" + total).replace(/{date}/g, date), - {spoilerText: this._config.contentWarning} - )).id - } - - const perContributor = new Histogram(changesets, cs => cs.properties.uid) - const topContributors = perContributor.sortedByCount({ - countMethod: cs => { - let sum = 0 - for (const metakey of Postbuilder.metakeys) { - if (cs.properties[metakey]) { - sum += cs.properties[metakey] - } - } - return sum - } - }); - - - const totalStats = this.getStatisticsFor() - const { - totalImagesCreated, - randomImages, - totalImageContributorCount - } = await this.prepareImages(changesets) - const imageUploader = new ImageUploader(randomImages, this._poster, this._globalConfig) - - let timePeriod = "yesterday" - if (this._config.numberOfDays > 1) { - timePeriod = "in the past " + this._config.numberOfDays + " days" - } - const singleTheme = this._config?.themeWhitelist?.length === 1 ? "/" + this._config.themeWhitelist[0] : "" - let toSend: string[] = [ - `${perContributor.keys().length} people made ${totalStats.total} changes ${timePeriod} to #OpenStreetMap using https://mapcomplete.org${singleTheme}`, - "" - ] - - - if (this._config.showTopContributors && topContributors.length > 0) { - - // We group contributors that only contributed to 'etymology' as they otherwise spam the first entry - - const { - alreadyMentioned, - message - } = await this.GroupTopcontributorsForTheme("etymology", topContributors, perContributor) - if(message?.length > 0){ - toSend.push(message) - } - - for (const topContributor of topContributors) { - const uid = topContributor.key - if (alreadyMentioned.has(uid)) { - continue - } - const changesetsMade = perContributor.get(uid) - try { - const userInfo = new OsmUserInfo(Number(uid), this._globalConfig) - const {nobot} = await userInfo.hasNoBotTag() - if (nobot) { - continue - } - const overview = await this.createOverviewForContributor(uid, changesetsMade) - if (overview.length + toSend.join("\n").length + 1 /*+1 for the separating \n*/ > 500) { - break - } - toSend.push(overview) - } catch (e) { - console.error("Could not add contributor " + uid, e) - } - } - lastPostId = (await this._poster.writeMessage(toSend.join("\n"), - { - inReplyToId: lastPostId, - mediaIds: await imageUploader.attemptToUpload(4), - spoilerText: this._config.contentWarning - })).id - toSend = [] - } - - const perTheme = new Histogram(changesets, cs => { - return cs.properties.theme; - }) - - const mostPopularThemes = perTheme.sortedByCount({ - countMethod: cs => this.getStatisticsFor([cs]).total, - dropZeroValues: true - }) - if (this._config.showTopThemes && mostPopularThemes.length > 0) { - - for (const theme of mostPopularThemes) { - const themeId = theme.key - const changesetsMade = perTheme.get(themeId) - const overview = await this.createOverviewForTheme(themeId, changesetsMade) - if (overview.length + toSend.join("\n").length + 1 > 500) { - break - } - toSend.push(overview) - } - - lastPostId = (await this._poster.writeMessage(toSend.join("\n"), { - inReplyToId: lastPostId, - mediaIds: await imageUploader.attemptToUpload(4), - spoilerText: this._config.contentWarning - })).id - toSend = [] - } - - - const images = await imageUploader.attemptToUpload(4) - const authors = Array.from(new Set(imageUploader.getCurrentAuthors())) - if (authors.length > 0) { - await this._poster.writeMessage([ - "In total, " + totalImageContributorCount + " different contributors uploaded " + totalImagesCreated + " images.\n", - "Images in this thread are randomly selected from them and were made by: ", - ...authors, - "", - "A big thanks to everyone who is contributing!", - "", - "All changes were made on " + date + (this._config.numberOfDays > 1 ? ` or at most ${this._config.numberOfDays} days before` : "") - - ].join("\n"), { - inReplyToId: lastPostId, - mediaIds: images, - spoilerText: this._config.contentWarning - } - ) - } - - - } - - private async prepareImages(changesets: ChangeSetData[]): Promise<{ randomImages: { image: string, changeset: ChangeSetData }[], totalImagesCreated: number, totalImageContributorCount: number }> { + private async prepareImages(changesets: ChangeSetData[]): Promise<{ + randomImages: { image: string, changeset: ChangeSetData }[], + totalImagesCreated: number, + totalImageContributorCount: number + }> { const withImage: ChangeSetData[] = changesets.filter(cs => cs.properties["add-image"] > 0) const totalImagesCreated = Utils.Sum(withImage.map(cs => cs.properties["add-image"])) diff --git a/src/index.ts b/src/index.ts index 87b0fce..d8483d4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,7 +11,7 @@ export class Main { private readonly _config: Config; constructor(config: string | Config) { - if(config === undefined){ + if (config === undefined) { console.log("Needs an argument: path of config file") throw "No path given" } @@ -37,19 +37,19 @@ export class Main { const start = Date.now() - try { - for (const action of this._config.actions) { - console.log("Running action", action) + for (const action of this._config.actions) { + try { + console.log("# Running action", action) await this.runMapCompleteOverviewAction(poster, action) + } catch (e) { + console.error("Caught top level exception: ", e) + console.log(e.stack) + const end = Date.now() + const timeNeeded = Math.floor((end - start) / 1000) + await poster.writeMessage("@pietervdvn@en.osm.town Running MapComplete bot failed in " + timeNeeded + "seconds, the error is " + e, { + visibility: "direct" + }) } - } catch (e) { - console.error("Caught top level exception: ", e) - console.log(e.stack) - const end = Date.now() - const timeNeeded = Math.floor((end - start) / 1000) - await poster.writeMessage("@pietervdvn@en.osm.town Running MapComplete bot failed in " + timeNeeded + "seconds, the error is " + e, { - visibility: "direct" - }) } } @@ -67,7 +67,7 @@ export class Main { for (let i = 0; i < days; i++) { const targetDay = new Date(today.getTime() - 24 * 60 * 60 * 1000 * (i + 1)) let changesetsDay: ChangeSetData[] = await osmcha.DownloadStatsForDay(targetDay.getUTCFullYear(), targetDay.getUTCMonth() + 1, targetDay.getUTCDate()) - console.log("OsmCha has", changesets.length,"changesets for", targetDay.toISOString()) + console.log("OsmCha has", changesets.length, "changesets for", targetDay.toISOString()) for (const changeSetDatum of changesetsDay) { if (changeSetDatum.properties.theme === undefined) { console.warn("Changeset", changeSetDatum.id, " does not have theme given") @@ -77,7 +77,7 @@ export class Main { } } - console.log("Found",changesets.length,"matching changesets") + console.log("Found", changesets.length, "matching changesets") if (action.themeWhitelist?.length > 0) { const allowedThemes = new Set(action.themeWhitelist) @@ -85,7 +85,7 @@ export class Main { changesets = changesets.filter(cs => allowedThemes.has(cs.properties.theme)) if (changesets.length == 0) { console.log("No changesets found for themes", action.themeWhitelist.join(", ")) - }else{ + } else { console.log("Filtering for ", action.themeWhitelist, "yielded", changesets.length, "changesets (" + beforeCount + " before)") } }