diff --git a/src/Config.ts b/src/Config.ts index 0ca6b02..a6291a9 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -1,5 +1,22 @@ import {LoginSettings} from "./Mastodon"; +export interface MapCompleteUsageOverview { + topContributorsNumberToShow: number, + topThemesNumberToShow: number, + + /** + * The number of days to look back. + */ + numberOfDays?: 1 | number + + /** + * Only show changes made with this theme. + * If omitted: show all themes + */ + themeWhitelist?: string[] + +} + export default interface Config { /** * Default: https://www.openstreetmap.org @@ -18,8 +35,5 @@ export default interface Config { /** IF set: prints to console instead of to Mastodon*/ dryrun?: boolean }, - postSettings:{ - topContributorsNumberToShow: number, - topThemesNumberToShow: number - } + actions: {mapCompleteUsageOverview : MapCompleteUsageOverview} [] } \ No newline at end of file diff --git a/src/OsmCha.ts b/src/OsmCha.ts index b2362c1..d5dfc05 100644 --- a/src/OsmCha.ts +++ b/src/OsmCha.ts @@ -56,7 +56,7 @@ export default class OsmCha { month: number, day: number ): Promise { - const path = this._cachepath + "_" + year + "_" + Utils. TwoDigits(month) + "_" + Utils. TwoDigits(day) + ".json"; + const path = this._cachepath + "_" + year + "_" + Utils.TwoDigits(month) + "_" + Utils.TwoDigits(day) + ".json"; if (fs.existsSync(path)) { try { return JSON.parse(fs.readFileSync(path, "utf8")).features @@ -66,15 +66,16 @@ export default class OsmCha { } let page = 1 let allFeatures = [] - let endDay = new Date(year, month - 1 /* Zero-indexed: 0 = january*/, day + 1) + const startDay = new Date(year, month - 1, day) + + let endDay = new Date(startDay.getTime() + 1000 * 60 * 60 * 24) + const start_date = year + "-" + Utils.TwoDigits(month) + "-" + Utils.TwoDigits(day) let endDate = `${endDay.getFullYear()}-${Utils.TwoDigits( endDay.getMonth() + 1 )}-${Utils.TwoDigits(endDay.getDate())}` + console.log(start_date, "-->", endDate) let url = this.urlTemplate - .replace( - "{start_date}", - year + "-" + Utils.TwoDigits(month) + "-" + Utils.TwoDigits(day) - ) + .replace("{start_date}", start_date) .replace("{end_date}", endDate) .replace("{page}", "" + page) @@ -96,11 +97,11 @@ export default class OsmCha { while (url) { const result = await Utils.DownloadJson(url, headers) page++ - allFeatures.push(...result.features) if (result.features === undefined) { console.log("ERROR", result) return } + allFeatures.push(...result.features) url = result.next } allFeatures = allFeatures.filter(f => f !== undefined && f !== null) diff --git a/src/OsmUserInfo.ts b/src/OsmUserInfo.ts index 62b80de..f5d23ce 100644 --- a/src/OsmUserInfo.ts +++ b/src/OsmUserInfo.ts @@ -57,7 +57,7 @@ export default class OsmUserInfo { div.innerHTML = userdata.description const links = Array.from(div.getElementsByTagName("a")) const meLinks = links.filter(link => link.getAttribute("rel")?.split(" ")?.indexOf("me") >= 0) - return meLinks.map(link => link.href.toString()) //*/ + return meLinks.map(link => link.href.toString()) } public async getUserInfo(): Promise { diff --git a/src/Postbuilder.ts b/src/Postbuilder.ts index 9abb4d6..6ba7123 100644 --- a/src/Postbuilder.ts +++ b/src/Postbuilder.ts @@ -2,7 +2,7 @@ import Histogram from "./Histogram"; import Utils from "./Utils"; import {ChangeSetData} from "./OsmCha"; import OsmUserInfo from "./OsmUserInfo"; -import Config from "./Config"; +import Config, {MapCompleteUsageOverview} from "./Config"; import MastodonPoster from "./Mastodon"; import ImgurAttribution from "./ImgurAttribution"; @@ -17,11 +17,14 @@ export class Postbuilder { "plantnet-ai-detection", "link-image" ] - private readonly _config: Config; + private readonly _config: MapCompleteUsageOverview; + private readonly _globalConfig: Config + private readonly _poster: MastodonPoster; private readonly _changesetsMade: ChangeSetData[]; - constructor(config: Config, poster: MastodonPoster, changesetsMade: ChangeSetData[]) { + constructor(config: MapCompleteUsageOverview, globalConfig: Config, poster: MastodonPoster, changesetsMade: ChangeSetData[]) { + this._globalConfig = globalConfig; this._poster = poster; this._config = config; // Ignore 'custom' themes, they can be confusing for uninitiated users and give ugly link + we don't endorse them @@ -112,7 +115,7 @@ export class Postbuilder { async createOverviewForContributor(uid: string, changesetsMade: ChangeSetData[]): Promise { - const userinfo = new OsmUserInfo(Number(uid), this._config) + const userinfo = new OsmUserInfo(Number(uid), this._globalConfig) const inf = await userinfo.getUserInfo() const themes = new Histogram(changesetsMade, cs => cs.properties.theme) @@ -170,7 +173,7 @@ export class Postbuilder { const props = image.changeset.properties const uid = "" + props.uid - if (result.indexOf(image) >= 0) { + if (result.findIndex(i => i.image === image.image) >= 0) { continue } @@ -226,11 +229,15 @@ export class Postbuilder { totalImageContributorCount } = await this.prepareImages(changesets, 12) + let timePeriod = "Yesterday" + if(this._config.numberOfDays > 1){ + timePeriod = "In the past "+this._config.numberOfDays+" days" + } let toSend: string[] = [ - "Yesterday, " + perContributor.keys().length + " different persons made " + totalStats.total + " changes to #OpenStreetMap using https://mapcomplete.osm.be .\n", + timePeriod+", " + perContributor.keys().length + " different persons made " + totalStats.total + " changes to #OpenStreetMap using https://mapcomplete.osm.be .\n", ] - for (let i = 0; i < this._config.postSettings.topContributorsNumberToShow - 1 && i < topContributors.length; i++) { + for (let i = 0; i < this._config.topContributorsNumberToShow - 1 && i < topContributors.length; i++) { const uid = topContributors[i].key const changesetsMade = perContributor.get(uid) try { @@ -257,7 +264,7 @@ export class Postbuilder { dropZeroValues: true }) toSend.push("") - for (let i = 0; i < this._config.postSettings.topThemesNumberToShow && i < mostPopularThemes.length; i++) { + for (let i = 0; i < this._config.topThemesNumberToShow && i < mostPopularThemes.length; i++) { const theme = mostPopularThemes[i].key const changesetsMade = perTheme.get(theme) toSend.push(await this.createOverviewForTheme(theme, changesetsMade)) @@ -275,7 +282,7 @@ export class Postbuilder { "Images in this thread are randomly selected from them and were made by: ", ...authorNames.map(auth => "- " + auth), "", - "All changes were made on " + date + "All changes were made on " + date + (this._config.numberOfDays > 1 ? " or at most "+this._config.numberOfDays+"days before": "") ].join("\n"), { inReplyToId: secondPost["id"], @@ -291,15 +298,20 @@ export class Postbuilder { const totalImagesCreated = Utils.Sum(withImage.map(cs => cs.properties["add-image"])) const images: ImageInfo[] = [] + const seenURLS = new Set() for (const changeset of withImage) { - const url = this._config.osmBackend + "/api/0.6/changeset/" + changeset.id + "/download" + const url = this._globalConfig.osmBackend + "/api/0.6/changeset/" + changeset.id + "/download" const osmChangeset = await Utils.DownloadXml(url) const osmChangesetTags: { k: string, v: string }[] = Array.from(osmChangeset.getElementsByTagName("tag")) .map(tag => ({k: tag.getAttribute("k"), v: tag.getAttribute("v")})) .filter(kv => kv.k.startsWith("image")) for (const kv of osmChangesetTags) { + if(seenURLS.has(kv.v)){ + continue + } + seenURLS.add(kv.v) images.push({image: kv.v, changeset}) } } @@ -312,19 +324,19 @@ export class Postbuilder { const cs = randomImage.changeset.properties let authorName = cs.user try { - const authorInfo = new OsmUserInfo(Number(cs.uid), this._config) + const authorInfo = new OsmUserInfo(Number(cs.uid), this._globalConfig) authorName = (await authorInfo.GetMastodonLink()) ?? cs.user }catch (e) { console.log("Could not fetch more info about contributor", authorName, cs.uid, "due to", e) } imgAuthors.push(authorName) - if (this._config.mastodonAuth.dryrun) { + if (this._globalConfig.mastodonAuth.dryrun) { console.log("Not uploading/downloading image:" + randomImage.image + " dryrun") continue } const attribution = await ImgurAttribution.DownloadAttribution(randomImage.image) const id = randomImage.image.substring(randomImage.image.lastIndexOf("/") + 1) - const path = this._config.cacheDir + "/image_" + id + const path = this._globalConfig.cacheDir + "/image_" + id await Utils.DownloadBlob(randomImage.image, path) const mediaId = await this._poster.uploadImage(path, "Image taken by " + authorName + ", available under " + attribution.license + ". It is made with the thematic map " + randomImage.changeset.properties.theme + " in changeset https://openstreetmap.org/changeset/" + randomImage.changeset.id) attachmentIds.push(mediaId) diff --git a/src/index.ts b/src/index.ts index fee7199..c906fbf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,10 +2,9 @@ import * as fakedom from "fake-dom" import * as fs from "fs" import MastodonPoster from "./Mastodon"; import OsmCha, {ChangeSetData} from "./OsmCha"; -import Config from "./Config"; +import Config, {MapCompleteUsageOverview} from "./Config"; import * as configF from "../config/config.json" import {Postbuilder} from "./Postbuilder"; -import {Dir} from "fs"; import Utils from "./Utils"; export class Main { @@ -30,29 +29,28 @@ export class Main { const poster = await MastodonPoster.construct(this._config.mastodonAuth) - const notice = await poster.writeMessage("@pietervdvn@en.osm.town Starting MapComplete bot...",{ + const notice = await poster.writeMessage("@pietervdvn@en.osm.town Starting MapComplete bot...", { visibility: "direct" }) const start = Date.now() try { - console.log("Fetching recent changesets...") - const osmcha = new OsmCha(this._config) - const today = new Date() - let changesets: ChangeSetData[] = await osmcha.DownloadStatsForDay(today.getUTCFullYear(), today.getUTCMonth() + 1, today.getUTCDate() - 1) + for (const action of this._config.actions) { + console.log("Running action", action) + await this.runMapCompleteOverviewAction(poster, action) + } - console.log("Building post...") - await new Postbuilder(this._config, poster, changesets).buildMessage(today.getUTCFullYear() + "-" + Utils.TwoDigits(today.getUTCMonth() + 1) +"-"+ Utils.TwoDigits (today.getUTCDate() - 1)) const end = Date.now() - const timeNeeded= Math.floor ((end - start) / 1000) - await poster.writeMessage("Finished running MapComplete bot, this took "+timeNeeded+"seconds",{ + const timeNeeded = Math.floor((end - start) / 1000) + await poster.writeMessage("Finished running MapComplete bot, this took " + timeNeeded + "seconds", { inReplyToId: notice.id, visibility: "direct" }) } catch (e) { + console.error(e) const end = Date.now() - const timeNeeded= Math.floor ((end - start) / 1000) - await poster.writeMessage("Running MapComplete bot failed in "+timeNeeded+"seconds, the error is "+e,{ + const timeNeeded = Math.floor((end - start) / 1000) + await poster.writeMessage("Running MapComplete bot failed in " + timeNeeded + "seconds, the error is " + e, { inReplyToId: notice.id, visibility: "direct" }) @@ -60,6 +58,46 @@ export class Main { } + private async runMapCompleteOverviewAction(poster: MastodonPoster, action: { mapCompleteUsageOverview: MapCompleteUsageOverview }) { + console.log("Fetching recent changesets...") + const osmcha = new OsmCha(this._config) + const today = new Date() + + const overviewSettings = action.mapCompleteUsageOverview + let changesets: ChangeSetData[] = [] + const days = overviewSettings.numberOfDays ?? 1 + if (days < 1) { + throw new Error("Invalid config: numberOfDays should be >= 1") + } + for (let i = 0; i < days; i++) { + const targetDay = new Date(today.getTime() - 24 * 60 * 60 * 1000 * (i + 1)) + let changesetsDay: ChangeSetData[] = await osmcha.DownloadStatsForDay(targetDay.getUTCFullYear(), targetDay.getUTCMonth() + 1, targetDay.getUTCDate()) + for (const changeSetDatum of changesetsDay) { + if (changeSetDatum.properties.theme === undefined) { + console.warn("Changeset", changeSetDatum.id, " does not have theme given") + } else { + changesets.push(changeSetDatum) + } + } + + } + + if (overviewSettings.themeWhitelist?.length > 0) { + const allowedThemes = new Set(overviewSettings.themeWhitelist) + const beforeCount = changesets.length + changesets = changesets.filter(cs => allowedThemes.has(cs.properties.theme)) + if (changesets.length == 0) { + console.log("No changesets found for themes", overviewSettings.themeWhitelist.join(", ")) + return console.log("No changesets found for themes", overviewSettings.themeWhitelist.join(", ")) + } + console.log("Filtering for ", overviewSettings.themeWhitelist, "yielded", changesets.length, "changesets (" + beforeCount + " before)") + } + + console.log("Building post...") + await new Postbuilder(overviewSettings, this._config, poster, changesets).buildMessage(today.getUTCFullYear() + "-" + Utils.TwoDigits(today.getUTCMonth() + 1) + "-" + Utils.TwoDigits(today.getUTCDate() - 1)) + + } + }