forked from MapComplete/MastodonBot
First version of mastodon-bot
This commit is contained in:
Pieter Vander Vennet
parent
dc913177bd
commit
48cf429499
14 changed files with 6055 additions and 0 deletions
3
.gitignore
vendored
Normal file
3
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
config/
|
||||
cache/
|
||||
node_modules/
|
||||
5069
package-lock.json
generated
Normal file
5069
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
42
package.json
Normal file
42
package.json
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
{
|
||||
"name": "mastodon-bot",
|
||||
"version": "0.0.1",
|
||||
"author": "Pietervdvn",
|
||||
"license": "GPL",
|
||||
"description": "Experimenting with mastodon-bot",
|
||||
"repository": "https://github.com/pietervdvn/matrix-bot",
|
||||
"bugs": {
|
||||
"url": "https://github.com/pietervdvn/matrix-bot/issues"
|
||||
},
|
||||
"keywords": [
|
||||
"Mastodon",
|
||||
"OpenStreetMap",
|
||||
"bot"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"lint": "tslint --project ./tsconfig.json -t stylish",
|
||||
"start": "ts-node src/index.ts",
|
||||
"test": "doctest-ts-improved src/ && mocha --require ts-node/register \"./**/*.doctest.ts\" && (find . -type f -name \"*.doctest.ts\" | xargs rm)"
|
||||
},
|
||||
"dependencies": {
|
||||
"@types/node-fetch": "^2.6.2",
|
||||
"@types/showdown": "^2.0.0",
|
||||
"doctest-ts-improved": "^0.8.7",
|
||||
"escape-html": "^1.0.3",
|
||||
"fake-dom": "^1.0.4",
|
||||
"fs": "^0.0.1-security",
|
||||
"https": "^1.0.0",
|
||||
"masto": "^5.4.0",
|
||||
"mocha": "^10.0.0",
|
||||
"node-fetch": "^3.3.0",
|
||||
"showdown": "^2.1.0",
|
||||
"ts-node": "^10.7.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/mocha": "^9.1.1",
|
||||
"@types/node": "^14.18.5",
|
||||
"tslint": "^6.1.3",
|
||||
"typescript": "^4.7.4"
|
||||
}
|
||||
}
|
||||
25
src/Config.ts
Normal file
25
src/Config.ts
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
import {LoginSettings} from "./Mastodon";
|
||||
|
||||
export default interface Config {
|
||||
/**
|
||||
* Default: https://www.openstreetmap.org
|
||||
*/
|
||||
osmBackend?: string,
|
||||
/**
|
||||
* Directory to place caching files.
|
||||
* If undefined: no caching will be used
|
||||
*/
|
||||
cacheDir?: string,
|
||||
|
||||
/**
|
||||
* Authentication options for mastodon
|
||||
*/
|
||||
mastodonAuth: LoginSettings & {
|
||||
/** IF set: prints to console instead of to Mastodon*/
|
||||
dryrun?: boolean
|
||||
},
|
||||
postSettings:{
|
||||
topContributorsNumberToShow: number,
|
||||
topThemesNumberToShow: number
|
||||
}
|
||||
}
|
||||
59
src/Histogram.ts
Normal file
59
src/Histogram.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
export default class Histogram<T> {
|
||||
|
||||
private readonly hist: Record<string, T[]>
|
||||
|
||||
constructor(items: T[], key: (t: T) => string) {
|
||||
const hist: Record<any, T[]> = {}
|
||||
for (const item of items) {
|
||||
const k: string = key(item)
|
||||
if (k === undefined) {
|
||||
continue
|
||||
}
|
||||
if (hist[k] === undefined) {
|
||||
hist[k] = []
|
||||
}
|
||||
hist[k].push(item)
|
||||
}
|
||||
this.hist = hist
|
||||
}
|
||||
|
||||
public sortedByCount(options?: {
|
||||
countMethod?: (t:T) => number,
|
||||
dropZeroValues?: boolean
|
||||
}): {key: string, count: number}[] {
|
||||
|
||||
const result :{key: string, count: number}[]= []
|
||||
|
||||
for (const key in this.hist) {
|
||||
const items = this.hist[key]
|
||||
let count = 0
|
||||
if(options?.countMethod){
|
||||
for (const item of items) {
|
||||
const v =options?.countMethod(item)
|
||||
if(v === undefined || v == null || Number.isNaN(v)){
|
||||
continue
|
||||
}
|
||||
count += v
|
||||
}
|
||||
}else{
|
||||
count = items.length
|
||||
}
|
||||
if(options?.dropZeroValues && count === 0){
|
||||
continue
|
||||
}
|
||||
result.push({key, count})
|
||||
}
|
||||
result.sort((a,b) => b.count - a.count)
|
||||
|
||||
return result
|
||||
|
||||
}
|
||||
|
||||
public keys():string[] {
|
||||
return Object.keys(this.hist)
|
||||
}
|
||||
|
||||
get(contribUid: string) {
|
||||
return this.hist[contribUid]
|
||||
}
|
||||
}
|
||||
28
src/ImgurAttribution.ts
Normal file
28
src/ImgurAttribution.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
import Utils from "./Utils";
|
||||
|
||||
export default class ImgurAttribution {
|
||||
|
||||
// MUST be private to prevent other people stealing this key! That I'll push this to github later on is not relevant
|
||||
private static ImgurApiKey = "7070e7167f0a25a"
|
||||
|
||||
/**
|
||||
* Download the attribution from a given URL
|
||||
*/
|
||||
public static async DownloadAttribution(url: string): Promise<{license: string, author: string}> {
|
||||
const hash = url.substr("https://i.imgur.com/".length).split(".jpg")[0]
|
||||
|
||||
const apiUrl = "https://api.imgur.com/3/image/" + hash
|
||||
const response = await Utils.DownloadJson(apiUrl, {
|
||||
Authorization: "Client-ID " + ImgurAttribution.ImgurApiKey,
|
||||
})
|
||||
|
||||
const descr: string = response.data.description ?? ""
|
||||
const data: any = {}
|
||||
for (const tag of descr.split("\n")) {
|
||||
const kv = tag.split(":")
|
||||
const k = kv[0]
|
||||
data[k] = kv[1]?.replace(/\r/g, "")
|
||||
}
|
||||
return data
|
||||
}
|
||||
}
|
||||
78
src/Mastodon.ts
Normal file
78
src/Mastodon.ts
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
import {login, LoginParams} from 'masto';
|
||||
import * as fs from "fs";
|
||||
import {stat} from "fs";
|
||||
|
||||
export interface LoginSettings {
|
||||
url: string,
|
||||
accessToken: string
|
||||
}
|
||||
|
||||
export interface CreateStatusParamsBase {
|
||||
/** ID of the status being replied to, if status is a reply */
|
||||
readonly inReplyToId?: string | null;
|
||||
/** Mark status and attached media as sensitive? */
|
||||
readonly sensitive?: boolean | null;
|
||||
/** Text to be shown as a warning or subject before the actual content. Statuses are generally collapsed behind this field. */
|
||||
readonly spoilerText?: string | null;
|
||||
/** Visibility of the posted status. Enumerable oneOf public, unlisted, private, direct. */
|
||||
readonly visibility?: "public" | "unlisted" | "private" | "direct" | null;
|
||||
/** ISO 639 language code for this status. */
|
||||
readonly language?: string | null;
|
||||
|
||||
readonly mediaIds?: readonly string[];
|
||||
|
||||
}
|
||||
export default class MastodonPoster {
|
||||
/**
|
||||
* The actual instance, see https://www.npmjs.com/package/mastodon
|
||||
* @private
|
||||
*/
|
||||
private readonly instance ;
|
||||
private _dryrun: boolean;
|
||||
private constructor(masto, dryrun: boolean) {
|
||||
this.instance = masto
|
||||
this._dryrun = dryrun;
|
||||
}
|
||||
|
||||
public async doStuff(){
|
||||
}
|
||||
|
||||
public async writeMessage(text: string, options?: CreateStatusParamsBase): Promise<{id: string}>{
|
||||
if(this._dryrun){
|
||||
console.log("Dryrun enabled; not posting:\n", text.split("\n").map(txt => " > "+txt).join("\n"))
|
||||
return {id: "some_id"}
|
||||
}
|
||||
const statusUpate = await this.instance.v1.statuses.create({
|
||||
visibility: 'public',
|
||||
...(options??{}),
|
||||
status: text
|
||||
})
|
||||
console.dir(statusUpate)
|
||||
console.log("Posted to", statusUpate.url)
|
||||
console.log(text.split("\n").map(txt => " > "+txt).join("\n"))
|
||||
return statusUpate
|
||||
}
|
||||
|
||||
public static async construct(settings: LoginParams & {dryrun?: boolean}) {
|
||||
return new MastodonPoster(await login(settings), settings.dryrun ?? false)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Uploads the image; returns the id of the image
|
||||
* @param path
|
||||
* @param description
|
||||
*/
|
||||
public async uploadImage(path: string, description: string): Promise<string> {
|
||||
if(this._dryrun){
|
||||
console.log("As dryrun is enabled: not uploading ", path)
|
||||
return "some_id"
|
||||
}
|
||||
console.log("Uploading", path)
|
||||
const mediaAttachment = await this.instance.v2.mediaAttachments.create({
|
||||
file: new Blob([fs.readFileSync(path)]),
|
||||
description
|
||||
})
|
||||
return mediaAttachment.id
|
||||
}
|
||||
}
|
||||
123
src/OsmCha.ts
Normal file
123
src/OsmCha.ts
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
import Utils from "./Utils";
|
||||
import * as fs from "fs";
|
||||
|
||||
|
||||
export interface ChangeSetData {
|
||||
id: number
|
||||
type: "Feature"
|
||||
geometry: {
|
||||
type: "Polygon"
|
||||
coordinates: [number, number][][]
|
||||
}
|
||||
properties: {
|
||||
check_user: null
|
||||
reasons: []
|
||||
tags: []
|
||||
features: []
|
||||
user: string
|
||||
uid: string
|
||||
editor: string
|
||||
comment: string
|
||||
comments_count: number
|
||||
source: string
|
||||
imagery_used: string
|
||||
date: string
|
||||
reviewed_features: []
|
||||
create: number
|
||||
modify: number
|
||||
delete: number
|
||||
area: number
|
||||
is_suspect: boolean
|
||||
harmful: any
|
||||
checked: boolean
|
||||
check_date: any
|
||||
host: string
|
||||
theme: string
|
||||
imagery: string
|
||||
language: string,
|
||||
tag_changes: Record<string, string[]>
|
||||
}
|
||||
}
|
||||
|
||||
export default class OsmCha {
|
||||
private readonly urlTemplate =
|
||||
"https://osmcha.org/api/v1/changesets/?date__gte={start_date}&date__lte={end_date}&page={page}&comment=%23mapcomplete&page_size=100"
|
||||
|
||||
private readonly _cachepath: string
|
||||
|
||||
constructor(options?: { cacheDir?: string }) {
|
||||
if (options?.cacheDir) {
|
||||
this._cachepath = options?.cacheDir + "/osmcha_"
|
||||
}
|
||||
}
|
||||
|
||||
public async DownloadStatsForDay(
|
||||
year: number,
|
||||
month: number,
|
||||
day: number
|
||||
): Promise<ChangeSetData[]> {
|
||||
const path = this._cachepath + "_" + year + "_" + Utils. TwoDigits(month) + "_" + Utils. TwoDigits(day) + ".json";
|
||||
if (fs.existsSync(path)) {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(path, "utf8")).features
|
||||
} catch (e) {
|
||||
fs.unlinkSync(path)
|
||||
}
|
||||
}
|
||||
let page = 1
|
||||
let allFeatures = []
|
||||
let endDay = new Date(year, month - 1 /* Zero-indexed: 0 = january*/, day + 1)
|
||||
let endDate = `${endDay.getFullYear()}-${Utils.TwoDigits(
|
||||
endDay.getMonth() + 1
|
||||
)}-${Utils.TwoDigits(endDay.getDate())}`
|
||||
let url = this.urlTemplate
|
||||
.replace(
|
||||
"{start_date}",
|
||||
year + "-" + Utils.TwoDigits(month) + "-" + Utils.TwoDigits(day)
|
||||
)
|
||||
.replace("{end_date}", endDate)
|
||||
.replace("{page}", "" + page)
|
||||
|
||||
let headers = {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
Referer:
|
||||
"https://osmcha.org/?filters=%7B%22date__gte%22%3A%5B%7B%22label%22%3A%222020-07-05%22%2C%22value%22%3A%222020-07-05%22%7D%5D%2C%22editor%22%3A%5B%7B%22label%22%3A%22mapcomplete%22%2C%22value%22%3A%22mapcomplete%22%7D%5D%7D",
|
||||
"Content-Type": "application/json",
|
||||
Authorization: "Token 6e422e2afedb79ef66573982012000281f03dc91",
|
||||
DNT: "1",
|
||||
Connection: "keep-alive",
|
||||
TE: "Trailers",
|
||||
Pragma: "no-cache",
|
||||
"Cache-Control": "no-cache",
|
||||
}
|
||||
|
||||
while (url) {
|
||||
const result = await Utils.DownloadJson(url, headers)
|
||||
page++
|
||||
allFeatures.push(...result.features)
|
||||
if (result.features === undefined) {
|
||||
console.log("ERROR", result)
|
||||
return
|
||||
}
|
||||
url = result.next
|
||||
}
|
||||
allFeatures = allFeatures.filter(f => f !== undefined && f !== null)
|
||||
allFeatures.forEach((f) => {
|
||||
f.properties = {...f.properties, ...f.properties.metadata}
|
||||
delete f.properties.metadata
|
||||
f.properties.id = f.id
|
||||
})
|
||||
|
||||
if (this._cachepath) {
|
||||
fs.writeFileSync(
|
||||
path,
|
||||
JSON.stringify({features: allFeatures}),
|
||||
"utf8"
|
||||
)
|
||||
}
|
||||
|
||||
return allFeatures
|
||||
}
|
||||
}
|
||||
81
src/OsmUserInfo.ts
Normal file
81
src/OsmUserInfo.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
import Utils from "./Utils";
|
||||
import * as fs from "fs";
|
||||
|
||||
export interface UserInfo {
|
||||
"id": number,
|
||||
"display_name": string,
|
||||
"account_created": string,
|
||||
"description": string,
|
||||
"contributor_terms": { "agreed": boolean, "pd": boolean },
|
||||
"img": { "href": string },
|
||||
"roles": [],
|
||||
"changesets": { "count": number },
|
||||
"traces": { "count": number },
|
||||
"blocks": { "received": { "count": number, "active": number } },
|
||||
}
|
||||
|
||||
export default class OsmUserInfo {
|
||||
private static readonly max_cache_age_seconds = 7 * 24 * 60 * 60;
|
||||
private readonly _userId: number;
|
||||
private readonly _backend: string;
|
||||
private _userData: UserInfo = undefined
|
||||
private readonly _cachingPath: string | undefined;
|
||||
|
||||
constructor(userId: number, options?:
|
||||
{
|
||||
osmBackend?: string,
|
||||
cacheDir?: string
|
||||
}) {
|
||||
if(userId === undefined || userId === null || Number.isNaN(userId)){
|
||||
throw new Error("Invalid userid: " + userId)
|
||||
}
|
||||
this._userId = userId;
|
||||
this._backend = options?.osmBackend ?? "https://www.openstreetmap.org/";
|
||||
if (options?.cacheDir) {
|
||||
this._cachingPath = options?.cacheDir + "/userinfo_" + userId + ".json"
|
||||
}
|
||||
if (!this._backend.endsWith("/")) {
|
||||
this._backend += "/"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public async getMeLinks(): Promise<string[]> {
|
||||
const userdata = await this.getUserInfo()
|
||||
const div = document.createElement("div")
|
||||
div.innerHTML = userdata.description
|
||||
const links = Array.from(div.getElementsByTagName("a"))
|
||||
const meLinks = links.filter(link => link.getAttribute("rel").split(" ").indexOf("me") >= 0)
|
||||
return meLinks.map(link => link.href.toString()) //*/
|
||||
}
|
||||
|
||||
public async getUserInfo(): Promise<UserInfo> {
|
||||
if (this._userData) {
|
||||
return this._userData
|
||||
}
|
||||
if (this._cachingPath !== undefined && fs.existsSync(this._cachingPath)) {
|
||||
const cacheCreatedTime: Date = fs.statSync(this._cachingPath).birthtime
|
||||
const cacheAgeInSeconds = (Date.now() - cacheCreatedTime.getTime()) / 1000
|
||||
if (cacheAgeInSeconds > OsmUserInfo.max_cache_age_seconds) {
|
||||
console.log("Cache is old, unlinking...")
|
||||
}else{
|
||||
|
||||
try {
|
||||
this._userData = JSON.parse(fs.readFileSync(this._cachingPath, "utf8"))
|
||||
return this._userData
|
||||
} catch (e) {
|
||||
fs.unlinkSync(this._cachingPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
const url = `${this._backend}api/0.6/user/${this._userId}.json`
|
||||
console.log("Looking up user info about ", this._userId)
|
||||
const res = await Utils.DownloadJson(url);
|
||||
this._userData = res.user
|
||||
if (this._cachingPath !== undefined) {
|
||||
fs.writeFileSync(this._cachingPath, JSON.stringify(this._userData), "utf8")
|
||||
}
|
||||
return this._userData
|
||||
}
|
||||
|
||||
}
|
||||
315
src/Postbuilder.ts
Normal file
315
src/Postbuilder.ts
Normal file
|
|
@ -0,0 +1,315 @@
|
|||
import Histogram from "./Histogram";
|
||||
import Utils from "./Utils";
|
||||
import {ChangeSetData} from "./OsmCha";
|
||||
import OsmUserInfo from "./OsmUserInfo";
|
||||
import Config from "./Config";
|
||||
import MastodonPoster from "./Mastodon";
|
||||
import ImgurAttribution from "./ImgurAttribution";
|
||||
|
||||
type ImageInfo = { image: string, changeset: ChangeSetData, tags: Record<string, string[]> }
|
||||
|
||||
export class Postbuilder {
|
||||
private static readonly metakeys = [
|
||||
"answer",
|
||||
"add-image",
|
||||
"move",
|
||||
"delete",
|
||||
"plantnet-ai-detection",
|
||||
"link-image"
|
||||
]
|
||||
private readonly _config: Config;
|
||||
private readonly _poster: MastodonPoster;
|
||||
private readonly _changesetsMade: ChangeSetData[];
|
||||
|
||||
constructor(config: Config, poster: MastodonPoster, changesetsMade: ChangeSetData[]) {
|
||||
this._poster = poster;
|
||||
this._config = config;
|
||||
// Ignore 'custom' themes, they can be confusing for uninitiated users and give ugly link + we don't endorse them
|
||||
this._changesetsMade = changesetsMade.filter(cs => !cs.properties.theme.startsWith("http://") && !cs.properties.theme.startsWith("https://"))
|
||||
;
|
||||
}
|
||||
|
||||
|
||||
getStatisticsFor(changesetsMade?: ChangeSetData[]): { total: number, addImage?: number, deleted: number, answered?: number, moved?: number, summaryText?: string } {
|
||||
|
||||
const stats: Record<string, number> = {}
|
||||
changesetsMade ??= this._changesetsMade
|
||||
let total = 0
|
||||
for (const changeset of changesetsMade) {
|
||||
for (const metakey of Postbuilder.metakeys) {
|
||||
if (changeset.properties[metakey]) {
|
||||
stats[metakey] = (stats[metakey] ?? 0) + changeset.properties[metakey]
|
||||
total += changeset.properties[metakey]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let overview: string[] = []
|
||||
const {answer, move} = stats
|
||||
const deleted = stats.delete
|
||||
const images = stats["add-image"]
|
||||
const plantnetDetected = stats["plantnet-ai-detection"]
|
||||
const linkedImages = stats["link-image"]
|
||||
if (answer) {
|
||||
if (answer == 1) {
|
||||
overview.push("answered one question")
|
||||
} else {
|
||||
overview.push("answered " + answer + " questions")
|
||||
}
|
||||
}
|
||||
if (images) {
|
||||
if (images == 1) {
|
||||
overview.push("uploaded one image")
|
||||
} else {
|
||||
overview.push("uploaded " + images + " images")
|
||||
}
|
||||
}
|
||||
|
||||
if (move) {
|
||||
if (move == 1) {
|
||||
overview.push("moved one point")
|
||||
} else {
|
||||
overview.push("moved " + move + " points")
|
||||
}
|
||||
}
|
||||
|
||||
if (deleted) {
|
||||
if (deleted == 1) {
|
||||
overview.push("delted one deleted")
|
||||
} else {
|
||||
overview.push("deleted " + deleted + " points")
|
||||
}
|
||||
}
|
||||
|
||||
if (plantnetDetected) {
|
||||
if (plantnetDetected == 1) {
|
||||
overview.push("detected one plant species with plantnet.org")
|
||||
} else {
|
||||
overview.push("detected " + plantnetDetected + " plant species with plantnet.org")
|
||||
}
|
||||
}
|
||||
|
||||
if (linkedImages) {
|
||||
if (linkedImages == 1) {
|
||||
overview.push("linked one linked")
|
||||
} else {
|
||||
overview.push("linked " + linkedImages + " images")
|
||||
}
|
||||
}
|
||||
|
||||
let summaryText = Utils.commasAnd(overview)
|
||||
|
||||
return {
|
||||
total: total,
|
||||
addImage: stats["add-image"],
|
||||
deleted: stats.delete,
|
||||
answered: stats.answer,
|
||||
moved: stats.move,
|
||||
summaryText
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
async createOverviewForContributor(uid: string, changesetsMade: ChangeSetData[]): Promise<string> {
|
||||
const userinfo = new OsmUserInfo(Number(uid), this._config)
|
||||
const inf = await userinfo.getUserInfo()
|
||||
const mastodonLinks = await userinfo.getMeLinks()
|
||||
|
||||
const themes = new Histogram(changesetsMade, cs => cs.properties.theme)
|
||||
|
||||
let username = inf.display_name
|
||||
if (mastodonLinks.length > 0) {
|
||||
const url = new URL(mastodonLinks[0])
|
||||
username = url.pathname.substring(1) + "@" + url.host
|
||||
}
|
||||
const statistics = this.getStatisticsFor(changesetsMade)
|
||||
|
||||
let thematicMaps = "maps " + Utils.commasAnd(themes.keys())
|
||||
if (themes.keys().length === 1) {
|
||||
thematicMaps = "map " + Utils.commasAnd(themes.keys())
|
||||
}
|
||||
|
||||
return username + " " + statistics.summaryText + " with the thematic " + thematicMaps
|
||||
}
|
||||
|
||||
async createOverviewForTheme(theme: string, changesetsMade: ChangeSetData[]): Promise<string> {
|
||||
const statistics = this.getStatisticsFor(changesetsMade)
|
||||
const contributorCount = new Set(changesetsMade.map(cs => cs.properties.uid)).size
|
||||
|
||||
let contribCountStr = contributorCount + " contributors"
|
||||
if (contributorCount == 1) {
|
||||
contribCountStr = "one contributor"
|
||||
}
|
||||
return `${contribCountStr} ${statistics.summaryText} on https://mapcomplete.osm.be/${theme}`
|
||||
}
|
||||
|
||||
public selectImages(images: ImageInfo[], targetCount: number = 4):
|
||||
ImageInfo[] {
|
||||
if (images.length <= targetCount) {
|
||||
return images
|
||||
}
|
||||
const themeBonus = {
|
||||
climbing: 1,
|
||||
rainbow_crossings: 1,
|
||||
binoculars: 2,
|
||||
artwork: 2,
|
||||
ghost_bikes: 1,
|
||||
trees: 2,
|
||||
bookcases: 1,
|
||||
playgrounds: 1,
|
||||
aed: 1,
|
||||
benches: 1,
|
||||
nature: 1
|
||||
}
|
||||
|
||||
const alreadyEncounteredUid = new Map<string, number>()
|
||||
|
||||
const result: ImageInfo[] = []
|
||||
for (let i = 0; i < targetCount; i++) {
|
||||
let bestImageScore: number = -999999999
|
||||
let bestImageOptions: ImageInfo[] = []
|
||||
|
||||
for (const image of images) {
|
||||
const props = image.changeset.properties
|
||||
const uid = ""+props.uid
|
||||
|
||||
|
||||
if (result.indexOf(image) >= 0) {
|
||||
continue
|
||||
}
|
||||
|
||||
let score = 0
|
||||
if (alreadyEncounteredUid.has(uid)) {
|
||||
score -= 100 * alreadyEncounteredUid.get(uid)
|
||||
}
|
||||
score += themeBonus[props.theme] ?? 0
|
||||
|
||||
if (score > bestImageScore) {
|
||||
bestImageScore = score
|
||||
bestImageOptions = [image]
|
||||
} else if (score === bestImageScore) {
|
||||
bestImageOptions.push(image)
|
||||
}
|
||||
}
|
||||
|
||||
const ri = Math.floor((bestImageOptions.length - 1) * Math.random())
|
||||
const randomBestImage = bestImageOptions[ri]
|
||||
result.push(randomBestImage)
|
||||
const theme = randomBestImage.changeset.properties.theme
|
||||
themeBonus[theme] = (themeBonus[theme] ?? 0) - 1
|
||||
const uid = randomBestImage.changeset.properties.uid
|
||||
alreadyEncounteredUid.set(uid, (alreadyEncounteredUid.get(uid) ?? 0) + 1)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
public async buildMessage(): Promise<void> {
|
||||
const changesets = this._changesetsMade
|
||||
const perContributor = new Histogram(changesets, cs => cs.properties.uid)
|
||||
|
||||
const topContributors = perContributor.sortedByCount({
|
||||
countMethod: cs => {
|
||||
let sum = 0
|
||||
for (const metakey of Postbuilder.metakeys) {
|
||||
if (cs.properties[metakey]) {
|
||||
sum += cs.properties[metakey]
|
||||
}
|
||||
}
|
||||
return sum
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
const totalStats = this.getStatisticsFor()
|
||||
const {totalImagesCreated, attachmentIds, imgAuthors, totalImageContributorCount} = await this.prepareImages(changesets, 12)
|
||||
|
||||
let toSend: string[] = [
|
||||
"Today, " + perContributor.keys().length + " different persons made " + totalStats.total + " changes to #OpenStreetMap using https://mapcomplete.osm.be .\n",
|
||||
]
|
||||
|
||||
for (let i = 0; i < this._config.postSettings.topContributorsNumberToShow - 1 && i < topContributors.length; i++) {
|
||||
const uid = topContributors[i].key
|
||||
const changesetsMade = perContributor.get(uid)
|
||||
try {
|
||||
const overview = await this.createOverviewForContributor(uid, changesetsMade)
|
||||
if (overview.length + toSend.join("\n").length > 500) {
|
||||
break
|
||||
}
|
||||
toSend.push(" - " + overview)
|
||||
} catch (e) {
|
||||
console.error("Could not add contributor " + uid, e)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
const firstPost = await this._poster.writeMessage(toSend.join("\n"), {mediaIds: attachmentIds.slice(0, 4)})
|
||||
toSend = []
|
||||
|
||||
const perTheme = new Histogram(changesets, cs => {
|
||||
return cs.properties.theme;
|
||||
})
|
||||
|
||||
const mostPopularThemes = perTheme.sortedByCount({
|
||||
countMethod: cs => this.getStatisticsFor([cs]).total,
|
||||
dropZeroValues: true
|
||||
})
|
||||
toSend.push("")
|
||||
for (let i = 0; i < this._config.postSettings.topThemesNumberToShow && i < mostPopularThemes.length; i++) {
|
||||
const theme = mostPopularThemes[i].key
|
||||
const changesetsMade = perTheme.get(theme)
|
||||
toSend.push(await this.createOverviewForTheme(theme, changesetsMade))
|
||||
}
|
||||
|
||||
const secondPost = await this._poster.writeMessage(toSend.join("\n"), {
|
||||
inReplyToId: firstPost["id"],
|
||||
mediaIds: attachmentIds.slice(4, 8)
|
||||
})
|
||||
|
||||
await this._poster.writeMessage([
|
||||
"In total, "+totalImageContributorCount+" different contributors uploaded "+totalImagesCreated+" images.\n",
|
||||
"Images in this thread are randomly selected from them and were made by: ",
|
||||
...Array.from(new Set<string>(imgAuthors)).map(auth => "- "+auth )
|
||||
|
||||
].join("\n"),{
|
||||
inReplyToId: secondPost["id"],
|
||||
mediaIds: attachmentIds.slice(8,12)
|
||||
})
|
||||
|
||||
|
||||
}
|
||||
|
||||
private async prepareImages(changesets: ChangeSetData[], targetCount: number = 4): Promise<{ imgAuthors: string[], attachmentIds: string[], totalImagesCreated: number, totalImageContributorCount: number }> {
|
||||
const withImage: ChangeSetData[] = changesets.filter(cs => cs.properties["add-image"] > 0)
|
||||
const totalImagesCreated = Utils.Sum(withImage.map(cs => cs.properties["add-image"]))
|
||||
|
||||
const images: ImageInfo[] = []
|
||||
for (const changeset of withImage) {
|
||||
const tags = changeset.properties.tag_changes
|
||||
for (const key in tags) {
|
||||
if (!key.startsWith("image")) {
|
||||
continue
|
||||
}
|
||||
const values: string[] = tags[key]
|
||||
for (const image of values) {
|
||||
images.push({image, changeset, tags})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const randomImages: ImageInfo[] = this.selectImages(images, targetCount)
|
||||
const attachmentIds: string[] = []
|
||||
const imgAuthors: string[] = []
|
||||
for (const randomImage of randomImages) {
|
||||
const id = randomImage.image.substring(randomImage.image.lastIndexOf("/") + 1)
|
||||
const path = this._config.cacheDir + "/image_" + id
|
||||
await Utils.DownloadBlob(randomImage.image, path)
|
||||
const attribution = await ImgurAttribution.DownloadAttribution(randomImage.image)
|
||||
const mediaId = await this._poster.uploadImage(path, "Image taken by " + attribution.author + ", available under " + attribution.license + ". It is made with the thematic map "+randomImage.changeset.properties.theme+" in changeset https://openstreetmap.org/changeset/"+randomImage.changeset.id)
|
||||
attachmentIds.push(mediaId)
|
||||
imgAuthors.push(attribution.author)
|
||||
}
|
||||
return {attachmentIds, imgAuthors, totalImagesCreated, totalImageContributorCount: new Set(changesets.map(cs => cs.properties.uid)).size}
|
||||
}
|
||||
}
|
||||
95
src/Utils.ts
Normal file
95
src/Utils.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import https from "https";
|
||||
import * as fs from "fs";
|
||||
|
||||
export default class Utils {
|
||||
public static async DownloadJson(url, headers?: any): Promise<any> {
|
||||
const data = await Utils.Download(url, headers)
|
||||
return JSON.parse(data.content)
|
||||
}
|
||||
|
||||
public static Sum(t: (number | undefined)[]) {
|
||||
let sum = 0;
|
||||
for (const n of t) {
|
||||
if (n === undefined || n === null || Number.isNaN(n)) {
|
||||
continue
|
||||
}
|
||||
sum += n
|
||||
}
|
||||
return sum
|
||||
}
|
||||
|
||||
public static TwoDigits(i: number) {
|
||||
if (i < 10) {
|
||||
return "0" + i
|
||||
}
|
||||
return "" + i
|
||||
}
|
||||
|
||||
public static Download(url, headers?: any): Promise<{ content: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
headers = headers ?? {}
|
||||
headers.accept = "application/json"
|
||||
const urlObj = new URL(url)
|
||||
https.get(
|
||||
{
|
||||
host: urlObj.host,
|
||||
path: urlObj.pathname + urlObj.search,
|
||||
|
||||
port: urlObj.port,
|
||||
headers: headers,
|
||||
},
|
||||
(res) => {
|
||||
const parts: string[] = []
|
||||
res.setEncoding("utf8")
|
||||
res.on("data", function (chunk) {
|
||||
// @ts-ignore
|
||||
parts.push(chunk)
|
||||
})
|
||||
|
||||
res.addListener("end", function () {
|
||||
resolve({content: parts.join("")})
|
||||
})
|
||||
}
|
||||
)
|
||||
} catch (e) {
|
||||
reject(e)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds commas and a single 'and' between the given items
|
||||
*
|
||||
* Utils.commasAnd(["A","B","C"]) // => "A, B and C"
|
||||
* Utils.commasAnd(["A"]) // => "A"
|
||||
* Utils.commasAnd([]) // => ""
|
||||
*/
|
||||
public static commasAnd(items: string[]){
|
||||
if(items.length === 1){
|
||||
return items[0]
|
||||
}
|
||||
if(items.length === 0){
|
||||
return ""
|
||||
}
|
||||
const last = items[items.length - 1 ]
|
||||
return items.slice(0, items.length - 1).join(", ") + " and "+last
|
||||
}
|
||||
|
||||
public static DownloadBlob(url: string, filepath: string): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
https.get(url, (res) => {
|
||||
if (res.statusCode === 200) {
|
||||
res.pipe(fs.createWriteStream(filepath))
|
||||
.on('error', reject)
|
||||
.once('close', () => resolve(filepath));
|
||||
} else {
|
||||
// Consume response data to free up memory
|
||||
res.resume();
|
||||
reject(new Error(`Request Failed With a Status Code: ${res.statusCode}`));
|
||||
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
44
src/index.ts
Normal file
44
src/index.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import * as fakedom from "fake-dom"
|
||||
import * as fs from "fs"
|
||||
import MastodonPoster from "./Mastodon";
|
||||
import OsmCha, {ChangeSetData} from "./OsmCha";
|
||||
import Config from "./Config";
|
||||
import * as configF from "../config/config.json"
|
||||
import {Postbuilder} from "./Postbuilder";
|
||||
|
||||
export class Main {
|
||||
|
||||
private readonly _config: Config;
|
||||
|
||||
constructor(config: Config) {
|
||||
this._config = config;
|
||||
}
|
||||
|
||||
async main() {
|
||||
|
||||
if (fakedom === undefined || window === undefined) {
|
||||
throw "FakeDom not initialized"
|
||||
}
|
||||
console.log("Starting...")
|
||||
if (this._config.cacheDir !== undefined && !fs.existsSync(this._config.cacheDir)) {
|
||||
fs.mkdirSync(this._config.cacheDir)
|
||||
console.log("Created the caching directory at", this._config.cacheDir)
|
||||
}
|
||||
|
||||
const poster = await MastodonPoster.construct(this._config.mastodonAuth)
|
||||
|
||||
console.log("Fetching recent changesets...")
|
||||
const osmcha = new OsmCha(this._config)
|
||||
const today = new Date()
|
||||
let changesets: ChangeSetData[] = await osmcha.DownloadStatsForDay(today.getUTCFullYear(), today.getUTCMonth() + 1, today.getUTCDate() - 1)
|
||||
|
||||
console.log("Building post...")
|
||||
await new Postbuilder(this._config, poster, changesets).buildMessage()
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
new Main(configF).main().then(_ => console.log("All done"))
|
||||
21
tsconfig.json
Normal file
21
tsconfig.json
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"experimentalDecorators": true,
|
||||
"emitDecoratorMetadata": true,
|
||||
"resolveJsonModule": true,
|
||||
"esModuleInterop": true,
|
||||
"module": "commonjs",
|
||||
"moduleResolution": "node",
|
||||
"target": "es2015",
|
||||
"noImplicitAny": false,
|
||||
"sourceMap": false,
|
||||
"outDir": "./lib",
|
||||
"types": [
|
||||
"node"
|
||||
],
|
||||
"lib": ["dom"]
|
||||
},
|
||||
"include": [
|
||||
"./src/**/*"
|
||||
]
|
||||
}
|
||||
72
tslint.json
Normal file
72
tslint.json
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
{
|
||||
"rules": {
|
||||
"class-name": false,
|
||||
"comment-format": [
|
||||
true
|
||||
],
|
||||
"curly": false,
|
||||
"eofline": false,
|
||||
"forin": false,
|
||||
"indent": [
|
||||
true,
|
||||
"spaces"
|
||||
],
|
||||
"label-position": true,
|
||||
"max-line-length": false,
|
||||
"member-access": false,
|
||||
"member-ordering": [
|
||||
true,
|
||||
"static-after-instance",
|
||||
"variables-before-functions"
|
||||
],
|
||||
"no-arg": true,
|
||||
"no-bitwise": false,
|
||||
"no-console": false,
|
||||
"no-construct": true,
|
||||
"no-debugger": true,
|
||||
"no-duplicate-variable": true,
|
||||
"no-empty": false,
|
||||
"no-eval": true,
|
||||
"no-inferrable-types": true,
|
||||
"no-shadowed-variable": true,
|
||||
"no-string-literal": false,
|
||||
"no-switch-case-fall-through": true,
|
||||
"no-trailing-whitespace": true,
|
||||
"no-unused-expression": true,
|
||||
"no-use-before-declare": false,
|
||||
"no-var-keyword": true,
|
||||
"object-literal-sort-keys": false,
|
||||
"one-line": [
|
||||
true,
|
||||
"check-open-brace",
|
||||
"check-catch",
|
||||
"check-else",
|
||||
"check-whitespace"
|
||||
],
|
||||
"quotemark": false,
|
||||
"radix": true,
|
||||
"semicolon": [
|
||||
"always"
|
||||
],
|
||||
"triple-equals": [],
|
||||
"typedef-whitespace": [
|
||||
true,
|
||||
{
|
||||
"call-signature": "nospace",
|
||||
"index-signature": "nospace",
|
||||
"parameter": "nospace",
|
||||
"property-declaration": "nospace",
|
||||
"variable-declaration": "nospace"
|
||||
}
|
||||
],
|
||||
"variable-name": false,
|
||||
"whitespace": [
|
||||
true,
|
||||
"check-branch",
|
||||
"check-decl",
|
||||
"check-operator",
|
||||
"check-separator",
|
||||
"check-type"
|
||||
]
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue