MapComplete/scripts/importscripts/openbenches.ts

import Script from "../Script"
import { promises as fs, writeFileSync } from "fs"
import { Feature, Point } from "geojson"
import { join } from "path"
import sqlite3, { Database } from "sqlite3"
import { open } from "sqlite"
import ScriptUtils from "../ScriptUtils"
import { Lists } from "../../src/Utils/Lists"

/**
 * Note:
 * npm i sqlite sqlite3
 * I didn't want this into the deps
 *     "sqlite": "^5.1.1",
 *     "sqlite3": "^5.1.7",
 */
interface Bench {
    benchID: number,
    latitude: number,
    longitude: number,
    address: string,
    inscription: string,
    description: string,
    present: 0 | 1,
    published: 0 | 1,
    added: string,
    userID: number
}

interface User {
    name: string,
    providerID: string,
    provider: string,
    userID: number
}

interface Tag {
    tagID: number,
    tagText: string
}

function mediaUrl(sha: string | { "sha1": string }): string {
    if (sha["sha1"]) {
        sha = sha["sha1"]
    }
    return `https://openbenches.org/image/${sha}.jpg`
}

class Openbenches extends Script {
    private db: Database

    constructor() {
        super("Creates the OpenBenches dataset to upload to maproulette")
    }

    async buildDatabase(sqlDir: string, dbFile: string) {
        const db = await open({
            filename: dbFile,
            driver: sqlite3.Database,
        })

        const files = await fs.readdir(sqlDir)
        const sqlFiles = files.filter(f => f.endsWith(".sql"))

        const skip = ["database.sql"]


        const order = ["tags", "users", "tag_map", "media_types", "benches", "media"]

        for (let file of order) {
            console.log("Exec file", file)
            file = "openbenc_benches_table_" + file + ".sql"
            let content = await fs.readFile(join(sqlDir, file), "utf-8")
            content = content.replaceAll("ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci", "")
                .replaceAll("\\'", "''")
            await db.exec(content)
        }

        await db.close()
        console.log("DB has been seeded")
    }

    all<T>(query): Promise<T[]> {
        return new Promise<T[]>((resolve, reject) => {
            this.db.all(query, (err, rows) => {
                if (err) {
                    reject(err)
                } else {
                    resolve(<any>rows)
                }
            })
        })
    }

    async loadDb(dbFile: string): Promise<Database> {
        const db = await open({
            filename: dbFile,
            driver: sqlite3.Database,
        })
        console.log(db)
        console.dir(db)
        return <any>db.db

    }

    async createBenchInfo(benchWithUser: Bench & User, tags: string[]): Promise<Feature<Point>> {
        const id = benchWithUser.benchID
        const media = await this.all<{
            sha1: string,
            media_type: string
        }>("SELECT * FROM media WHERE media.benchID = " + id)
        const mediaBench = media.filter(m => m.media_type === "bench")
        const mediaInscr = media.filter(m => m.media_type === "inscription")
        const mediaView = media.filter(m => m.media_type === "view")

        const properties = {
            // added: benchWithUser.added,
            "openbenches:id": id,
            inscription: benchWithUser.inscription.replaceAll("\\r\\n", "\n"),
            amenity: "bench",
            // lastModifiedBy: benchWithUser.name,
        }

        let mediaMerged = Lists.dedup(mediaBench.concat(mediaInscr).map(m => mediaUrl(m)))
        for (let i = 0; i < mediaMerged.length; i++) {
            const m = mediaMerged[i]
            if (i === 0) {
                properties["image"] = m
            } else {
                properties["image:" + (i - 1)] = m

            }
        }

        for (let i = 0; i < mediaView.length; i++) {
            const m = mediaView[i]
            if (i === 0) {
                properties["image:view"] = mediaUrl(m)
            } else {
                properties["image:view:" + (i - 1)] = mediaUrl(m)

            }
        }

        const tagsToProperties = {
            "wooden": "material=wood",
            "metal": "material=metal",
            "indoors": "indoor=yes",
            "stone": "material=stone",
            "poem": "artwork=poem",
            "statue": "artwork=statue",
            "composite": "material=plastic",
            /*"cat":"subject=cat",
            "dog":"subject=dog" Not always a pet, sometimes also a 'dogwalker', someone mentioning their cat, ... */
            // EMOJI: very broad category, basically that a little image is part of the 'inscription'. Should be handled by adding the emoji directly
            // Twinned: basically, two people are remembered, often a couple -> inscription and/or subject handles this
            // Picture: plaque has a little picture -> subset of plaque
            // Famous: someone "famous" is remembered, although I don't know half of 'm. Too subjective for OSM
            // FUnny: talk about subjective...
        }

        for (const tag of (tags ?? [])) {
            const match = tagsToProperties[tag]
            if (!match) {
                continue
            }
            const [k, v] = match.split("=")
            properties[k] = v
            tags.splice(tags.indexOf(tag), 1)
        }

        return {
            type: "Feature",
            properties: { tags: JSON.stringify(properties) },
            geometry: {
                type: "Point",
                coordinates: [benchWithUser.longitude, benchWithUser.latitude],
            },
        }
    }


    async main(args: string[]): Promise<void> {
        const dbFile = "openbenches.sqlite"
        let createTest = false

        // rmSync(dbFile)
        // await this.buildDatabase("/home/pietervdvn/git/openbenches.org/database", dbFile)
        this.db = await this.loadDb(dbFile)

        const tags = new Map<number, string>()
        const tagRows = await this.all<Tag>("SELECT * FROM tags")
        for (const tag of tagRows) {
            tags.set(tag.tagID, tag.tagText)
        }
        const tagsOnBenches = new Map<number, string[]>()
        const tagOnBench = await this.all<{ benchID: number, tagID: number }>("SELECT * from tag_map")
        for (const tg of tagOnBench) {
            const bench = tg.benchID
            if (!tagsOnBenches.has(bench)) {
                tagsOnBenches.set(bench, [])
            }
            tagsOnBenches.get(bench).push(tags.get(tg.tagID))
        }

        const r = await this.all<Bench & User>("SELECT * FROM benches INNER JOIN users ON benches.userID = users.userID")
        const features: Feature<Point>[] = []
        for (let i = 0; i < r.length; i++) {
            const benchWithUser = r[i]
            if (benchWithUser.present === 0 || benchWithUser.published === 0) {
                continue
            }
            const tags = tagsOnBenches.get(benchWithUser.benchID)
            if (i % 100 === 0) {
                ScriptUtils.erasableLog(`Processing bench ${i}/${r.length} (${Math.round(100 * i / r.length)}%)         `)
            }
            features.push(await this.createBenchInfo(benchWithUser, tags))
            if (createTest && features.length > 1000) {
                break
            }
        }

        writeFileSync(`openbenches_export${createTest ? "_test" : ""}.geojson`, JSON.stringify({
            type: "FeatureCollection", features,
        }, null, "  "), "utf-8")
        //console.log(r)
    }
}

new Openbenches().run()