MapComplete/scripts/importscripts/openbenches.ts

import Script from "../Script"
import { existsSync, promises as fs, readFileSync, writeFile, writeFileSync } from "fs"
import { Feature, FeatureCollection, Point } from "geojson"
import { join } from "path"
import sqlite3, { Database } from "sqlite3"
import { open } from "sqlite"
import { Lists } from "../../src/Utils/Lists"
import { Overpass } from "../../src/Logic/Osm/Overpass"
import { TagUtils } from "../../src/Logic/Tags/TagUtils"
import Constants from "../../src/Models/Constants"
import { BBox } from "../../src/Logic/BBox"
import LinkImageAction from "../../src/Logic/Osm/Actions/LinkImageAction"
import OsmChangeAction from "../../src/Logic/Osm/Actions/OsmChangeAction"
import ChangeTagAction from "../../src/Logic/Osm/Actions/ChangeTagAction"
import { Tag as OsmTag } from "../../src/Logic/Tags/Tag"
import { Changes } from "../../src/Logic/Osm/Changes"
import ScriptUtils from "../ScriptUtils"

/**
 * Note:
 * npm i sqlite sqlite3
 * I didn't want this into the deps
 *     "sqlite": "^5.1.1",
 *     "sqlite3": "^5.1.7",
 */
interface Bench {
    benchID: number,
    latitude: number,
    longitude: number,
    address: string,
    inscription: string,
    description: string,
    present: 0 | 1,
    published: 0 | 1,
    /* time of creation (or possibly last edit?) */
    added: string,
    userID: number
}

interface User {
    name: string,
    providerID: string,
    provider: string,
    userID: number
}

interface Tag {
    tagID: number,
    tagText: string
}

function mediaUrl(sha: string | { "sha1": string }): string {
    if (sha["sha1"]) {
        sha = sha["sha1"]
    }
    return `https://openbenches.org/image/${sha}.jpg`
}

class Openbenches extends Script {
    private db: Database

    constructor() {
        super("Creates the OpenBenches dataset to upload to maproulette")
    }

    async buildDatabase(sqlDir: string, dbFile: string) {
        const db = await open({
            filename: dbFile,
            driver: sqlite3.Database,
        })

        const files = await fs.readdir(sqlDir)
        const sqlFiles = files.filter(f => f.endsWith(".sql"))

        const skip = ["database.sql"]


        const order = ["tags", "users", "tag_map", "media_types", "benches", "media"]

        for (let file of order) {
            console.log("Exec file", file)
            file = "openbenc_benches_table_" + file + ".sql"
            let content = await fs.readFile(join(sqlDir, file), "utf-8")
            content = content.replaceAll("ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci", "")
                .replaceAll("\\'", "''")
            await db.exec(content)
        }

        await db.close()
        console.log("DB has been seeded")
    }

    all<T>(query): Promise<T[]> {
        return new Promise<T[]>((resolve, reject) => {
            this.db.all(query, (err, rows) => {
                if (err) {
                    reject(err)
                } else {
                    resolve(<any>rows)
                }
            })
        })
    }

    async loadDb(dbFile: string): Promise<Database> {
        const db = await open({
            filename: dbFile,
            driver: sqlite3.Database,
        })
        return <any>db.db

    }

    async createBenchInfo(benchWithUser: Bench & User, tags: string[]): Promise<Feature<Point>> {
        const id = benchWithUser.benchID
        const media = await this.all<{
            sha1: string,
            media_type: string
        }>("SELECT * FROM media WHERE media.benchID = " + id)
        const mediaBench = media.filter(m => m.media_type === "bench")
        const mediaInscr = media.filter(m => m.media_type === "inscription")
        const mediaView = media.filter(m => m.media_type === "view")

        const properties = {
            lastModifiedTime: benchWithUser.added,
            "openbenches:id": id,
            inscription: benchWithUser.inscription.replaceAll("\\r\\n", "\n"),
            amenity: "bench",
            lastModifiedBy: benchWithUser.name,
        }

        let mediaMerged = Lists.dedup(mediaBench.concat(mediaInscr).map(m => mediaUrl(m)))
        for (let i = 0; i < mediaMerged.length; i++) {
            const m = mediaMerged[i]
            if (i === 0) {
                properties["image"] = m
            } else {
                properties["image:" + (i - 1)] = m

            }
        }

        for (let i = 0; i < mediaView.length; i++) {
            const m = mediaView[i]
            if (i === 0) {
                properties["image:view"] = mediaUrl(m)
            } else {
                properties["image:view:" + (i - 1)] = mediaUrl(m)

            }
        }

        const tagsToProperties = {
            "wooden": "material=wood",
            "metal": "material=metal",
            "indoors": "indoor=yes",
            "stone": "material=stone",
            "poem": "artwork=poem",
            "statue": "artwork=statue",
            "composite": "material=plastic",
            /*"cat":"subject=cat",
            "dog":"subject=dog" Not always a pet, sometimes also a 'dogwalker', someone mentioning their cat, ... */
            // EMOJI: very broad category, basically that a little image is part of the 'inscription'. Should be handled by adding the emoji directly
            // Twinned: basically, two people are remembered, often a couple -> inscription and/or subject handles this
            // Picture: plaque has a little picture -> subset of plaque
            // Famous: someone "famous" is remembered, although I don't know half of 'm. Too subjective for OSM
            // FUnny: talk about subjective...
        }

        for (const tag of (tags ?? [])) {
            const match = tagsToProperties[tag]
            if (!match) {
                continue
            }
            const [k, v] = match.split("=")
            properties[k] = v
            tags.splice(tags.indexOf(tag), 1)
        }

        return {
            type: "Feature",
            properties,
            geometry: {
                type: "Point",
                coordinates: [benchWithUser.longitude, benchWithUser.latitude],
            },
        }
    }

    async getAlreadyImported(): Promise<FeatureCollection> {
        const alreadyImportedPath = "openbenches_linked_in_osm.geojson"
        if (!existsSync(alreadyImportedPath)) {
            const overpass = new Overpass(Constants.defaultOverpassUrls[0], TagUtils.Tag("openbenches:id~*"))
            const dataAndDate = await overpass.queryGeoJson(BBox.global)
            const data = dataAndDate[0]
            writeFileSync(alreadyImportedPath, JSON.stringify(data), "utf-8")
            return data
        }

        return JSON.parse(readFileSync(alreadyImportedPath, "utf-8"))
    }

    async conflate(osmData: FeatureCollection, openBenchesData: FeatureCollection) {
        const dict: Map<string, Feature> = new Map()
        for (const bench of openBenchesData.features) {
            const obid = bench.properties["openbenches:id"]
            dict.set("" + obid, bench)
        }
        const changes: OsmChangeAction[] = []
        for (const bench of osmData.features) {
            const obid = bench.properties["openbenches:id"]
            const ob = dict.get(obid)
            if (!ob) {
                console.log("No match found for", obid, "https://osm.org/" + bench.properties.id)
                continue
            }
            // console.log(`https://osm.org/${bench.properties.id} = https://openbenches.org/bench/${obid}`)
            for (const key in ob.properties) {
                if (key.startsWith("lastModified")) {
                    continue
                }
                if (key.startsWith("image")) {
                    const imgValue = ob.properties[key]
                    if (Object.values(bench.properties).some(v => v === imgValue || (v + ".jpg") === imgValue)) {
                        continue
                    }
                    let ikey = "image"
                    let i = -1
                    while (bench.properties[ikey]) {
                        i++
                        ikey = "image:" + i
                    }
                    const li = new ChangeTagAction(bench.properties.id, new OsmTag(ikey, imgValue), bench.properties, {
                        theme: "openbenches",
                        changeType: "link-image",
                    })
                    changes.push(li)
                    bench.properties[ikey] = imgValue
                    console.log(` + ${ikey}=${imgValue}`)
                } else if (!bench.properties[key]) {
                    const v = ob.properties[key]
                    if(v.length >= 255){
                        console.log("Text too long:", v.replaceAll("\n"," "))
                        continue
                    }
                    changes.push(new ChangeTagAction(
                        bench.properties.id,
                        new OsmTag(key, v),
                        bench.properties,
                        {
                            theme: "openbenches",
                            changeType: "answer",
                        },
                    ))
                    console.log(` - ${key}=${ob.properties[key].replaceAll("\n", " ")}`)
                }
            }
        }
        const xml = await Changes.createChangesetXMLForJosm(changes)
        writeFileSync("attributes_import.osc",xml, "utf-8")
    }

    async main(args: string[]): Promise<void> {
        const dbFile = "openbenches.sqlite"
        let createTest = false

        const osmData = await this.getAlreadyImported()

        // rmSync(dbFile)
        if(!existsSync(dbFile)){
            console.log("No database file found at "+dbFile+", recreating the database")
            await this.buildDatabase("/home/pietervdvn/git/openbenches.org/database", dbFile)
        }

        this.db = await this.loadDb(dbFile)

        const tags = new Map<number, string>()
        const tagRows = await this.all<Tag>("SELECT * FROM tags")
        for (const tag of tagRows) {
            tags.set(tag.tagID, tag.tagText)
        }
        const tagsOnBenches = new Map<number, string[]>()
        const tagOnBench = await this.all<{ benchID: number, tagID: number }>("SELECT * from tag_map")
        for (const tg of tagOnBench) {
            const bench = tg.benchID
            if (!tagsOnBenches.has(bench)) {
                tagsOnBenches.set(bench, [])
            }
            tagsOnBenches.get(bench).push(tags.get(tg.tagID))
        }

        const r = await this.all<Bench & User>("SELECT * FROM benches INNER JOIN users ON benches.userID = users.userID")
        const features: Feature<Point>[] = []
        for (let i = 0; i < r.length; i++) {
            const benchWithUser = r[i]
            if (benchWithUser.present === 0 || benchWithUser.published === 0) {
                continue
            }
            const tags = tagsOnBenches.get(benchWithUser.benchID)
            if (i % 100 === 0) {
                ScriptUtils.erasableLog(`Processing bench ${i}/${r.length} (${Math.round(100 * i / r.length)}%)         `)
            }
            features.push(await this.createBenchInfo(benchWithUser, tags))
            if (createTest && features.length > 1000) {
                break
            }
        }

        writeFileSync(`openbenches_export_josm_${createTest ? "_test" : ""}.geojson`, JSON.stringify({
            type: "FeatureCollection", features,
        }, null, "  "), "utf-8")

        const maproulette = features.map(f => {
            const properties = {tags: JSON.stringify(f.properties)}
            properties["id"] = "openbenches/"+f.properties["openbenches:id"]
            return {...f, properties}
        })
        writeFileSync(`openbenches_export_maproulette${createTest ? "_test" : ""}.geojson`, JSON.stringify({
            type: "FeatureCollection", features: maproulette,
        }, null, "  "), "utf-8")


        const openBenches = JSON.parse(readFileSync("openbenches_export_josm_.geojson", "utf-8"))
       // await this.conflate(osmData, openBenches)

    }
}

new Openbenches().run()