Themes(benches): add script that reads the openbenches-dataset from their git repo, add (experimental) maproulette import layer to benches

scripts/importscripts/openbenches.ts

@@ -0,0 +1,230 @@
+import Script from "../Script"
+import { promises as fs, writeFileSync } from "fs"
+import { Feature, Point } from "geojson"
+import { join } from "path"
+import sqlite3, { Database } from "sqlite3"
+import { open } from "sqlite"
+import ScriptUtils from "../ScriptUtils"
+import { Lists } from "../../src/Utils/Lists"
+
+/**
+ * Note:
+ * npm i sqlite sqlite3
+ * I didn't want this into the deps
+ *     "sqlite": "^5.1.1",
+ *     "sqlite3": "^5.1.7",
+ */
+interface Bench {
+    benchID: number,
+    latitude: number,
+    longitude: number,
+    address: string,
+    inscription: string,
+    description: string,
+    present: 0 | 1,
+    published: 0 | 1,
+    added: string,
+    userID: number
+}
+
+interface User {
+    name: string,
+    providerID: string,
+    provider: string,
+    userID: number
+}
+
+interface Tag {
+    tagID: number,
+    tagText: string
+}
+
+function mediaUrl(sha: string | { "sha1": string }): string {
+    if (sha["sha1"]) {
+        sha = sha["sha1"]
+    }
+    return `https://openbenches.org/image/${sha}.jpg`
+}
+
+class Openbenches extends Script {
+    private db: Database
+
+    constructor() {
+        super("Creates the OpenBenches dataset to upload to maproulette")
+    }
+
+    async buildDatabase(sqlDir: string, dbFile: string) {
+        const db = await open({
+            filename: dbFile,
+            driver: sqlite3.Database,
+        })
+
+        const files = await fs.readdir(sqlDir)
+        const sqlFiles = files.filter(f => f.endsWith(".sql"))
+
+        const skip = ["database.sql"]
+
+
+        const order = ["tags", "users", "tag_map", "media_types", "benches", "media"]
+
+        for (let file of order) {
+            console.log("Exec file", file)
+            file = "openbenc_benches_table_" + file + ".sql"
+            let content = await fs.readFile(join(sqlDir, file), "utf-8")
+            content = content.replaceAll("ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci", "")
+                .replaceAll("\\'", "''")
+            await db.exec(content)
+        }
+
+        await db.close()
+        console.log("DB has been seeded")
+    }
+
+    all<T>(query): Promise<T[]> {
+        return new Promise<T[]>((resolve, reject) => {
+            this.db.all(query, (err, rows) => {
+                if (err) {
+                    reject(err)
+                } else {
+                    resolve(<any>rows)
+                }
+            })
+        })
+    }
+
+    async loadDb(dbFile: string): Promise<Database> {
+        const db = await open({
+            filename: dbFile,
+            driver: sqlite3.Database,
+        })
+        console.log(db)
+        console.dir(db)
+        return <any>db.db
+
+    }
+
+    async createBenchInfo(benchWithUser: Bench & User, tags: string[]): Promise<Feature<Point>> {
+        const id = benchWithUser.benchID
+        const media = await this.all<{
+            sha1: string,
+            media_type: string
+        }>("SELECT * FROM media WHERE media.benchID = " + id)
+        const mediaBench = media.filter(m => m.media_type === "bench")
+        const mediaInscr = media.filter(m => m.media_type === "inscription")
+        const mediaView = media.filter(m => m.media_type === "view")
+
+        const properties = {
+            // added: benchWithUser.added,
+            "openbenches:id": id,
+            inscription: benchWithUser.inscription.replaceAll("\\r\\n", "\n"),
+            amenity: "bench",
+            // lastModifiedBy: benchWithUser.name,
+        }
+
+        let mediaMerged = Lists.dedup(mediaBench.concat(mediaInscr).map(m => mediaUrl(m)))
+        for (let i = 0; i < mediaMerged.length; i++) {
+            const m = mediaMerged[i]
+            if (i === 0) {
+                properties["image"] = m
+            } else {
+                properties["image:" + (i - 1)] = m
+
+            }
+        }
+
+        for (let i = 0; i < mediaView.length; i++) {
+            const m = mediaView[i]
+            if (i === 0) {
+                properties["image:view"] = mediaUrl(m)
+            } else {
+                properties["image:view:" + (i - 1)] = mediaUrl(m)
+
+            }
+        }
+
+        const tagsToProperties = {
+            "wooden": "material=wood",
+            "metal": "material=metal",
+            "indoors": "indoor=yes",
+            "stone": "material=stone",
+            "poem": "artwork=poem",
+            "statue": "artwork=statue",
+            "composite": "material=plastic",
+            /*"cat":"subject=cat",
+            "dog":"subject=dog" Not always a pet, sometimes also a 'dogwalker', someone mentioning their cat, ... */
+            // EMOJI: very broad category, basically that a little image is part of the 'inscription'. Should be handled by adding the emoji directly
+            // Twinned: basically, two people are remembered, often a couple -> inscription and/or subject handles this
+            // Picture: plaque has a little picture -> subset of plaque
+            // Famous: someone "famous" is remembered, although I don't know half of 'm. Too subjective for OSM
+            // FUnny: talk about subjective...
+        }
+
+        for (const tag of (tags ?? [])) {
+            const match = tagsToProperties[tag]
+            if (!match) {
+                continue
+            }
+            const [k, v] = match.split("=")
+            properties[k] = v
+            tags.splice(tags.indexOf(tag), 1)
+        }
+
+        return {
+            type: "Feature",
+            properties: { tags: JSON.stringify(properties) },
+            geometry: {
+                type: "Point",
+                coordinates: [benchWithUser.longitude, benchWithUser.latitude],
+            },
+        }
+    }
+
+
+    async main(args: string[]): Promise<void> {
+        const dbFile = "openbenches.sqlite"
+        let createTest = false
+
+        // rmSync(dbFile)
+        // await this.buildDatabase("/home/pietervdvn/git/openbenches.org/database", dbFile)
+        this.db = await this.loadDb(dbFile)
+
+        const tags = new Map<number, string>()
+        const tagRows = await this.all<Tag>("SELECT * FROM tags")
+        for (const tag of tagRows) {
+            tags.set(tag.tagID, tag.tagText)
+        }
+        const tagsOnBenches = new Map<number, string[]>()
+        const tagOnBench = await this.all<{ benchID: number, tagID: number }>("SELECT * from tag_map")
+        for (const tg of tagOnBench) {
+            const bench = tg.benchID
+            if (!tagsOnBenches.has(bench)) {
+                tagsOnBenches.set(bench, [])
+            }
+            tagsOnBenches.get(bench).push(tags.get(tg.tagID))
+        }
+
+        const r = await this.all<Bench & User>("SELECT * FROM benches INNER JOIN users ON benches.userID = users.userID")
+        const features: Feature<Point>[] = []
+        for (let i = 0; i < r.length; i++) {
+            const benchWithUser = r[i]
+            if (benchWithUser.present === 0 || benchWithUser.published === 0) {
+                continue
+            }
+            const tags = tagsOnBenches.get(benchWithUser.benchID)
+            if (i % 100 === 0) {
+                ScriptUtils.erasableLog(`Processing bench ${i}/${r.length} (${Math.round(100 * i / r.length)}%)         `)
+            }
+            features.push(await this.createBenchInfo(benchWithUser, tags))
+            if (createTest && features.length > 1000) {
+                break
+            }
+        }
+
+        writeFileSync(`openbenches_export${createTest ? "_test" : ""}.geojson`, JSON.stringify({
+            type: "FeatureCollection", features,
+        }, null, "  "), "utf-8")
+        //console.log(r)
+    }
+}
+
+new Openbenches().run()