import Script from "../Script" import { promises as fs, writeFileSync } from "fs" import { Feature, Point } from "geojson" import { join } from "path" import sqlite3, { Database } from "sqlite3" import { open } from "sqlite" import ScriptUtils from "../ScriptUtils" import { Lists } from "../../src/Utils/Lists" /** * Note: * npm i sqlite sqlite3 * I didn't want this into the deps * "sqlite": "^5.1.1", * "sqlite3": "^5.1.7", */ interface Bench { benchID: number, latitude: number, longitude: number, address: string, inscription: string, description: string, present: 0 | 1, published: 0 | 1, added: string, userID: number } interface User { name: string, providerID: string, provider: string, userID: number } interface Tag { tagID: number, tagText: string } function mediaUrl(sha: string | { "sha1": string }): string { if (sha["sha1"]) { sha = sha["sha1"] } return `https://openbenches.org/image/${sha}.jpg` } class Openbenches extends Script { private db: Database constructor() { super("Creates the OpenBenches dataset to upload to maproulette") } async buildDatabase(sqlDir: string, dbFile: string) { const db = await open({ filename: dbFile, driver: sqlite3.Database, }) const files = await fs.readdir(sqlDir) const sqlFiles = files.filter(f => f.endsWith(".sql")) const skip = ["database.sql"] const order = ["tags", "users", "tag_map", "media_types", "benches", "media"] for (let file of order) { console.log("Exec file", file) file = "openbenc_benches_table_" + file + ".sql" let content = await fs.readFile(join(sqlDir, file), "utf-8") content = content.replaceAll("ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci", "") .replaceAll("\\'", "''") await db.exec(content) } await db.close() console.log("DB has been seeded") } all(query): Promise { return new Promise((resolve, reject) => { this.db.all(query, (err, rows) => { if (err) { reject(err) } else { resolve(rows) } }) }) } async loadDb(dbFile: string): Promise { const db = await open({ filename: dbFile, driver: sqlite3.Database, }) console.log(db) console.dir(db) return db.db } async createBenchInfo(benchWithUser: Bench & User, tags: string[]): Promise> { const id = benchWithUser.benchID const media = await this.all<{ sha1: string, media_type: string }>("SELECT * FROM media WHERE media.benchID = " + id) const mediaBench = media.filter(m => m.media_type === "bench") const mediaInscr = media.filter(m => m.media_type === "inscription") const mediaView = media.filter(m => m.media_type === "view") const properties = { // added: benchWithUser.added, "openbenches:id": id, inscription: benchWithUser.inscription.replaceAll("\\r\\n", "\n"), amenity: "bench", // lastModifiedBy: benchWithUser.name, } let mediaMerged = Lists.dedup(mediaBench.concat(mediaInscr).map(m => mediaUrl(m))) for (let i = 0; i < mediaMerged.length; i++) { const m = mediaMerged[i] if (i === 0) { properties["image"] = m } else { properties["image:" + (i - 1)] = m } } for (let i = 0; i < mediaView.length; i++) { const m = mediaView[i] if (i === 0) { properties["image:view"] = mediaUrl(m) } else { properties["image:view:" + (i - 1)] = mediaUrl(m) } } const tagsToProperties = { "wooden": "material=wood", "metal": "material=metal", "indoors": "indoor=yes", "stone": "material=stone", "poem": "artwork=poem", "statue": "artwork=statue", "composite": "material=plastic", /*"cat":"subject=cat", "dog":"subject=dog" Not always a pet, sometimes also a 'dogwalker', someone mentioning their cat, ... */ // EMOJI: very broad category, basically that a little image is part of the 'inscription'. Should be handled by adding the emoji directly // Twinned: basically, two people are remembered, often a couple -> inscription and/or subject handles this // Picture: plaque has a little picture -> subset of plaque // Famous: someone "famous" is remembered, although I don't know half of 'm. Too subjective for OSM // FUnny: talk about subjective... } for (const tag of (tags ?? [])) { const match = tagsToProperties[tag] if (!match) { continue } const [k, v] = match.split("=") properties[k] = v tags.splice(tags.indexOf(tag), 1) } return { type: "Feature", properties: { tags: JSON.stringify(properties) }, geometry: { type: "Point", coordinates: [benchWithUser.longitude, benchWithUser.latitude], }, } } async main(args: string[]): Promise { const dbFile = "openbenches.sqlite" let createTest = false // rmSync(dbFile) // await this.buildDatabase("/home/pietervdvn/git/openbenches.org/database", dbFile) this.db = await this.loadDb(dbFile) const tags = new Map() const tagRows = await this.all("SELECT * FROM tags") for (const tag of tagRows) { tags.set(tag.tagID, tag.tagText) } const tagsOnBenches = new Map() const tagOnBench = await this.all<{ benchID: number, tagID: number }>("SELECT * from tag_map") for (const tg of tagOnBench) { const bench = tg.benchID if (!tagsOnBenches.has(bench)) { tagsOnBenches.set(bench, []) } tagsOnBenches.get(bench).push(tags.get(tg.tagID)) } const r = await this.all("SELECT * FROM benches INNER JOIN users ON benches.userID = users.userID") const features: Feature[] = [] for (let i = 0; i < r.length; i++) { const benchWithUser = r[i] if (benchWithUser.present === 0 || benchWithUser.published === 0) { continue } const tags = tagsOnBenches.get(benchWithUser.benchID) if (i % 100 === 0) { ScriptUtils.erasableLog(`Processing bench ${i}/${r.length} (${Math.round(100 * i / r.length)}%) `) } features.push(await this.createBenchInfo(benchWithUser, tags)) if (createTest && features.length > 1000) { break } } writeFileSync(`openbenches_export${createTest ? "_test" : ""}.geojson`, JSON.stringify({ type: "FeatureCollection", features, }, null, " "), "utf-8") //console.log(r) } } new Openbenches().run()