import Script from "../Script" import { existsSync, promises as fs, readFileSync, writeFile, writeFileSync } from "fs" import { Feature, FeatureCollection, Point, Polygon } from "geojson" import { join } from "path" import sqlite3, { Database } from "sqlite3" import { open } from "sqlite" import { Lists } from "../../src/Utils/Lists" import { Overpass } from "../../src/Logic/Osm/Overpass" import { TagUtils } from "../../src/Logic/Tags/TagUtils" import Constants from "../../src/Models/Constants" import { BBox } from "../../src/Logic/BBox" import LinkImageAction from "../../src/Logic/Osm/Actions/LinkImageAction" import OsmChangeAction from "../../src/Logic/Osm/Actions/OsmChangeAction" import ChangeTagAction from "../../src/Logic/Osm/Actions/ChangeTagAction" import { Tag as OsmTag } from "../../src/Logic/Tags/Tag" import { Changes } from "../../src/Logic/Osm/Changes" import ScriptUtils from "../ScriptUtils" import { GeoOperations } from "../../src/Logic/GeoOperations" /** * Note: * npm i sqlite sqlite3 * I didn't want this into the deps * "sqlite": "^5.1.1", * "sqlite3": "^5.1.7", */ interface Bench { benchID: number, latitude: number, longitude: number, address: string, inscription: string, description: string, present: 0 | 1, published: 0 | 1, /* time of creation (or possibly last edit?) */ added: string, userID: number } interface User { name: string, providerID: string, provider: string, userID: number } interface Tag { tagID: number, tagText: string } function mediaUrl(sha: string | { "sha1": string }): string { if (sha["sha1"]) { sha = sha["sha1"] } return `https://openbenches.org/image/${sha}.jpg` } const uk: Feature = { "type": "Feature", "properties": {}, "geometry": { "coordinates": [ [ [ 3.139397666817615, 53.112746745001914 ], [ 0.12546232547020963, 61.34289409315957 ], [ -5.193638926198332, 60.3858935023425 ], [ -12.316831332595541, 56.76308878364702 ], [ -12.586640816376246, 51.076733390490034 ], [ -3.6443836396576046, 49.4256703574342 ], [ 1.0194660085441853, 50.442813369706585 ], [ 3.139397666817615, 53.112746745001914 ] ] ], "type": "Polygon" } } const us : Feature = { "type": "Feature", "properties": {}, "geometry": { "coordinates": [ [ [ -171.55472370762342, 71.44263911390138 ], [ -171.31347027402668, 33.24735774004321 ], [ -105.9804086342826, -3.5292610992716362 ], [ -57.00596161415962, 15.805666337324794 ], [ -32.880618254493015, 49.584578264365916 ], [ -47.35582427029317, 72.85409976292118 ], [ -101.60890406091582, 79.0557752859543 ], [ -171.55472370762342, 71.44263911390138 ] ] ], "type": "Polygon" } } const australia: Feature = { "type": "Feature", "properties": {}, "geometry": { "coordinates": [ [ [ 177.6309142850211, -48.72845301037672 ], [ 177.6309142850211, -8.050870320392335 ], [ 107.59695622498174, -8.050870320392335 ], [ 107.59695622498174, -48.72845301037672 ], [ 177.6309142850211, -48.72845301037672 ] ] ], "type": "Polygon" } } const areas = {uk, us, australia} class Openbenches extends Script { private db: Database constructor() { super("Creates the OpenBenches dataset to upload to maproulette") } async buildDatabase(sqlDir: string, dbFile: string) { const db = await open({ filename: dbFile, driver: sqlite3.Database, }) const files = await fs.readdir(sqlDir) const sqlFiles = files.filter(f => f.endsWith(".sql")) const skip = ["database.sql"] const order = ["tags", "users", "tag_map", "media_types", "benches", "media"] for (let file of order) { console.log("Exec file", file) file = "openbenc_benches_table_" + file + ".sql" let content = await fs.readFile(join(sqlDir, file), "utf-8") content = content.replaceAll("ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci", "") .replaceAll("\\'", "''") await db.exec(content) } await db.close() console.log("DB has been seeded") } all(query): Promise { return new Promise((resolve, reject) => { this.db.all(query, (err, rows) => { if (err) { reject(err) } else { resolve(rows) } }) }) } async loadDb(dbFile: string): Promise { const db = await open({ filename: dbFile, driver: sqlite3.Database, }) return db.db } async createBenchInfo(benchWithUser: Bench & User, tags: string[]): Promise> { const id = benchWithUser.benchID const media = await this.all<{ sha1: string, media_type: string }>("SELECT * FROM media WHERE media.benchID = " + id) const mediaBench = media.filter(m => m.media_type === "bench") const mediaInscr = media.filter(m => m.media_type === "inscription") const mediaView = media.filter(m => m.media_type === "view") const properties = { lastModifiedTime: benchWithUser.added, "openbenches:id": id, inscription: benchWithUser.inscription.replaceAll("\\r\\n", "\n"), amenity: "bench", lastModifiedBy: benchWithUser.name, } let mediaMerged = Lists.dedup(mediaBench.concat(mediaInscr).map(m => mediaUrl(m))) for (let i = 0; i < mediaMerged.length; i++) { const m = mediaMerged[i] if (i === 0) { properties["image"] = m } else { properties["image:" + (i - 1)] = m } } for (let i = 0; i < mediaView.length; i++) { const m = mediaView[i] if (i === 0) { properties["image:view"] = mediaUrl(m) } else { properties["image:view:" + (i - 1)] = mediaUrl(m) } } const tagsToProperties = { "wooden": "material=wood", "metal": "material=metal", "indoors": "indoor=yes", "stone": "material=stone", "poem": "artwork=poem", "statue": "artwork=statue", "composite": "material=plastic", /*"cat":"subject=cat", "dog":"subject=dog" Not always a pet, sometimes also a 'dogwalker', someone mentioning their cat, ... */ // EMOJI: very broad category, basically that a little image is part of the 'inscription'. Should be handled by adding the emoji directly // Twinned: basically, two people are remembered, often a couple -> inscription and/or subject handles this // Picture: plaque has a little picture -> subset of plaque // Famous: someone "famous" is remembered, although I don't know half of 'm. Too subjective for OSM // FUnny: talk about subjective... } for (const tag of (tags ?? [])) { const match = tagsToProperties[tag] if (!match) { continue } const [k, v] = match.split("=") properties[k] = v tags.splice(tags.indexOf(tag), 1) } return { type: "Feature", properties, geometry: { type: "Point", coordinates: [benchWithUser.longitude, benchWithUser.latitude], }, } } async getAlreadyImported(): Promise { const alreadyImportedPath = "openbenches_linked_in_osm.geojson" if (!existsSync(alreadyImportedPath)) { const overpass = new Overpass(Constants.defaultOverpassUrls[0], TagUtils.Tag("openbenches:id~*")) const dataAndDate = await overpass.queryGeoJson(BBox.global) const data = dataAndDate[0] writeFileSync(alreadyImportedPath, JSON.stringify(data), "utf-8") return data } return JSON.parse(readFileSync(alreadyImportedPath, "utf-8")) } async conflate(osmData: Feature[], openBenchesData: FeatureCollection, area: string = "") { const dict: Map = new Map() for (const bench of openBenchesData.features) { const obid = bench.properties["openbenches:id"] dict.set("" + obid, bench) } const changes: OsmChangeAction[] = [] for (const bench of osmData) { const obid = bench.properties["openbenches:id"] const ob = dict.get(obid) if (!ob) { console.log("No match found for", obid, "https://osm.org/" + bench.properties.id) continue } // console.log(`https://osm.org/${bench.properties.id} = https://openbenches.org/bench/${obid}`) for (const key in ob.properties) { if (key.startsWith("lastModified")) { continue } if (key.startsWith("image")) { const imgValue = ob.properties[key] if (Object.values(bench.properties).some(v => v === imgValue || (v + ".jpg") === imgValue)) { continue } let ikey = "image" let i = -1 while (bench.properties[ikey]) { i++ ikey = "image:" + i } const li = new ChangeTagAction(bench.properties.id, new OsmTag(ikey, imgValue), bench.properties, { theme: "openbenches", changeType: "link-image", }) changes.push(li) bench.properties[ikey] = imgValue console.log(` + ${ikey}=${imgValue}`) } else if (!bench.properties[key]) { const v = ob.properties[key] if(v.length >= 255){ console.log("Text too long:", v.replaceAll("\n"," ")) continue } changes.push(new ChangeTagAction( bench.properties.id, new OsmTag(key, v), bench.properties, { theme: "openbenches", changeType: "answer", }, )) console.log(` - ${key}=${ob.properties[key].replaceAll("\n", " ")}`) } } } if(changes.length === 0){ return } const xml = await Changes.createChangesetXMLForJosm(changes) writeFileSync(`attributes_import${area}.osc`,xml, "utf-8") } async main(args: string[]): Promise { const dbFile = "openbenches.sqlite" let createTest = false const osmData = await this.getAlreadyImported() // rmSync(dbFile) if(!existsSync(dbFile)){ console.log("No database file found at "+dbFile+", recreating the database") await this.buildDatabase("/home/pietervdvn/git/openbenches.org/database", dbFile) } this.db = await this.loadDb(dbFile) const tags = new Map() const tagRows = await this.all("SELECT * FROM tags") for (const tag of tagRows) { tags.set(tag.tagID, tag.tagText) } const tagsOnBenches = new Map() const tagOnBench = await this.all<{ benchID: number, tagID: number }>("SELECT * from tag_map") for (const tg of tagOnBench) { const bench = tg.benchID if (!tagsOnBenches.has(bench)) { tagsOnBenches.set(bench, []) } tagsOnBenches.get(bench).push(tags.get(tg.tagID)) } const alreadyLinked = new Set(osmData.features.map(f => f.properties["openbenches:id"])) const r = await this.all("SELECT * FROM benches INNER JOIN users ON benches.userID = users.userID") const features: Feature[] = [] for (let i = 0; i < r.length; i++) { const benchWithUser = r[i] if (benchWithUser.present === 0 || benchWithUser.published === 0) { continue } const tags = tagsOnBenches.get(benchWithUser.benchID) if (i % 100 === 0) { ScriptUtils.erasableLog(`Processing bench ${i}/${r.length} (${Math.round(100 * i / r.length)}%) `) } features.push(await this.createBenchInfo(benchWithUser, tags)) if (createTest && features.length > 1000) { break } } writeFileSync(`openbenches_export_josm_${createTest ? "_test" : ""}.geojson`, JSON.stringify({ type: "FeatureCollection", features, }, null, " "), "utf-8") const maproulette = features .filter(f => { const openbenchesId = f.properties["openbenches:id"] return !alreadyLinked.has(openbenchesId) }) .map(f => { const properties = {tags: JSON.stringify(f.properties)} properties["id"] = "openbenches/"+f.properties["openbenches:id"] return {...f, properties} }) writeFileSync(`openbenches_export_maproulette${createTest ? "_test" : ""}.geojson`, JSON.stringify({ type: "FeatureCollection", features: maproulette, }, null, " "), "utf-8") if(!createTest){ writeFileSync(`openbenches_export_maproulette_first_100.geojson`, JSON.stringify({ type: "FeatureCollection", features: maproulette.slice(0, 100), }, null, " "), "utf-8") } const openBenches = JSON.parse(readFileSync("openbenches_export_josm_.geojson", "utf-8")) for (const area in areas) { const areaGeo = areas[area] await this.conflate(osmData.features.filter( f => GeoOperations.inside(GeoOperations.centerpointCoordinates(f), areaGeo) ), openBenches, "_"+area) } await this.conflate(osmData.features, openBenches, "_all") } } new Openbenches().run()