forked from MapComplete/MapComplete
Themes(benches): more work on integrating openbenches, polish a bit more
This commit is contained in:
parent
87823caabc
commit
11ab4965c6
6 changed files with 212 additions and 221 deletions
|
|
@ -25,156 +25,93 @@ import { GeoOperations } from "../../src/Logic/GeoOperations"
|
|||
* "sqlite3": "^5.1.7",
|
||||
*/
|
||||
interface Bench {
|
||||
benchID: number,
|
||||
latitude: number,
|
||||
longitude: number,
|
||||
address: string,
|
||||
inscription: string,
|
||||
description: string,
|
||||
present: 0 | 1,
|
||||
published: 0 | 1,
|
||||
benchID: number
|
||||
latitude: number
|
||||
longitude: number
|
||||
address: string
|
||||
inscription: string
|
||||
description: string
|
||||
present: 0 | 1
|
||||
published: 0 | 1
|
||||
/* time of creation (or possibly last edit?) */
|
||||
added: string,
|
||||
added: string
|
||||
userID: number
|
||||
}
|
||||
|
||||
interface User {
|
||||
name: string,
|
||||
providerID: string,
|
||||
provider: string,
|
||||
name: string
|
||||
providerID: string
|
||||
provider: string
|
||||
userID: number
|
||||
}
|
||||
|
||||
interface Tag {
|
||||
tagID: number,
|
||||
tagID: number
|
||||
tagText: string
|
||||
}
|
||||
|
||||
function mediaUrl(sha: string | { "sha1": string }): string {
|
||||
function mediaUrl(sha: string | { sha1: string }): string {
|
||||
if (sha["sha1"]) {
|
||||
sha = sha["sha1"]
|
||||
}
|
||||
return `https://openbenches.org/image/${sha}.jpg`
|
||||
}
|
||||
|
||||
const uk: Feature<Polygon> = {
|
||||
"type": "Feature",
|
||||
"properties": {},
|
||||
"geometry": {
|
||||
"coordinates": [
|
||||
const uk: Feature<Polygon> = {
|
||||
type: "Feature",
|
||||
properties: {},
|
||||
geometry: {
|
||||
coordinates: [
|
||||
[
|
||||
[
|
||||
3.139397666817615,
|
||||
53.112746745001914
|
||||
],
|
||||
[
|
||||
0.12546232547020963,
|
||||
61.34289409315957
|
||||
],
|
||||
[
|
||||
-5.193638926198332,
|
||||
60.3858935023425
|
||||
],
|
||||
[
|
||||
-12.316831332595541,
|
||||
56.76308878364702
|
||||
],
|
||||
[
|
||||
-12.586640816376246,
|
||||
51.076733390490034
|
||||
],
|
||||
[
|
||||
-3.6443836396576046,
|
||||
49.4256703574342
|
||||
],
|
||||
[
|
||||
1.0194660085441853,
|
||||
50.442813369706585
|
||||
],
|
||||
[
|
||||
3.139397666817615,
|
||||
53.112746745001914
|
||||
]
|
||||
]
|
||||
[3.139397666817615, 53.112746745001914],
|
||||
[0.12546232547020963, 61.34289409315957],
|
||||
[-5.193638926198332, 60.3858935023425],
|
||||
[-12.316831332595541, 56.76308878364702],
|
||||
[-12.586640816376246, 51.076733390490034],
|
||||
[-3.6443836396576046, 49.4256703574342],
|
||||
[1.0194660085441853, 50.442813369706585],
|
||||
[3.139397666817615, 53.112746745001914],
|
||||
],
|
||||
],
|
||||
"type": "Polygon"
|
||||
}
|
||||
type: "Polygon",
|
||||
},
|
||||
}
|
||||
const us : Feature<Polygon> = {
|
||||
"type": "Feature",
|
||||
"properties": {},
|
||||
"geometry": {
|
||||
"coordinates": [
|
||||
const us: Feature<Polygon> = {
|
||||
type: "Feature",
|
||||
properties: {},
|
||||
geometry: {
|
||||
coordinates: [
|
||||
[
|
||||
[
|
||||
-171.55472370762342,
|
||||
71.44263911390138
|
||||
],
|
||||
[
|
||||
-171.31347027402668,
|
||||
33.24735774004321
|
||||
],
|
||||
[
|
||||
-105.9804086342826,
|
||||
-3.5292610992716362
|
||||
],
|
||||
[
|
||||
-57.00596161415962,
|
||||
15.805666337324794
|
||||
],
|
||||
[
|
||||
-32.880618254493015,
|
||||
49.584578264365916
|
||||
],
|
||||
[
|
||||
-47.35582427029317,
|
||||
72.85409976292118
|
||||
],
|
||||
[
|
||||
-101.60890406091582,
|
||||
79.0557752859543
|
||||
],
|
||||
[
|
||||
-171.55472370762342,
|
||||
71.44263911390138
|
||||
]
|
||||
]
|
||||
[-171.55472370762342, 71.44263911390138],
|
||||
[-171.31347027402668, 33.24735774004321],
|
||||
[-105.9804086342826, -3.5292610992716362],
|
||||
[-57.00596161415962, 15.805666337324794],
|
||||
[-32.880618254493015, 49.584578264365916],
|
||||
[-47.35582427029317, 72.85409976292118],
|
||||
[-101.60890406091582, 79.0557752859543],
|
||||
[-171.55472370762342, 71.44263911390138],
|
||||
],
|
||||
],
|
||||
"type": "Polygon"
|
||||
}
|
||||
type: "Polygon",
|
||||
},
|
||||
}
|
||||
const australia: Feature<Polygon> = {
|
||||
"type": "Feature",
|
||||
"properties": {},
|
||||
"geometry": {
|
||||
"coordinates": [
|
||||
const australia: Feature<Polygon> = {
|
||||
type: "Feature",
|
||||
properties: {},
|
||||
geometry: {
|
||||
coordinates: [
|
||||
[
|
||||
[
|
||||
177.6309142850211,
|
||||
-48.72845301037672
|
||||
],
|
||||
[
|
||||
177.6309142850211,
|
||||
-8.050870320392335
|
||||
],
|
||||
[
|
||||
107.59695622498174,
|
||||
-8.050870320392335
|
||||
],
|
||||
[
|
||||
107.59695622498174,
|
||||
-48.72845301037672
|
||||
],
|
||||
[
|
||||
177.6309142850211,
|
||||
-48.72845301037672
|
||||
]
|
||||
]
|
||||
[177.6309142850211, -48.72845301037672],
|
||||
[177.6309142850211, -8.050870320392335],
|
||||
[107.59695622498174, -8.050870320392335],
|
||||
[107.59695622498174, -48.72845301037672],
|
||||
[177.6309142850211, -48.72845301037672],
|
||||
],
|
||||
],
|
||||
"type": "Polygon"
|
||||
}
|
||||
type: "Polygon",
|
||||
},
|
||||
}
|
||||
const areas = {uk, us, australia}
|
||||
const areas = { uk, us, australia }
|
||||
|
||||
class Openbenches extends Script {
|
||||
private db: Database
|
||||
|
|
@ -190,18 +127,18 @@ class Openbenches extends Script {
|
|||
})
|
||||
|
||||
const files = await fs.readdir(sqlDir)
|
||||
const sqlFiles = files.filter(f => f.endsWith(".sql"))
|
||||
const sqlFiles = files.filter((f) => f.endsWith(".sql"))
|
||||
|
||||
const skip = ["database.sql"]
|
||||
|
||||
|
||||
const order = ["tags", "users", "tag_map", "media_types", "benches", "media"]
|
||||
|
||||
for (let file of order) {
|
||||
console.log("Exec file", file)
|
||||
file = "openbenc_benches_table_" + file + ".sql"
|
||||
let content = await fs.readFile(join(sqlDir, file), "utf-8")
|
||||
content = content.replaceAll("ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci", "")
|
||||
content = content
|
||||
.replaceAll("ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci", "")
|
||||
.replaceAll("\\'", "''")
|
||||
await db.exec(content)
|
||||
}
|
||||
|
|
@ -228,39 +165,37 @@ class Openbenches extends Script {
|
|||
driver: sqlite3.Database,
|
||||
})
|
||||
return <any>db.db
|
||||
|
||||
}
|
||||
|
||||
async createBenchInfo(benchWithUser: Bench & User, tags: string[]): Promise<Feature<Point>> {
|
||||
const id = benchWithUser.benchID
|
||||
const media = await this.all<{
|
||||
sha1: string,
|
||||
sha1: string
|
||||
media_type: string
|
||||
}>("SELECT * FROM media WHERE media.benchID = " + id)
|
||||
const mediaBench = media.filter(m => m.media_type === "bench")
|
||||
const mediaInscr = media.filter(m => m.media_type === "inscription")
|
||||
const mediaView = media.filter(m => m.media_type === "view")
|
||||
const mediaBench = media.filter((m) => m.media_type === "bench")
|
||||
const mediaInscr = media.filter((m) => m.media_type === "inscription")
|
||||
const mediaView = media.filter((m) => m.media_type === "view")
|
||||
|
||||
const inscription = benchWithUser.inscription.replaceAll("\\r\\n", "\n")
|
||||
const properties = {
|
||||
lastModifiedTime: benchWithUser.added,
|
||||
"openbenches:id": id,
|
||||
inscription: inscription.slice(0,255),
|
||||
inscription: inscription.slice(0, 255),
|
||||
amenity: "bench",
|
||||
lastModifiedBy: benchWithUser.name,
|
||||
}
|
||||
if(inscription.length >= 255){
|
||||
if (inscription.length >= 255) {
|
||||
properties["inscription:0"] = inscription.slice(255)
|
||||
}
|
||||
|
||||
let mediaMerged = Lists.dedup(mediaBench.concat(mediaInscr).map(m => mediaUrl(m)))
|
||||
let mediaMerged = Lists.dedup(mediaBench.concat(mediaInscr).map((m) => mediaUrl(m)))
|
||||
for (let i = 0; i < mediaMerged.length; i++) {
|
||||
const m = mediaMerged[i]
|
||||
if (i === 0) {
|
||||
properties["image"] = m
|
||||
} else {
|
||||
properties["image:" + (i - 1)] = m
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -270,18 +205,17 @@ class Openbenches extends Script {
|
|||
properties["image:view"] = mediaUrl(m)
|
||||
} else {
|
||||
properties["image:view:" + (i - 1)] = mediaUrl(m)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
const tagsToProperties = {
|
||||
"wooden": "material=wood",
|
||||
"metal": "material=metal",
|
||||
"indoors": "indoor=yes",
|
||||
"stone": "material=stone",
|
||||
"poem": "artwork=poem",
|
||||
"statue": "artwork=statue",
|
||||
"composite": "material=plastic",
|
||||
wooden: "material=wood",
|
||||
metal: "material=metal",
|
||||
indoors: "indoor=yes",
|
||||
stone: "material=stone",
|
||||
poem: "artwork=poem",
|
||||
statue: "artwork=statue",
|
||||
composite: "material=plastic",
|
||||
/*"cat":"subject=cat",
|
||||
"dog":"subject=dog" Not always a pet, sometimes also a 'dogwalker', someone mentioning their cat, ... */
|
||||
// EMOJI: very broad category, basically that a little image is part of the 'inscription'. Should be handled by adding the emoji directly
|
||||
|
|
@ -291,7 +225,7 @@ class Openbenches extends Script {
|
|||
// FUnny: talk about subjective...
|
||||
}
|
||||
|
||||
for (const tag of (tags ?? [])) {
|
||||
for (const tag of tags ?? []) {
|
||||
const match = tagsToProperties[tag]
|
||||
if (!match) {
|
||||
continue
|
||||
|
|
@ -314,7 +248,10 @@ class Openbenches extends Script {
|
|||
async getAlreadyImported(): Promise<FeatureCollection> {
|
||||
const alreadyImportedPath = "openbenches_linked_in_osm.geojson"
|
||||
if (!existsSync(alreadyImportedPath)) {
|
||||
const overpass = new Overpass(Constants.defaultOverpassUrls[0], TagUtils.Tag("openbenches:id~*"))
|
||||
const overpass = new Overpass(
|
||||
Constants.defaultOverpassUrls[0],
|
||||
TagUtils.Tag("openbenches:id~*")
|
||||
)
|
||||
const dataAndDate = await overpass.queryGeoJson(BBox.global)
|
||||
const data = dataAndDate[0]
|
||||
writeFileSync(alreadyImportedPath, JSON.stringify(data), "utf-8")
|
||||
|
|
@ -345,7 +282,11 @@ class Openbenches extends Script {
|
|||
}
|
||||
if (key.startsWith("image")) {
|
||||
const imgValue = ob.properties[key]
|
||||
if (Object.values(bench.properties).some(v => v === imgValue || (v + ".jpg") === imgValue)) {
|
||||
if (
|
||||
Object.values(bench.properties).some(
|
||||
(v) => v === imgValue || v + ".jpg" === imgValue
|
||||
)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
let ikey = "image"
|
||||
|
|
@ -354,37 +295,44 @@ class Openbenches extends Script {
|
|||
i++
|
||||
ikey = "image:" + i
|
||||
}
|
||||
const li = new ChangeTagAction(bench.properties.id, new OsmTag(ikey, imgValue), bench.properties, {
|
||||
theme: "openbenches",
|
||||
changeType: "link-image",
|
||||
})
|
||||
const li = new ChangeTagAction(
|
||||
bench.properties.id,
|
||||
new OsmTag(ikey, imgValue),
|
||||
bench.properties,
|
||||
{
|
||||
theme: "openbenches",
|
||||
changeType: "link-image",
|
||||
}
|
||||
)
|
||||
changes.push(li)
|
||||
bench.properties[ikey] = imgValue
|
||||
console.log(` + ${ikey}=${imgValue}`)
|
||||
} else if (!bench.properties[key]) {
|
||||
const v = ob.properties[key]
|
||||
if(v.length >= 255){
|
||||
console.log("Text too long:", v.replaceAll("\n"," "))
|
||||
if (v.length >= 255) {
|
||||
console.log("Text too long:", v.replaceAll("\n", " "))
|
||||
continue
|
||||
}
|
||||
changes.push(new ChangeTagAction(
|
||||
bench.properties.id,
|
||||
new OsmTag(key, v),
|
||||
bench.properties,
|
||||
{
|
||||
theme: "openbenches",
|
||||
changeType: "answer",
|
||||
},
|
||||
))
|
||||
changes.push(
|
||||
new ChangeTagAction(
|
||||
bench.properties.id,
|
||||
new OsmTag(key, v),
|
||||
bench.properties,
|
||||
{
|
||||
theme: "openbenches",
|
||||
changeType: "answer",
|
||||
}
|
||||
)
|
||||
)
|
||||
console.log(` - ${key}=${ob.properties[key].replaceAll("\n", " ")}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(changes.length === 0){
|
||||
if (changes.length === 0) {
|
||||
return
|
||||
}
|
||||
const xml = await Changes.createChangesetXMLForJosm(changes)
|
||||
writeFileSync(`attributes_import${area}.osc`,xml, "utf-8")
|
||||
writeFileSync(`attributes_import${area}.osc`, xml, "utf-8")
|
||||
}
|
||||
|
||||
async main(args: string[]): Promise<void> {
|
||||
|
|
@ -394,12 +342,13 @@ class Openbenches extends Script {
|
|||
const osmData = await this.getAlreadyImported()
|
||||
|
||||
// rmSync(dbFile)
|
||||
if(!existsSync(dbFile)){
|
||||
console.log("No database file found at "+dbFile+", recreating the database")
|
||||
if (!existsSync(dbFile)) {
|
||||
console.log("No database file found at " + dbFile + ", recreating the database")
|
||||
await this.buildDatabase("/home/pietervdvn/git/openbenches.org/database", dbFile)
|
||||
}
|
||||
const alreadyLinked: Set<number> = new Set(osmData.features.map(f => Number(f.properties["openbenches:id"])))
|
||||
|
||||
const alreadyLinked: Set<number> = new Set(
|
||||
osmData.features.map((f) => Number(f.properties["openbenches:id"]))
|
||||
)
|
||||
|
||||
this.db = await this.loadDb(dbFile)
|
||||
|
||||
|
|
@ -409,7 +358,9 @@ class Openbenches extends Script {
|
|||
tags.set(tag.tagID, tag.tagText)
|
||||
}
|
||||
const tagsOnBenches = new Map<number, string[]>()
|
||||
const tagOnBench = await this.all<{ benchID: number, tagID: number }>("SELECT * from tag_map")
|
||||
const tagOnBench = await this.all<{ benchID: number; tagID: number }>(
|
||||
"SELECT * from tag_map"
|
||||
)
|
||||
for (const tg of tagOnBench) {
|
||||
const bench = tg.benchID
|
||||
if (!tagsOnBenches.has(bench)) {
|
||||
|
|
@ -418,11 +369,13 @@ class Openbenches extends Script {
|
|||
tagsOnBenches.get(bench).push(tags.get(tg.tagID))
|
||||
}
|
||||
|
||||
const openbenches = await this.all<Bench & User>("SELECT * FROM benches INNER JOIN users ON benches.userID = users.userID")
|
||||
const openbenches = await this.all<Bench & User>(
|
||||
"SELECT * FROM benches INNER JOIN users ON benches.userID = users.userID"
|
||||
)
|
||||
const features: Feature<Point>[] = []
|
||||
let skipped = 0
|
||||
for (let i = 0; i < openbenches.length; i++) {
|
||||
if(alreadyLinked.has(i)){
|
||||
if (alreadyLinked.has(i)) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
|
@ -432,33 +385,50 @@ class Openbenches extends Script {
|
|||
}
|
||||
const tags = tagsOnBenches.get(benchWithUser.benchID)
|
||||
if (i % 100 === 0) {
|
||||
ScriptUtils.erasableLog(`Processing bench ${i}/${openbenches.length} (${Math.round(100 * i / openbenches.length)}%) `)
|
||||
ScriptUtils.erasableLog(
|
||||
`Processing bench ${i}/${openbenches.length} (${Math.round(
|
||||
(100 * i) / openbenches.length
|
||||
)}%) `
|
||||
)
|
||||
}
|
||||
features.push(await this.createBenchInfo(benchWithUser, tags))
|
||||
if (createTest && features.length > 1000) {
|
||||
break
|
||||
}
|
||||
}
|
||||
/*
|
||||
/*
|
||||
writeFileSync(`openbenches_export_josm_${createTest ? "_test" : ""}.geojson`, JSON.stringify({
|
||||
type: "FeatureCollection", features,
|
||||
}, null, " "), "utf-8")*/
|
||||
|
||||
const maproulette = features
|
||||
.map(f => {
|
||||
const properties = {tags: JSON.stringify(f.properties)}
|
||||
properties["id"] = "openbenches/"+f.properties["openbenches:id"]
|
||||
return {...f, properties}
|
||||
const maproulette = features.map((f) => {
|
||||
const fProps = {
|
||||
...f.properties
|
||||
}
|
||||
|
||||
delete fProps["lastModifiedBy"]
|
||||
delete fProps["lastModifiedTime"]
|
||||
const properties = { ...f.properties, tags: JSON.stringify(fProps) }
|
||||
delete properties["amenity"] // Makes sure mapcomplete doesn't think this is a bench...
|
||||
properties["id"] = "openbenches/" + f.properties["openbenches:id"]
|
||||
|
||||
return { ...f, properties }
|
||||
})
|
||||
console.log("Skipped",skipped,"benches as already linked/imported")
|
||||
writeFileSync(`openbenches_export_maproulette${createTest ? "_test" : ""}.geojson`, JSON.stringify({
|
||||
type: "FeatureCollection", features: maproulette,
|
||||
}, null, " "), "utf-8")
|
||||
|
||||
|
||||
console.log("Skipped", skipped, "benches as already linked/imported")
|
||||
writeFileSync(
|
||||
`openbenches_export_maproulette${createTest ? "_test" : ""}.geojson`,
|
||||
JSON.stringify(
|
||||
{
|
||||
type: "FeatureCollection",
|
||||
features: maproulette,
|
||||
},
|
||||
null,
|
||||
" "
|
||||
),
|
||||
"utf-8"
|
||||
)
|
||||
|
||||
await this.conflate(osmData.features, { type: "FeatureCollection", features }, "_all")
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue