Themes(benches): add script that reads the openbenches-dataset from their git repo, add (experimental) maproulette import layer to benches

Docs/Studio/Integrating_Maproulette.md

@@ -21,12 +21,12 @@ Hint: MapRoulette has a button 'rebuild task', where you can first 'remove all i
 **Most of the heavy lifting is done in [layer `maproulette-challenge`](./Docs/Layers/maproulette_challenge.md). Extend this layer with your needs.**
 The API is shortly discussed here for future reference only.
 
-There is an API-endpoint at `https://maproulette.org/api/v2/tasks/box/{x_min}/{y_min}/{x_max}/{y_max}` which can be used
-to query _all_ tasks in a bbox and returns this as geojson. Hint:
-use [the maproulette theme in debug mode](https://mapcomplete.org/maproulette?debug=true) to inspect all properties.
+Hint: use [the maproulette theme in debug mode](https://mapcomplete.org/maproulette?debug=true) to inspect all properties.
 
 To view the overview a single challenge, visit `https://maproulette.org/browse/challenges/<challenge-id>` with your
 browser.
+To get the challenge-id, visit the overview of a challenge in your browser. The URL will end with `/project/<project-id>/challenge/<challenge-id>?`. You can ignore the project-id and use the `challenge-id`. Challenge ids are unique and straight from the database, even over users and projects.
+
 The API endpoint for a single challenge is `https://maproulette.org/api/v2/challenge/view/<challenge-id>` which returns a
 geojson. This endpoint supports a bbox to only return a part of the challenge: `https://maproulette.org/api/v2/challenge/view/<challenge-id>?bbox=west,south,east,north`
 

assets/layers/maproulette_challenge/maproulette_challenge.json

@@ -236,7 +236,8 @@
             "nl": "Toon taken die zijn gecreëerd",
             "pl": "Pokaż zadania, które zostały stworzone"
           },
-          "osmTags": "mr_taskStatus=Created"
+          "osmTags": "mr_taskStatus=Created",
+          "default": true
         },
         {
           "question": {

assets/themes/benches/benches.json

@@ -81,7 +81,84 @@
   "layers": [
     "picnic_table",
     "bench",
-    "bench_at_pt"
+    "bench_at_pt",
+    {
+      "builtin": "maproulette_challenge",
+      "override": {
+        "name=": {
+          "en": "Data from OpenBenches.org"
+        },
+        "minzoom": 10,
+        "source": {
+          "#": "Alternatively, if combined with `geojsonZoomLevel`, use geoJson with {y_min}, {y_max}, {x_min} and {x_max}. Only replace <id> in the next string: `https://maproulette.org/api/v2/challenge/view/<id>?bbox={x_min},{y_max},{x_max},{y_min}`",
+          "geoJsonZoomLevel": 12,
+          "idKey": "",
+          "geoJson": "https://maproulette.org/api/v2/challenge/view/53298?bbox={x_min},{y_max},{x_max},{y_min}"
+        },
+        "calculatedTags": [
+          "_id:=''+get(feat)('tags')['openbenches:id']",
+          "image:=get(feat)('tags')['image']",
+          "_osm_poi_with_this_ref=closestn(feat)('bench',25, undefined, 50).filter(f => f.feat.properties['openbenches:id'] === feat.properties['_id']).map(f => f.feat.properties.id).join(';')",
+          "_nearby_osm_poi=closestn(feat)('bench', 5, undefined, 15)",
+          "_nearby_osm_poi:count=get(feat)('_nearby_osm_poi')?.length",
+          "_nearby_osm_poi:props=get(feat)('_nearby_osm_poi')?.map(f => ({_distance: Math.round(f.distance), _mr_id: feat.properties.id, ...f.feat.properties}))"
+        ],
+        "=tagRenderings": [
+          {
+            "id": "images",
+            "render": "<img src='{image}'/>"
+          },
+          {
+            "id": "closeness-indicator",
+            "condition": "_has_closeby_feature=yes",
+            "render": {
+              "en": "OpenStreetMap knows about <a href='#{_closest_osm_poi}'>a bench which is {_closest_osm_poi_distance} meter away.</a> "
+            }
+          },
+          {
+            "id": "list_nearby_pois",
+            "condition": {
+              "and": [
+                "mr_taskStatus=Created",
+                "_nearby_osm_poi:count>0"
+              ]
+            },
+            "render": {
+              "before": {
+                "en": "Choose below which bench you want to link.",
+                "nl": "Kies hieronder welke bank je wilt linken."
+              },
+              "special": {
+                "classes": "p-2 m-1 my-4 border-2 border-dashed border-black",
+                "key": "_nearby_osm_poi:props",
+                "tagrendering": "<b><a href='#{id}'>{id}</a></b> ({_distance}m, {openbenches:id}) {minimap(17,id;_original:id)} {tag_apply($_original:tags,Link this object.,link,id,_original:id)}",
+                "type": "multi"
+              }
+            }
+          },
+          {
+            "id": "import_point",
+            "condition": "mr_taskStatus=Created",
+            "render": {
+              "special": {
+                "type": "import_button",
+                "targetLayer": "bench",
+                "tags": "$tags",
+                "text": {
+                  "en": "Create a bench in OSM with the properties of openBenches.org",
+                  "nl": "Maak een bank aan in OSM met de attributen van openBenches.org"
+                },
+                "to_point": "yes",
+                "maproulette_id": "mr_taskId"
+              }
+            }
+          },
+
+          "maproulette.controls",
+          "all_tags"
+        ]
+      }
+    }
   ],
   "widenFactor": 1.5
 }

scripts/importscripts/openbenches.ts

@@ -0,0 +1,230 @@
+import Script from "../Script"
+import { promises as fs, writeFileSync } from "fs"
+import { Feature, Point } from "geojson"
+import { join } from "path"
+import sqlite3, { Database } from "sqlite3"
+import { open } from "sqlite"
+import ScriptUtils from "../ScriptUtils"
+import { Lists } from "../../src/Utils/Lists"
+
+/**
+ * Note:
+ * npm i sqlite sqlite3
+ * I didn't want this into the deps
+ *     "sqlite": "^5.1.1",
+ *     "sqlite3": "^5.1.7",
+ */
+interface Bench {
+    benchID: number,
+    latitude: number,
+    longitude: number,
+    address: string,
+    inscription: string,
+    description: string,
+    present: 0 | 1,
+    published: 0 | 1,
+    added: string,
+    userID: number
+}
+
+interface User {
+    name: string,
+    providerID: string,
+    provider: string,
+    userID: number
+}
+
+interface Tag {
+    tagID: number,
+    tagText: string
+}
+
+function mediaUrl(sha: string | { "sha1": string }): string {
+    if (sha["sha1"]) {
+        sha = sha["sha1"]
+    }
+    return `https://openbenches.org/image/${sha}.jpg`
+}
+
+class Openbenches extends Script {
+    private db: Database
+
+    constructor() {
+        super("Creates the OpenBenches dataset to upload to maproulette")
+    }
+
+    async buildDatabase(sqlDir: string, dbFile: string) {
+        const db = await open({
+            filename: dbFile,
+            driver: sqlite3.Database,
+        })
+
+        const files = await fs.readdir(sqlDir)
+        const sqlFiles = files.filter(f => f.endsWith(".sql"))
+
+        const skip = ["database.sql"]
+
+
+        const order = ["tags", "users", "tag_map", "media_types", "benches", "media"]
+
+        for (let file of order) {
+            console.log("Exec file", file)
+            file = "openbenc_benches_table_" + file + ".sql"
+            let content = await fs.readFile(join(sqlDir, file), "utf-8")
+            content = content.replaceAll("ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci", "")
+                .replaceAll("\\'", "''")
+            await db.exec(content)
+        }
+
+        await db.close()
+        console.log("DB has been seeded")
+    }
+
+    all<T>(query): Promise<T[]> {
+        return new Promise<T[]>((resolve, reject) => {
+            this.db.all(query, (err, rows) => {
+                if (err) {
+                    reject(err)
+                } else {
+                    resolve(<any>rows)
+                }
+            })
+        })
+    }
+
+    async loadDb(dbFile: string): Promise<Database> {
+        const db = await open({
+            filename: dbFile,
+            driver: sqlite3.Database,
+        })
+        console.log(db)
+        console.dir(db)
+        return <any>db.db
+
+    }
+
+    async createBenchInfo(benchWithUser: Bench & User, tags: string[]): Promise<Feature<Point>> {
+        const id = benchWithUser.benchID
+        const media = await this.all<{
+            sha1: string,
+            media_type: string
+        }>("SELECT * FROM media WHERE media.benchID = " + id)
+        const mediaBench = media.filter(m => m.media_type === "bench")
+        const mediaInscr = media.filter(m => m.media_type === "inscription")
+        const mediaView = media.filter(m => m.media_type === "view")
+
+        const properties = {
+            // added: benchWithUser.added,
+            "openbenches:id": id,
+            inscription: benchWithUser.inscription.replaceAll("\\r\\n", "\n"),
+            amenity: "bench",
+            // lastModifiedBy: benchWithUser.name,
+        }
+
+        let mediaMerged = Lists.dedup(mediaBench.concat(mediaInscr).map(m => mediaUrl(m)))
+        for (let i = 0; i < mediaMerged.length; i++) {
+            const m = mediaMerged[i]
+            if (i === 0) {
+                properties["image"] = m
+            } else {
+                properties["image:" + (i - 1)] = m
+
+            }
+        }
+
+        for (let i = 0; i < mediaView.length; i++) {
+            const m = mediaView[i]
+            if (i === 0) {
+                properties["image:view"] = mediaUrl(m)
+            } else {
+                properties["image:view:" + (i - 1)] = mediaUrl(m)
+
+            }
+        }
+
+        const tagsToProperties = {
+            "wooden": "material=wood",
+            "metal": "material=metal",
+            "indoors": "indoor=yes",
+            "stone": "material=stone",
+            "poem": "artwork=poem",
+            "statue": "artwork=statue",
+            "composite": "material=plastic",
+            /*"cat":"subject=cat",
+            "dog":"subject=dog" Not always a pet, sometimes also a 'dogwalker', someone mentioning their cat, ... */
+            // EMOJI: very broad category, basically that a little image is part of the 'inscription'. Should be handled by adding the emoji directly
+            // Twinned: basically, two people are remembered, often a couple -> inscription and/or subject handles this
+            // Picture: plaque has a little picture -> subset of plaque
+            // Famous: someone "famous" is remembered, although I don't know half of 'm. Too subjective for OSM
+            // FUnny: talk about subjective...
+        }
+
+        for (const tag of (tags ?? [])) {
+            const match = tagsToProperties[tag]
+            if (!match) {
+                continue
+            }
+            const [k, v] = match.split("=")
+            properties[k] = v
+            tags.splice(tags.indexOf(tag), 1)
+        }
+
+        return {
+            type: "Feature",
+            properties: { tags: JSON.stringify(properties) },
+            geometry: {
+                type: "Point",
+                coordinates: [benchWithUser.longitude, benchWithUser.latitude],
+            },
+        }
+    }
+
+
+    async main(args: string[]): Promise<void> {
+        const dbFile = "openbenches.sqlite"
+        let createTest = false
+
+        // rmSync(dbFile)
+        // await this.buildDatabase("/home/pietervdvn/git/openbenches.org/database", dbFile)
+        this.db = await this.loadDb(dbFile)
+
+        const tags = new Map<number, string>()
+        const tagRows = await this.all<Tag>("SELECT * FROM tags")
+        for (const tag of tagRows) {
+            tags.set(tag.tagID, tag.tagText)
+        }
+        const tagsOnBenches = new Map<number, string[]>()
+        const tagOnBench = await this.all<{ benchID: number, tagID: number }>("SELECT * from tag_map")
+        for (const tg of tagOnBench) {
+            const bench = tg.benchID
+            if (!tagsOnBenches.has(bench)) {
+                tagsOnBenches.set(bench, [])
+            }
+            tagsOnBenches.get(bench).push(tags.get(tg.tagID))
+        }
+
+        const r = await this.all<Bench & User>("SELECT * FROM benches INNER JOIN users ON benches.userID = users.userID")
+        const features: Feature<Point>[] = []
+        for (let i = 0; i < r.length; i++) {
+            const benchWithUser = r[i]
+            if (benchWithUser.present === 0 || benchWithUser.published === 0) {
+                continue
+            }
+            const tags = tagsOnBenches.get(benchWithUser.benchID)
+            if (i % 100 === 0) {
+                ScriptUtils.erasableLog(`Processing bench ${i}/${r.length} (${Math.round(100 * i / r.length)}%)         `)
+            }
+            features.push(await this.createBenchInfo(benchWithUser, tags))
+            if (createTest && features.length > 1000) {
+                break
+            }
+        }
+
+        writeFileSync(`openbenches_export${createTest ? "_test" : ""}.geojson`, JSON.stringify({
+            type: "FeatureCollection", features,
+        }, null, "  "), "utf-8")
+        //console.log(r)
+    }
+}
+
+new Openbenches().run()

src/Models/ThemeConfig/Conversion/PrepareLayer.ts

@@ -471,7 +471,7 @@ export class RewriteSpecial extends DesugaringStep<TagRenderingConfigJson> {
 
     private static escapeStr(v: string, context: ConversionContext): string {
         if (typeof v !== "string") {
-            context.err("Detected a non-string value where one expected a string: " + v)
+            context.err("Detected a non-string value where one expected a string (while rewriting a special): " + JSON.stringify(v))
             return RewriteSpecial.escapeStr("" + v, context)
         }
         return v

src/Models/ThemeConfig/Conversion/PrevalidateLayer.ts

@@ -72,7 +72,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
                         .enters("source", "osmTags")
                         .err(
                             "The tags that will be used to load data from OpenStreetMap are all negative - this means that they all match something that _doesn't_ have a certain tag. For example, `key=` means anything without `key`. Did you perhaps mean to use `key~*`, meaning anything _with_ this key set? The tags are:\n\t" +
-                                osmTags.asHumanString(false, false, {})
+                                osmTags.asHumanString(false)
                         )
                 }
             }
@@ -156,6 +156,9 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
                 context.err("Layer " + json.id + " does not have an explicit 'allowMove'")
             }
         }
+        if(json.source["geojsonZoomLevel"]){
+            context.enter("source").err("Use geoJsonZoomLevel (with capital J) instead of geojsonZoomLevel")
+        }
 
         if (context.hasErrors()) {
             return undefined
@@ -408,7 +411,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
                                 "This preset does not match the required tags of this layer. This implies that a newly added point will not show up.\n    A newly created point will have properties: " +
                                     tags.asHumanString(false, false, {}) +
                                     "\n    The required tags are: " +
-                                    baseTags.asHumanString(false, false, {})
+                                    baseTags.asHumanString(false)
                             )
                     }
                 }

src/UI/Popup/MinimapViz.svelte

@@ -17,15 +17,11 @@
   export let idkeys: string[]
   export let feature: Feature
   export let clss: string = "h-40 rounded"
-  const keys = idkeys
-  let featuresToShow: Store<Feature<Geometry, OsmTags>[]> = state.indexedFeatures.featuresById.map(
-    (featuresById) => {
-      if (featuresById === undefined) {
-        return []
-      }
+  let featuresToShow: Store<Feature<Geometry, OsmTags>[]> = state.indexedFeatures.featuresById.mapD(
+    (featuresById: Map<string, Feature>) => {
       const properties = tags.data
       const features: Feature<Geometry, OsmTags>[] = []
-      for (const key of keys) {
+      for (const key of idkeys) {
         const value = properties[key]
         if (value === undefined || value === null) {
           continue
@@ -59,7 +55,7 @@
   let mla = new MapLibreAdaptor(mlmap, {
     rasterLayer: state.mapProperties.rasterLayer,
     zoom: new UIEventSource<number>(17),
-    maxzoom: new UIEventSource<number>(17),
+    maxzoom: new UIEventSource<number>(22),
     rotation: state.mapProperties.rotation.followingClone(),
     allowRotating: state.mapProperties.allowRotating.followingClone(),
   })

src/UI/SpecialVisualisations/TagApplyButton.svelte

@@ -22,10 +22,12 @@
   export let onApply: () => Promise<void>
   export let state: SpecialVisualizationState
   const t = Translations.t.general.apply_button
+  export let maprouletteIdKey: string
+
+
   // THis button might be shown on MapRoulette-items, which might already have been applied
   // This will default to 'false' for non-maproulette challenges
-  let isMaprouletteAndApplied =
-    tags?.data?.["mr_taskStatus"] !== undefined && tags?.data?.["mr_taskStatus"] !== "Created"
+  let isMaprouletteAndApplied = tags?.data?.["mr_taskStatus"] !== undefined && tags?.data?.["mr_taskStatus"] !== "Created"
 
   let currentState: UIEventSource<"init" | "applying" | "applied"> = new UIEventSource(
     isMaprouletteAndApplied ? "applied" : "init"
@@ -33,15 +35,17 @@
 
   async function apply() {
     currentState.set("applying")
+    window.requestIdleCallback(async () => {
       await onApply()
       currentState.set("applied")
+    }, {timeout: 2000})
   }
 </script>
 
 <LoginToggle {state} ignoreLoading>
   {#if $currentState === "init"}
     <button on:click={() => apply()}>
-      <Icon icon={image} />
+      <Icon icon={image} clss="w-8" />
       <div class="flex flex-col">
         <div>{msg}</div>
         {#if targetIdKey}
@@ -55,6 +59,11 @@
         {:else}
           <TagHint tags={$tagsToApply} />
         {/if}
+        {#if maprouletteIdKey}
+          <div class="subtle">
+            Closes maproulette {$tags[maprouletteIdKey]}
+          </div>
+        {/if}
       </div>
     </button>
   {:else if $currentState === "applying"}

src/UI/SpecialVisualisations/TagApplyViz.ts

@@ -26,7 +26,7 @@ export default class TagApplyViz extends SpecialVisualization implements AutoAct
     public readonly args = [
         {
             name: "tags_to_apply",
-            doc: "A specification of the tags to apply. This is either hardcoded in the layer or the `$name` of a property containing the tags to apply. If redirected and the value of the linked property starts with `{`, the other property will be interpreted as a json object",
+            doc: "A specification of the tags to apply. This is either hardcoded in the layer or the `$name` of a property containing the tags to apply. If redirected and the value of the linked property starts with `{`, this value will be interpreted as a json object. All the keys and values will be applied from this object",
         },
         {
             name: "message",
@@ -68,7 +68,7 @@ export default class TagApplyViz extends SpecialVisualization implements AutoAct
             const properties = JSON.parse(spec)
             tgsSpec = []
             for (const key of Object.keys(properties)) {
-                tgsSpec.push([key, properties[key]])
+                tgsSpec.push([key, ""+ properties[key]])
             }
         } else {
             tgsSpec = TagApplyViz.parseTagSpec(spec)
@@ -182,10 +182,11 @@ export default class TagApplyViz extends SpecialVisualization implements AutoAct
         const tagsToApply: Store<Tag[]> = TagApplyViz.generateTagsToApply(args[0], tags)
         const msg = args[1]
         let image = args[2]?.trim()
+        const targetIdKey = args[3]
+        const maprouletteId = args[4]
         if (image === "" || image === "undefined") {
             image = undefined
         }
-        const targetIdKey = args[3]
 
         const onApply = async () => {
             await this.applyActionOn(feature, state, tags, args)
@@ -199,6 +200,7 @@ export default class TagApplyViz extends SpecialVisualization implements AutoAct
             image,
             targetIdKey,
             onApply,
+            maprouletteIdKey: maprouletteId
         })
     }
 }

src/UI/SpecialVisualisations/TagrenderingManipulationSpecialVisualisations.ts

@@ -96,7 +96,7 @@ class Multi extends SpecialVisualization {
     funcName = "multi"
     group = "tagrendering_manipulation"
     docs =
-        "Given an embedded tagRendering (read only) and a key, will read the keyname as a JSON-list. Every element of this list will be considered as tags and rendered with the tagRendering"
+        "Given an embedded tagRendering (read only) and a key, will read `properties[key]` as a JSON-list. Every element of this list will be considered as the main 'tags' and rendered with the tagRendering. The original tags are available as _original:<key>"
     example =
         "```json\n" +
         JSON.stringify(
@@ -139,13 +139,13 @@ class Multi extends SpecialVisualization {
         const translation = new Translation({ "*": tr })
         return new VariableUiElement(
             tags.map((tags) => {
-                let properties: object[]
+                let propertiess: object[]
                 if (typeof tags[key] === "string") {
-                    properties = JSON.parse(tags[key])
+                    propertiess = JSON.parse(tags[key])
                 } else {
-                    properties = <object[]>(<unknown>tags[key])
+                    propertiess = <object[]>(<unknown>tags[key])
                 }
-                if (!properties) {
+                if (!propertiess) {
                     console.debug(
                         "Could not create a special visualization for multi(",
                         args.join(", ") + ")",
@@ -155,10 +155,15 @@ class Multi extends SpecialVisualization {
                     return undefined
                 }
                 const elements = []
-                for (const property of properties) {
+
+                for (const properties of propertiess) {
+                    const propertiesWithOriginal = {...properties}
+                    for (const k in tags) {
+                        propertiesWithOriginal["_original:"+k]=tags[k]
+                    }
                     const subsTr = new SvelteUIElement(SpecialTranslation, {
                         t: translation,
-                        tags: new ImmutableStore(property),
+                        tags: new ImmutableStore(propertiesWithOriginal),
                         state,
                         feature,
                         layer,