Make metatagging lazy (fix #503), fix null-distances (#504), various other small fixes

Logic/Actors/SelectedElementTagsUpdater.ts

@@ -35,9 +35,20 @@ export default class SelectedElementTagsUpdater {
 
 
         state.selectedElement.addCallbackAndRunD(s => {
-            const id = s.properties?.id
+            let id = s.properties?.id
-            OsmObject.DownloadObjectAsync(id).then(obj => {
+            
-                SelectedElementTagsUpdater.applyUpdate(state, obj, id)
+            const backendUrl = state.osmConnection._oauth_config.url
+            if(id.startsWith(backendUrl)){
+                id = id.substring(backendUrl.length)
+            }
+            
+            if(!(id.startsWith("way") || id.startsWith("node") || id.startsWith("relation"))){
+                // This object is _not_ from OSM, so we skip it!
+                return;
+            }
+            
+            OsmObject.DownloadPropertiesOf(id).then(tags => {
+                SelectedElementTagsUpdater.applyUpdate(state, tags, id)
             }).catch(e => {
                 console.error("Could not update tags of ", id, "due to", e)
             })
@@ -50,13 +61,11 @@ export default class SelectedElementTagsUpdater {
                                    allElements: ElementStorage,
                                    changes: Changes,
                                    osmConnection: OsmConnection
-                               }, obj: OsmObject, id: string
+                               }, latestTags: any, id: string
     ) {
         const pendingChanges = state.changes.pendingChanges.data
-            .filter(change => change.type === obj.type && change.id === obj.id)
+            .filter(change => change.type +"/"+ change.id === id)
             .filter(change => change.tags !== undefined);
-        const latestTags = obj.tags
-        console.log("Applying updates of ", id, " got tags", latestTags, "and still have to apply changes: ", pendingChanges)
        
         for (const pendingChange of pendingChanges) {
             const tagChanges = pendingChange.tags;
@@ -84,7 +93,7 @@ export default class SelectedElementTagsUpdater {
             
             const localValue = currentTags[key]
             if (localValue !== osmValue) {
-                console.log("Local value:", localValue, "upstream", osmValue)
+                console.log("Local value for ", key ,":", localValue, "upstream", osmValue)
                 somethingChanged = true;
                 currentTags[key] = osmValue
             }
@@ -92,6 +101,8 @@ export default class SelectedElementTagsUpdater {
         if (somethingChanged) {
             console.log("Detected upstream changes to the object when opening it, updating...")
             currentTagsSource.ping()
+        }else{
+            console.debug("Fetched latest tags for ", id, "but detected no changes")
         }
 
     }

Logic/ExtraFunction.ts

@@ -224,6 +224,7 @@ export class ExtraFunction {
         const maxFeatures = options?.maxFeatures ?? 1
         const maxDistance = options?.maxDistance ?? 500
         const uniqueTag: string | undefined = options?.uniqueTag
+        console.log("Requested closestN")
         if (typeof features === "string") {
             const name = features
             const bbox = GeoOperations.bbox(GeoOperations.buffer(GeoOperations.bbox(feature), maxDistance))

Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts

@@ -4,12 +4,14 @@
  * Technically, more an Actor then a featuresource, but it fits more neatly this ay
  */
 import {FeatureSourceForLayer} from "../FeatureSource";
+import SimpleMetaTagger from "../../SimpleMetaTagger";
 
 export default class SaveTileToLocalStorageActor {
     public static readonly storageKey: string = "cached-features";
     public static readonly formatVersion: string = "1"
 
     constructor(source: FeatureSourceForLayer, tileIndex: number) {
+        
         source.features.addCallbackAndRunD(features => {
             const key = `${SaveTileToLocalStorageActor.storageKey}-${source.layer.layerDef.id}-${tileIndex}`
             const now = new Date()

Logic/FeatureSource/FeaturePipeline.ts

@@ -243,7 +243,7 @@ export default class FeaturePipeline {
         this.runningQuery = updater.runningQuery.map(
             overpass => {
                 console.log("FeaturePipeline: runningQuery state changed. Overpass", overpass ? "is querying," : "is idle,",
-                    "osmFeatureSource is", osmFeatureSource.isRunning ? "is running ("+  +")" : "is idle")
+                    "osmFeatureSource is", osmFeatureSource.isRunning ? "is running and needs "+neededTilesFromOsm.data?.length+" tiles (already got "+ osmFeatureSource.downloadedTiles.size  +" tiles )" : "is idle")
                 return overpass || osmFeatureSource.isRunning.data;
             }, [osmFeatureSource.isRunning]
         )
@@ -361,7 +361,6 @@ export default class FeaturePipeline {
         const self = this
         window.setTimeout(
             () => {
-                console.debug("Applying metatagging onto ", src.name)
                 const layerDef = src.layer.layerDef;
                 MetaTagging.addMetatags(
                     src.features.data,
@@ -384,7 +383,6 @@ export default class FeaturePipeline {
 
     private updateAllMetaTagging() {
         const self = this;
-        console.log("Reupdating all metatagging")
         this.perLayerHierarchy.forEach(hierarchy => {
             hierarchy.loadedTiles.forEach(src => {
                 self.applyMetaTags(src)

Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts

@@ -28,7 +28,7 @@ export default class OsmFeatureSource {
         },
         markTileVisited?: (tileId: number) => void
     };
-    private readonly downloadedTiles = new Set<number>()
+    public readonly downloadedTiles = new Set<number>()
     private readonly allowedTags: TagsFilter;
 
     constructor(options: {
@@ -53,13 +53,16 @@ export default class OsmFeatureSource {
                 return;
             }
             
+            neededTiles = neededTiles.filter(tile => !self.downloadedTiles.has(tile))
+
+            if(neededTiles.length == 0){
+                return;
+            }
+            
             self.isRunning.setData(true)
             try {
 
                 for (const neededTile of neededTiles) {
-                    if (self.downloadedTiles.has(neededTile)) {
-                        continue;
-                    }
                     console.log("Tile download", Tiles.tile_from_index(neededTile).join("/"), "started")
                     self.downloadedTiles.add(neededTile)
                     self.LoadTile(...Tiles.tile_from_index(neededTile)).then(_ => {

Logic/MetaTagging.ts

@@ -1,6 +1,5 @@
 import SimpleMetaTagger from "./SimpleMetaTagger";
 import {ExtraFuncParams, ExtraFunction} from "./ExtraFunction";
-import {UIEventSource} from "./UIEventSource";
 import LayerConfig from "../Models/ThemeConfig/LayerConfig";
 import State from "../State";
 
@@ -57,10 +56,19 @@ export default class MetaTagging {
             let somethingChanged = false
             for (const metatag of metatagsToApply) {
                 try {
-                        if(!metatag.keys.some(key => feature.properties[key] === undefined)){
+                    if (!metatag.keys.some(key => feature.properties[key] === undefined)) {
                         // All keys are already defined, we probably already ran this one
                         continue
                     }
+                    
+                    if(metatag.isLazy){
+                        somethingChanged = true;
+                        
+                        metatag.applyMetaTagsOnFeature(feature, freshness)
+                        
+                    }else{
+                        
+                    
                         const newValueAdded = metatag.applyMetaTagsOnFeature(feature, freshness)
                         /* Note that the expression:
                         * `somethingChanged = newValueAdded || metatag.applyMetaTagsOnFeature(feature, freshness)`
@@ -70,12 +78,13 @@ export default class MetaTagging {
                         * thus not running an update!
                         */
                         somethingChanged = newValueAdded || somethingChanged
+                    }
                 } catch (e) {
                     console.error("Could not calculate metatag for ", metatag.keys.join(","), ":", e, e.stack)
                 }
             }
 
-                if(layerFuncs !== undefined){
+            if (layerFuncs !== undefined) {
                 try {
                     layerFuncs(params, feature)
                 } catch (e) {
@@ -84,21 +93,15 @@ export default class MetaTagging {
                 somethingChanged = true
             }
 
-                if(somethingChanged){
+            if (somethingChanged) {
                 State.state?.allElements?.getEventSourceById(feature.properties.id)?.ping()
             }
         }
     }
 
 
-    private static createRetaggingFunc(layer: LayerConfig):
+    private static createFunctionsForFeature(calculatedTags: [string, string][]): ((feature: any) => void)[] {
-        ((params: ExtraFuncParams, feature: any) => void) {
+        const functions: ((feature: any) => void)[] = [];
-        const calculatedTags: [string, string][] = layer.calculatedTags;
-        if (calculatedTags === undefined) {
-            return undefined;
-        }
-
-        const functions: ((params: ExtraFuncParams, feature: any) => void)[] = [];
         for (const entry of calculatedTags) {
             const key = entry[0]
             const code = entry[1];
@@ -108,32 +111,30 @@ export default class MetaTagging {
 
             const func = new Function("feat", "return " + code + ";");
 
-            try {
+            const f = (feature: any) => {
-                const f = (featuresPerLayer, feature: any) => {
+                delete feature.properties[key]
-                    try {
-                        let result = func(feature);
-                        if (result instanceof UIEventSource) {
-                            result.addCallbackAndRunD(d => {
-                                if (typeof d !== "string") {
-                                    // Make sure it is a string!
-                                    d = JSON.stringify(d);
-                                }
-                                feature.properties[key] = d;
-                            })
-                            result = result.data
-                        }
 
-                        if (result === undefined || result === "") {
+                Object.defineProperty(feature.properties, key, {
-                            return;
+                    configurable: true,
+                    enumerable: false, // By setting this as not enumerable, the localTileSaver will _not_ calculate this
+                    get: function () {
+                        try {
+                            // Lazyness for the win!
+                            let result = func(feature);
+
+                            if (result === "") {
+                                result === undefined
                             }
-                        if (typeof result !== "string") {
+                            if (result !== undefined && typeof result !== "string") {
                                 // Make sure it is a string!
                                 result = JSON.stringify(result);
                             }
+                            delete feature.properties[key]
                             feature.properties[key] = result;
+                            return result;
                         } catch (e) {
                             if (MetaTagging.errorPrintCount < MetaTagging.stopErrorOutputAt) {
-                            console.warn("Could not calculate a calculated tag defined by " + code + " due to " + e + ". This is code defined in the theme. Are you the theme creator? Doublecheck your code. Note that the metatags might not be stable on new features", e,e.stack)
+                                console.warn("Could not calculate a calculated tag defined by " + code + " due to " + e + ". This is code defined in the theme. Are you the theme creator? Doublecheck your code. Note that the metatags might not be stable on new features", e, e.stack)
                                 MetaTagging.errorPrintCount++;
                                 if (MetaTagging.errorPrintCount == MetaTagging.stopErrorOutputAt) {
                                     console.error("Got ", MetaTagging.stopErrorOutputAt, " errors calculating this metatagging - stopping output now")
@@ -141,26 +142,41 @@ export default class MetaTagging {
                             }
                         }
                         
+                    }} )
+
             }
+            
+            
             functions.push(f)
-            } catch (e) {
-                console.error("Could not create a dynamic function: ", e)
         }
+        return functions;
     }
+
+    private static createRetaggingFunc(layer: LayerConfig):
+        ((params: ExtraFuncParams, feature: any) => void) {
+
+        const calculatedTags: [string, string][] = layer.calculatedTags;
+        if (calculatedTags === undefined || calculatedTags.length === 0) {
+            return undefined;
+        }
+
         return (params: ExtraFuncParams, feature) => {
             const tags = feature.properties
             if (tags === undefined) {
                 return;
             }
 
-            ExtraFunction.FullPatchFeature(params, feature);
             try {
+                const functions = MetaTagging.createFunctionsForFeature(calculatedTags)
+
+
+                ExtraFunction.FullPatchFeature(params, feature);
                 for (const f of functions) {
-                    f(params, feature);
+                    f(feature);
                 }
                 State.state?.allElements?.getEventSourceById(feature.properties.id)?.ping();
             } catch (e) {
-                console.error("While calculating a tag value: ", e)
+                console.error("Invalid syntax in calculated tags or some other error: ", e)
             }
         }
     }

Logic/Osm/OsmObject.ts

@@ -56,6 +56,19 @@ export abstract class OsmObject {
         return src;
     }
     
+    static async DownloadPropertiesOf(id: string): Promise<any> {
+        const splitted = id.split("/");
+        const type = splitted[0];
+        const idN = Number(splitted[1]);
+        if (idN < 0) {
+            return undefined;
+        }
+
+        const url = `${OsmObject.backendURL}api/0.6/${id}`;
+        const rawData = await Utils.downloadJson(url)
+        return rawData.elements[0].tags
+    }
+
     static async DownloadObjectAsync(id: string): Promise<OsmObject> {
         const splitted = id.split("/");
         const type = splitted[0];
@@ -64,7 +77,7 @@ export abstract class OsmObject {
             return undefined;
         }
 
-        const full = !id.startsWith("way") ? "" : "/full";
+        const full = (id.startsWith("way")) ? "/full" : "";
         const url = `${OsmObject.backendURL}api/0.6/${id}${full}`;
         const rawData = await Utils.downloadJson(url)
         // A full query might contain more then just the requested object (e.g. nodes that are part of a way, where we only want the way)

Logic/SimpleMetaTagger.ts

@@ -65,13 +65,34 @@ export default class SimpleMetaTagger {
     private static surfaceArea = new SimpleMetaTagger(
         {
             keys: ["_surface", "_surface:ha"],
-            doc: "The surface area of the feature, in square meters and in hectare. Not set on points and ways"
+            doc: "The surface area of the feature, in square meters and in hectare. Not set on points and ways",
+            isLazy: true
         },
         (feature => {
+            
+            Object.defineProperty(feature.properties, "_surface", {
+                enumerable: false,
+                configurable: true,
+                get: () => {
+                    const sqMeters = ""+ GeoOperations.surfaceAreaInSqMeters(feature);
+                    delete feature.properties["_surface"]
+                    feature.properties["_surface"] = sqMeters;
+                    return sqMeters
+                }
+            })
+
+            Object.defineProperty(feature.properties, "_surface:ha", {
+                enumerable: false,
+                configurable: true,
+                get: () => {
                     const sqMeters = GeoOperations.surfaceAreaInSqMeters(feature);
-            feature.properties["_surface"] = "" + sqMeters;
+                    const sqMetersHa = "" + Math.floor(sqMeters / 1000) / 10;
-            feature.properties["_surface:ha"] = "" + Math.floor(sqMeters / 1000) / 10;
+                    delete feature.properties["_surface:ha"]
-            feature.area = sqMeters;
+                    feature.properties["_surface:ha"] = sqMetersHa;
+                    return sqMetersHa
+                }
+            })
+            
             return true;
         })
     );
@@ -173,7 +194,8 @@ export default class SimpleMetaTagger {
         {
             keys: ["_isOpen", "_isOpen:description"],
             doc: "If 'opening_hours' is present, it will add the current state of the feature (being 'yes' or 'no')",
-            includesDates: true
+            includesDates: true,
+            isLazy: true
         },
         (feature => {
             if (Utils.runningFromConsole) {
@@ -182,16 +204,22 @@ export default class SimpleMetaTagger {
                 return false
             }
             
+            Object.defineProperty(feature.properties, "_isOpen",{
+                enumerable: false,
+                configurable: true,
+                get: () => {
+                    delete feature.properties._isOpen
+                    feature.properties._isOpen = ""
                     const tagsSource = State.state.allElements.getEventSourceById(feature.properties.id);
                     tagsSource.addCallbackAndRunD(tags => {
                         if (tags.opening_hours === undefined || tags._country === undefined) {
                             return;
                         }
                         try {
-
+                            const [lon, lat] = GeoOperations.centerpointCoordinates(feature)
                             const oh = new opening_hours(tags["opening_hours"], {
-                        lat: tags._lat,
+                                lat: lat,
-                        lon: tags._lon,
+                                lon: lon,
                                 address: {
                                     country_code: tags._country.toLowerCase()
                                 }
@@ -232,13 +260,17 @@ export default class SimpleMetaTagger {
                                 }
                             }
                             updateTags();
-                    return true;
+                            return true; // Our job is done, lets unregister!
                         } catch (e) {
                             console.warn("Error while parsing opening hours of ", tags.id, e);
                             tags["_isOpen"] = "parse_error";
                         }
 
                     })
+                    return feature.properties["_isOpen"]
+                }
+            })
+
         })
     )
     private static directionSimplified = new SimpleMetaTagger(
@@ -306,8 +338,12 @@ export default class SimpleMetaTagger {
         SimpleMetaTagger.objectMetaInfo
 
     ];
+    public static readonly lazyTags: string[] = [].concat(...SimpleMetaTagger.metatags.filter(tagger => tagger.isLazy)
+        .map(tagger => tagger.keys));
+
     public readonly keys: string[];
     public readonly doc: string;
+    public readonly isLazy: boolean;
     public readonly includesDates: boolean
     public readonly applyMetaTagsOnFeature: (feature: any, freshness: Date) => boolean;
     
@@ -316,10 +352,11 @@ export default class SimpleMetaTagger {
      * @param docs: what does this extra data do?
      * @param f: apply the changes. Returns true if something changed
      */
-    constructor(docs: { keys: string[], doc: string, includesDates?: boolean },
+    constructor(docs: { keys: string[], doc: string, includesDates?: boolean, isLazy?: boolean },
                 f: ((feature: any, freshness: Date) => boolean)) {
         this.keys = docs.keys;
         this.doc = docs.doc;
+        this.isLazy = docs.isLazy
         this.applyMetaTagsOnFeature = f;
         this.includesDates = docs.includesDates ?? false;
         for (const key of docs.keys) {
@@ -345,7 +382,8 @@ export default class SimpleMetaTagger {
         for (const metatag of SimpleMetaTagger.metatags) {
             subElements.push(
                 new Title(metatag.keys.join(", "), 3),
-                metatag.doc
+                metatag.doc,
+                metatag.isLazy ? "This is a lazy metatag and is only calculated when needed" : ""
             )
         }
 

UI/SpecialVisualizations.ts

@@ -27,6 +27,7 @@ import ShowDataMultiLayer from "./ShowDataLayer/ShowDataMultiLayer";
 import Minimap from "./Base/Minimap";
 import AllImageProviders from "../Logic/ImageProviders/AllImageProviders";
 import WikipediaBox from "./WikipediaBox";
+import SimpleMetaTagger from "../Logic/SimpleMetaTagger";
 
 export interface SpecialVisualization {
     funcName: string,
@@ -45,14 +46,26 @@ export default class SpecialVisualizations {
                 docs: "Prints all key-value pairs of the object - used for debugging",
                 args: [],
                 constr: ((state: State, tags: UIEventSource<any>) => {
+                    const calculatedTags = [].concat(
+                        SimpleMetaTagger.lazyTags,
+                        ... state.layoutToUse.layers.map(l => l.calculatedTags?.map(c => c[0]) ?? []))
                     return new VariableUiElement(tags.map(tags => {
                         const parts = [];
                         for (const key in tags) {
                             if (!tags.hasOwnProperty(key)) {
-                                continue;
+                                continue
                             }
                             parts.push([key, tags[key] ?? "<b>undefined</b>"]);
                         }
+                        
+                        for(const key of calculatedTags){
+                            const value = tags[key]
+                            if(value === undefined){
+                                continue
+                            }
+                            parts.push([ "<i>"+key+"</i>", value ])
+                        }
+                        
                         return new Table(
                             ["key", "value"],
                             parts

assets/layers/cycleways_and_roads/cycleways_and_roads.json

@@ -27,9 +27,6 @@
             ]
         }
     },
-    "calculatedTags": [
-        "_comfort_score=feat.score('https://raw.githubusercontent.com/pietervdvn/AspectedRouting/master/Examples/bicycle/aspects/bicycle.comfort.json')"
-    ],
     "title": {
         "render": {
             "en": "Cycleways",

assets/themes/speelplekken/speelplekken.json

@@ -152,7 +152,7 @@
           ]
         },
         "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{layer}_{z}_{x}_{y}.geojson",
-        "geoJsonZoomLevel": 11,
+        "geoJsonZoomLevel": 14,
         "isOsmCache": true
       },
       "title": {

assets/themes/uk_addresses/uk_addresses.json

@@ -80,8 +80,7 @@
         {
           "id": "uk_addresses_import_button",
           "render": "{import_button(ref:inspireid=$inspireid, Add this address, ./assets/themes/uk_addresses/housenumber_add.svg)}"
-        },
+        }
-        "all_tags"
       ],
       "calculatedTags": [
         "_embedding_object=feat.overlapWith('addresses')[0]?.feat?.properties ?? null",
@@ -122,16 +121,10 @@
         }
       },
       "calculatedTags": [
-        "_closest_3_street_names=feat.properties['addr:street'] === undefined ? feat.closestn('named_streets',3, 'name').map(f => ({name: f.feat.properties.name, distance: Math.round(1000*f.distance), id: f.id})) : []",
+        "_closest_3_street_names=feat.closestn('named_streets',3, 'name').map(f => f.feat.properties.name)",
-        "_closest_street:0:name=JSON.parse(feat.properties._closest_3_street_names)[0]?.name",
+        "_closest_street:0:name=JSON.parse(feat.properties._closest_3_street_names)[0]",
-        "_closest_street:1:name=JSON.parse(feat.properties._closest_3_street_names)[1]?.name",
+        "_closest_street:1:name=JSON.parse(feat.properties._closest_3_street_names)[1]",
-        "_closest_street:2:name=JSON.parse(feat.properties._closest_3_street_names)[2]?.name",
+        "_closest_street:2:name=JSON.parse(feat.properties._closest_3_street_names)[2]"
-        "_closest_street:0:distance=JSON.parse(feat.properties._closest_3_street_names)[0]?.distance",
-        "_closest_street:1:distance=JSON.parse(feat.properties._closest_3_street_names)[1]?.distance",
-        "_closest_street:2:distance=JSON.parse(feat.properties._closest_3_street_names)[2]?.distance",
-        "_closest_street:0:id=JSON.parse(feat.properties._closest_3_street_names)[0]?.id",
-        "_closest_street:1:id=JSON.parse(feat.properties._closest_3_street_names)[1]?.id",
-        "_closest_street:2:id=JSON.parse(feat.properties._closest_3_street_names)[2]?.id"
       ],
       "title": {
         "render": {
@@ -157,7 +150,8 @@
             "en": "What is the number of this house?"
           },
           "freeform": {
-            "key": "addr:housenumber"
+            "key": "addr:housenumber",
+            "addExtraTags": "nohousenumber="
           },
           "mappings": [
             {
@@ -186,17 +180,17 @@
           "mappings": [
             {
               "if": "addr:street:={_closest_street:0:name}",
-              "then": "Located in <b>{_closest_street:0:name}</b> (~{_closest_street:0:distance}m away)",
+              "then": "Located in <b>{_closest_street:0:name}</b>",
               "hideInAnswer": "_closest_street:0:name="
             },
             {
               "if": "addr:street:={_closest_street:1:name}",
-              "then": "Located in <b>{_closest_street:1:name}</b> (~{_closest_street:1:distance}m away)",
+              "then": "Located in <b>{_closest_street:1:name}</b>",
               "hideInAnswer": "_closest_street:1:name="
             },
             {
               "if": "addr:street:={_closest_street:2:name}",
-              "then": "Located in <b>{_closest_street:2:name}</b> (~{_closest_street:2:distance}m away)",
+              "then": "Located in <b>{_closest_street:2:name}</b>",
               "hideInAnswer": "_closest_street:2:name="
             }
           ],
@@ -254,7 +248,6 @@
     },
     {
       "id": "named_streets",
-      "name": "Named streets",
       "minzoom": 18,
       "source": {
         "osmTags": {
@@ -264,16 +257,11 @@
           ]
         }
       },
-      "title": {
-        "render": {
-          "en": "{name}"
-        }
-      },
       "color": {
         "render": "#ccc"
       },
       "width": {
-        "render": "3"
+        "render": "0"
       }
     }
   ],