Refactoring of metatagging and extrafunctions to splice out the relation memberships, add calculatedTags and metatags into cache

Logic/ExtraFunction.ts

@@ -2,6 +2,7 @@ import {GeoOperations} from "./GeoOperations";
 import {UIElement} from "../UI/UIElement";
 import Combine from "../UI/Base/Combine";
 import State from "../State";
+import {Relation} from "./Osm/ExtractRelations";
 
 export class ExtraFunction {
 
@@ -40,11 +41,11 @@ Some advanced functions are available on <b>feat</b> as well:
         "overlapWith",
         "Gives a list of features from the specified layer which this feature overlaps with, the amount of overlap in m². The returned value is <b>{ feat: GeoJSONFeature, overlap: number}</b>",
         ["...layerIds - one or more layer ids  of the layer from which every feature is checked for overlap)"],
-        (featuresPerLayer, feat) => {
+        (params, feat) => {
             return (...layerIds: string[]) => {
                 const result = []
                 for (const layerId of layerIds) {
-                    const otherLayer = featuresPerLayer.get(layerId);
+                    const otherLayer = params.featuresPerLayer.get(layerId);
                     if (otherLayer === undefined) {
                         continue;
                     }
@@ -80,10 +81,10 @@ Some advanced functions are available on <b>feat</b> as well:
         "closest",
         "Given either a list of geojson features or a single layer name, gives the single object which is nearest to the feature. In the case of ways/polygons, only the centerpoint is considered.",
         ["list of features"],
-        (featuresPerLayer, feature) => {
+        (params, feature) => {
             return (features) => {
                 if (typeof features === "string") {
-                    features = featuresPerLayer.get(features)
+                    features = params.featuresPerLayer.get(features)
                 }
                 let closestFeature = undefined;
                 let closestDistance = undefined;
@@ -118,11 +119,8 @@ Some advanced functions are available on <b>feat</b> as well:
         "memberships",
         "Gives a list of {role: string, relation: Relation}-objects, containing all the relations that this feature is part of. \n\nFor example: `_part_of_walking_routes=feat.memberships().map(r => r.relation.tags.name).join(';')`",
         [],
-        (featuresPerLayer, feature) => {
+        (params, feature) => {
-            return () => {
+            return () =>   params.relations ?? [];
-               return State.state.knownRelations.data?.get(feature.id) ?? [];
-            }
-
         }
     )
 
@@ -130,9 +128,9 @@ Some advanced functions are available on <b>feat</b> as well:
     private readonly _name: string;
     private readonly _args: string[];
     private readonly _doc: string;
-    private readonly _f: (featuresPerLayer: Map<string, any[]>, feat: any) => any;
+    private readonly _f: (params: {featuresPerLayer: Map<string, any[]>, relations: {role: string, relation: Relation}[]}, feat: any) => any;
 
-    constructor(name: string, doc: string, args: string[], f: ((featuresPerLayer: Map<string, any[]>, feat: any) => any)) {
+    constructor(name: string, doc: string, args: string[], f: ((params: {featuresPerLayer: Map<string, any[]>, relations: {role: string, relation: Relation}[]}, feat: any) => any)) {
         this._name = name;
         this._doc = doc;
         this._args = args;
@@ -140,9 +138,9 @@ Some advanced functions are available on <b>feat</b> as well:
 
     }
 
-    public static FullPatchFeature(featuresPerLayer: Map<string, any[]>, feature) {
+    public static FullPatchFeature(featuresPerLayer: Map<string, any[]>,relations: {role: string, relation: Relation}[], feature) {
         for (const func of ExtraFunction.allFuncs) {
-            func.PatchFeature(featuresPerLayer, feature);
+            func.PatchFeature(featuresPerLayer, relations, feature);
         }
     }
 
@@ -168,7 +166,8 @@ Some advanced functions are available on <b>feat</b> as well:
         ]);
     }
 
-    public PatchFeature(featuresPerLayer: Map<string, any[]>, feature: any) {
+    public PatchFeature(featuresPerLayer: Map<string, any[]>, relations: {role: string, relation: Relation}[], feature: any) {
-        feature[this._name] = this._f(featuresPerLayer, feature);
+     
+        feature[this._name] = this._f({featuresPerLayer: featuresPerLayer, relations: relations}, feature);
     }
 }

Logic/FeatureSource/FeatureDuplicatorPerLayer.ts

@@ -12,6 +12,10 @@ export default class FeatureDuplicatorPerLayer implements FeatureSource {
     public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
 
     
+    public static GetMatchingLayerId(){
+        
+    }
+
     constructor(layers: UIEventSource<{ layerDef: LayerConfig }[]>, upstream: FeatureSource) {
         this.features = upstream.features.map(features => {
             const newFeatures: { feature: any, freshness: Date }[] = [];

Logic/FeatureSource/FeaturePipeline.ts

@@ -35,7 +35,7 @@ export default class FeaturePipeline implements FeatureSource {
 
         const geojsonSources: FeatureSource [] = GeoJsonSource
             .ConstructMultiSource(flayers.data, locationControl)
-            .map(geojsonSource => new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, geojsonSource)));
+            .map(geojsonSource =>  new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, geojsonSource)));
         
         const amendedLocalStorageSource =
             new RememberingSource(new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, new LocalStorageSource(layout))

Logic/FeatureSource/MetaTaggingFeatureSource.ts

@@ -3,6 +3,7 @@ import {UIEventSource} from "../UIEventSource";
 import State from "../../State";
 import Hash from "../Web/Hash";
 import MetaTagging from "../MetaTagging";
+import ExtractRelations from "../Osm/ExtractRelations";
 
 export default class MetaTaggingFeatureSource implements FeatureSource {
     features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{feature: any; freshness: Date}[]>(undefined);
@@ -21,7 +22,7 @@ export default class MetaTaggingFeatureSource implements FeatureSource {
                     }
                 })
 
-                MetaTagging.addMetatags(featuresFreshness, State.state.layoutToUse.data.layers);
+                MetaTagging.addMetatags(featuresFreshness, State.state.knownRelations.data, State.state.layoutToUse.data.layers);
                 self.features.setData(featuresFreshness);
         });
     }

Logic/MetaTagging.ts

@@ -1,6 +1,14 @@
 import LayerConfig from "../Customizations/JSON/LayerConfig";
 import SimpleMetaTagger from "./SimpleMetaTagger";
 import {ExtraFunction} from "./ExtraFunction";
+import State from "../State";
+import {Relation} from "./Osm/ExtractRelations";
+
+
+interface Params {
+    featuresPerLayer: Map<string, any[]>,
+    memberships: Map<string, { role: string, relation: Relation }[]>
+}
 
 /**
  * Metatagging adds various tags to the elements, e.g. lat, lon, surface area, ...
@@ -14,7 +22,8 @@ export default class MetaTagging {
      * An actor which adds metatags on every feature in the given object
      * The features are a list of geojson-features, with a "properties"-field and geometry
      */
-    static addMetatags(features: { feature: any; freshness: Date }[], layers: LayerConfig[]) {
+    static addMetatags(features: { feature: any; freshness: Date }[],
+                       relations: Map<string, { role: string, relation: Relation }[]>, layers: LayerConfig[]) {
 
         for (const metatag of SimpleMetaTagger.metatags) {
             try {
@@ -26,7 +35,7 @@ export default class MetaTagging {
         }
 
         // The functions - per layer - which add the new keys
-        const layerFuncs = new Map<string, ((featursPerLayer: Map<string, any[]>, feature: any) => void)>();
+        const layerFuncs = new Map<string, ((params: Params, feature: any) => void)>();
         for (const layer of layers) {
             layerFuncs.set(layer.id, this.createRetaggingFunc(layer));
         }
@@ -48,27 +57,26 @@ export default class MetaTagging {
             if (f === undefined) {
                 continue;
             }
-
+            f({featuresPerLayer: featuresPerLayer, memberships: relations}, feature.feature)
-            f(featuresPerLayer, feature.feature)
         }
 
     }
 
 
-    private static createRetaggingFunc(layer: LayerConfig): ((featuresPerLayer: Map<string, any[]>, feature: any) => void) {
+    private static createRetaggingFunc(layer: LayerConfig):
+        ((params: Params, feature: any) => void) {
         const calculatedTags: [string, string][] = layer.calculatedTags;
         if (calculatedTags === undefined) {
             return undefined;
         }
 
-        const functions: ((featuresPerLayer: Map<string, any[]>, feature: any) => void)[] = [];
+        const functions: ((params: Params, feature: any) => void)[] = [];
         for (const entry of calculatedTags) {
             const key = entry[0]
             const code = entry[1];
             if (code === undefined) {
                 continue;
             }
-
             const func = new Function("feat", "return " + code + ";");
 
             const f = (featuresPerLayer, feature: any) => {
@@ -76,16 +84,17 @@ export default class MetaTagging {
             }
             functions.push(f)
         }
-        return (featuresPerLayer: Map<string, any[]>, feature) => {
+        return (params: Params, feature) => {
             const tags = feature.properties
             if (tags === undefined) {
                 return;
             }
 
-            ExtraFunction.FullPatchFeature(featuresPerLayer, feature);
+            const relations = params.memberships.get(feature.properties.id)
+            ExtraFunction.FullPatchFeature(params.featuresPerLayer, relations, feature);
             try {
                 for (const f of functions) {
-                    f(featuresPerLayer, feature);
+                    f(params, feature);
                 }
             } catch (e) {
                 console.error("While calculating a tag value: ", e)

Logic/Osm/ExtractRelations.ts

@@ -17,7 +17,6 @@ export default class ExtractRelations {
 
     public static RegisterRelations(overpassJson: any) : void{
         const memberships = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(overpassJson))
-        console.log("Assigned memberships: ", memberships)
         State.state.knownRelations.setData(memberships)
     }
 
@@ -40,7 +39,7 @@ export default class ExtractRelations {
      * @param relations
      * @constructor
      */
-    private static BuildMembershipTable(relations: Relation[]): Map<string, { role: string, relation: Relation, }[]> {
+    public static BuildMembershipTable(relations: Relation[]): Map<string, { role: string, relation: Relation }[]> {
         const memberships = new Map<string, { role: string, relation: Relation }[]>()
 
         for (const relation of relations) {

Logic/SimpleMetaTagger.ts

@@ -9,6 +9,7 @@ import {UIElement} from "../UI/UIElement";
 import Combine from "../UI/Base/Combine";
 import UpdateTagsFromOsmAPI from "./Actors/UpdateTagsFromOsmAPI";
 
+
 export default class SimpleMetaTagger {
     public readonly keys: string[];
     public readonly doc: string;
@@ -89,6 +90,11 @@ export default class SimpleMetaTagger {
         ["_isOpen", "_isOpen:description"],
         "If 'opening_hours' is present, it will add the current state of the feature (being 'yes' or 'no')",
         (feature => {
+            if(Utils.runningFromConsole){
+                // We are running from console, thus probably creating a cache
+                // isOpen is irrelevant
+                return
+            }
             
             const tagsSource = State.state.allElements.getEventSourceById(feature.properties.id);
             tagsSource.addCallbackAndRun(tags => {
@@ -317,7 +323,7 @@ export default class SimpleMetaTagger {
     ];
 
     static GetCountryCodeFor(lon: number, lat: number, callback: (country: string) => void) {
-        SimpleMetaTagger.coder.GetCountryCodeFor(lon, lat, callback)
+        SimpleMetaTagger.coder?.GetCountryCodeFor(lon, lat, callback)
     }
 
     static HelpText(): UIElement {

assets/layers/slow_roads/slow_roads.json

@@ -159,32 +159,34 @@
   "width": {
     "render": "4"
   },
-  "dashArray": "2 10 6 10",
+  "dashArray": {
-  "color": {
+    "render": "",
-    "render": "#bb2",
     "mappings": [
       {
         "if": "highway=cycleway",
-        "then": "#00c"
+        "then": ""
       },
       {
         "if": "highway=path",
-        "then": "#bb2"
+        "then": "10 3"
       },
       {
         "if": "highway=footway",
-        "then": "#c30"
+        "then": "10 10"
       },
       {
         "if": "highway=pedestrian",
-        "then": "#3c3"
+        "then": "10 10"
       },
       {
         "if": "highway=living_street",
-        "then": "#ccc"
+        "then": "10 5 3 5"
       }
     ]
   },
+  "color": {
+    "render": "#eaba2a"
+  },
   "presets": [
   ]
 }

package.json

@@ -15,7 +15,7 @@
     "generate:translations": "ts-node scripts/generateTranslations.ts",
     "generate:layouts": "ts-node scripts/generateLayouts.ts",
     "generate:docs": "ts-node scripts/generateDocs.ts && ts-node scripts/generateTaginfoProjectFiles.ts",
-    "generate:cache:speelplekken": "ts-node scripts/generateCache.ts speelplekken 14 ./cache/speelplekken 51.2003 4.3925 51.1058 4.5087",
+    "generate:cache:speelplekken": "ts-node scripts/generateCache.ts speelplekken 14 ../pietervdvn.github.io/speelplekken_cache/ 51.20 4.37 51.11 4.51",
     "generate:layeroverview": "ts-node scripts/generateLayerOverview.ts --no-fail",
     "generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail",
     "validate:layeroverview": "ts-node scripts/generateLayerOverview.ts --report",

scripts/generateCache.ts

@@ -14,6 +14,8 @@ import ScriptUtils from "./ScriptUtils";
 import ExtractRelations from "../Logic/Osm/ExtractRelations";
 import * as OsmToGeoJson from "osmtogeojson";
 import {Script} from "vm";
+import MetaTagging from "../Logic/MetaTagging";
+import State from "../State";
 
 function createOverpassObject(theme: LayoutConfig) {
     let filters: TagsFilter[] = [];
@@ -26,8 +28,8 @@ function createOverpassObject(theme: LayoutConfig) {
             continue;
         }
         if (layer.source.geojsonSource !== undefined) {
-            // Not our responsibility to download this layer!
+            // We download these anyway - we are building the cache after all!
-            continue;
+            //continue;
         }
 
 
@@ -68,17 +70,17 @@ function metaJsonName(targetDir: string, x: number, y: number, z: number): strin
     return targetDir + "_" + z + "_" + x + "_" + y + ".meta.json"
 }
 
-async function downloadRaw(targetdir: string,  r: TileRange, overpass: Overpass) {
+async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass) {
     let downloaded = 0
     for (let x = r.xstart; x <= r.xend; x++) {
         for (let y = r.ystart; y <= r.yend; y++) {
-            console.log("x:", (x - r.xstart), "/", (r.xend - r.xstart), "; y:", (y - r.ystart), "/", (r.yend - r.ystart), "; total: ", downloaded, "/", r.total)
             downloaded++;
             const filename = rawJsonName(targetdir, x, y, r.zoomlevel)
             if (existsSync(filename)) {
                 console.log("Already exists: ", filename)
                 continue;
             }
+            console.log("x:", (x - r.xstart), "/", (r.xend - r.xstart), "; y:", (y - r.ystart), "/", (r.yend - r.ystart), "; total: ", downloaded, "/", r.total)
 
             const boundsArr = Utils.tile_bounds(r.zoomlevel, x, y)
             const bounds = {
@@ -87,28 +89,35 @@ async function downloadRaw(targetdir: string,  r: TileRange, overpass: Overpass)
                 east: Math.max(boundsArr[0][1], boundsArr[1][1]),
                 west: Math.min(boundsArr[0][1], boundsArr[1][1])
             }
-            console.log("Downloading tile", r.zoomlevel, x, y, "with bounds", bounds)
             const url = overpass.buildQuery("[bbox:" + bounds.south + "," + bounds.west + "," + bounds.north + "," + bounds.east + "]")
 
+            let gotResponse = false
             ScriptUtils.DownloadJSON(url,
                 chunks => {
+                    gotResponse = true;
                     saveResponse(chunks, filename)
                 })
 
-            await ScriptUtils.sleep(10000)
+            while (!gotResponse) {
-            console.debug("Waking up")
+                await ScriptUtils.sleep(10000)
+                console.debug("Waking up")
+                if (!gotResponse) {
+                    console.log("Didn't get an answer yet - waiting more")
+                }
+            }
+
 
         }
     }
 }
 
-async function postProcess(targetdir: string, r: TileRange) {
+async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig) {
     let processed = 0;
     for (let x = r.xstart; x <= r.xend; x++) {
         for (let y = r.ystart; y <= r.yend; y++) {
             processed++;
             const filename = rawJsonName(targetdir, x, y, r.zoomlevel)
-            console.log(" Post processing", processed, "/",r. total, filename)
+            console.log(" Post processing", processed, "/", r.total, filename)
             if (!existsSync(filename)) {
                 throw "Not found - and not downloaded. Run this script again!: " + filename
             }
@@ -118,11 +127,26 @@ async function postProcess(targetdir: string, r: TileRange) {
 
             // Create and save the geojson file - which is the main chunk of the data
             const geojson = OsmToGeoJson.default(rawOsm);
+            const osmTime = new Date(rawOsm.osm3s.timestamp_osm_base);
+
+            for (const feature of geojson.features) {
+
+                for (const layer of theme.layers) {
+                    if (layer.source.osmTags.matchesProperties(feature.properties)) {
+                        feature["_matching_layer_id"] = layer.id;
+                        break;
+                    }
+                }
+            }
+            const featuresFreshness = geojson.features.map(feature => ({
+                freshness: osmTime,
+                feature: feature
+            }));
+            // Extract the relationship information
+            const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm))
+            MetaTagging.addMetatags(featuresFreshness, relations, theme.layers);
             writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson))
 
-            // Extract the relationship information
-            const relations = ExtractRelations.GetRelationElements(rawOsm)
-            const osmTime = new Date(rawOsm.osm3s.timestamp_osm_base);
 
             const meta = {
                 freshness: osmTime,
@@ -145,7 +169,7 @@ async function main(args: string[]) {
     }
     const themeName = args[0]
     const zoomlevel = Number(args[1])
-    const targetdir = args[2]
+    const targetdir = args[2] + "/" + themeName
     const lat0 = Number(args[3])
     const lon0 = Number(args[4])
     const lat1 = Number(args[5])
@@ -167,7 +191,7 @@ async function main(args: string[]) {
 
 
     await downloadRaw(targetdir, tileRange, overpass)
-    await postProcess(targetdir, tileRange)
+    await postProcess(targetdir, tileRange, theme)
 }