Feature: layer validation system now builds a dependency graph and only updates what is needed, makes "refresh:layeroverview" redundant

assets/layers/toilet_at_amenity/toilet_at_amenity.json

@@ -141,11 +141,12 @@
   "lineRendering": [],
   "tagRenderings": [
     {
+      "id": "images",
       "render": {
         "special": {
           "before": "{image_carousel(toilets:panoramax;toilets:mapillary;toilets:images)}",
           "type": "image_upload",
-          "key": "toilets:panoramax",
+          "image_key": "toilets:panoramax",
           "label": {
             "en": "Add a picture of the toilets",
             "nl": "Voeg een foto van de toiletten toe"
@@ -153,19 +154,6 @@
         }
       }
     },
-    {
-      "id": "dead",
-      "labels": [
-        "hidden",
-        "relevant_questions"
-      ],
-      "condition": {
-        "and": [
-          "id="
-        ]
-      },
-      "render": "Only used to make sure 'relevant-questions' is known"
-    },
     {
       "builtin": "toilet.amenity-no-prefix",
       "prefix": "toilets",

assets/layers/toilet_at_amenity_lib/toilet_at_amenity_lib.json

@@ -18,6 +18,11 @@
     },
     {
       "id": "grouptitle",
+      "labels": [
+        "all",
+        "hidden"
+      ],
+      "icon": "./assets/layers/toilet/toilets.svg",
       "render": {
         "en": "Toilet information",
         "nl": "Informatie over de toiletten"
@@ -61,7 +66,8 @@
       "id": "toilet-question-box",
       "labels": [
         "toilet-questions",
-        "all"
+        "all",
+        "hidden"
       ],
       "render": {
         "special": {

package.json

@@ -98,16 +98,15 @@
     "reset:translations": "vite-node scripts/generateTranslations.ts -- --ignore-weblate",
     "generate:layouts": "export NODE_OPTIONS=\"--max-old-space-size=8192\" &&  vite-node scripts/generateLayouts.ts",
     "generate:docs": "rm -rf Docs/Themes/* && rm -rf Docs/Layers/* && rm -rf Docs/TagInfo && mkdir Docs/TagInfo && export NODE_OPTIONS=\"--max-old-space-size=16000\" && vite-node scripts/generateDocs.ts && vite-node scripts/generateTaginfoProjectFiles.ts",
-    "generate:layeroverview": "export NODE_OPTIONS=\"--max-old-space-size=16000\" && vite-node scripts/generateLayerOverview.ts",
     "generate:mapcomplete-changes-theme": "export NODE_OPTIONS=\"--max-old-space-size=8192\" && vite-node scripts/generateLayerOverview.ts -- --generate-change-map",
-    "refresh:layeroverview": "export NODE_OPTIONS=\"--max-old-space-size=8192\" && vite-node scripts/generateLayerOverview.ts -- --force",
     "generate:licenses": "vite-node scripts/generateLicenseInfo.ts -- --no-fail",
+    "generate:layeroverview": "export NODE_OPTIONS=\"--max-old-space-size=16000\" && vite-node scripts/generateLayerOverview.ts",
+    "prep:layeroverview": "./scripts/initFiles.sh",
+    "reset:layeroverview": "npm run prep:layeroverview && npm run generate:layeroverview",
     "query:licenses": "vite-node scripts/generateLicenseInfo.ts -- --query && npm run generate:licenses",
     "clean:licenses": "find . -type f -name \"*.license\" -exec rm -f {} +",
     "generate:contributor-list": "vite-node scripts/generateContributors.ts",
     "generate:service-worker": "tsc src/service-worker.ts --outFile public/service-worker.js && git_hash=$(git rev-parse HEAD) && sed -i.bak \"s/GITHUB-COMMIT/$git_hash/\" public/service-worker.js && rm public/service-worker.js.bak",
-    "reset:layeroverview": "npm run prep:layeroverview && npm run generate:layeroverview && npm run refresh:layeroverview",
-    "prep:layeroverview": "./scripts/initFiles.sh",
     "generate": "npm run generate:licenses && npm run generate:images && npm run generate:charging-stations && npm run generate:translations && npm run refresh:layeroverview && npm run generate:service-worker",
     "generate:charging-stations": "cd ./assets/layers/charging_station && vite-node csvToJson.ts && cd -",
     "clean:tests": "find . -type f -name \"*.doctest.ts\" | xargs -r rm",
@@ -224,8 +223,7 @@
     "latlon2country": "^1.2.7",
     "libphonenumber-js": "^1.11.19",
     "mangrove-reviews-typescript": "^1.3.1",
-    "maplibre": "^0.0.1-security",
+    "maplibre-gl": "^5.1.0",
-    "maplibre-gl": "^5.1.0  ",
     "marked": "^12.0.2",
     "monaco-editor": "^0.46.0",
     "mvt-to-geojson": "^0.0.6",

scripts/generateFavouritesLayer.ts

@@ -6,13 +6,14 @@ import { AllKnownLayoutsLazy } from "../src/Customizations/AllKnownLayouts"
 import { Utils } from "../src/Utils"
 import {
     MappingConfigJson,
-    QuestionableTagRenderingConfigJson,
+    QuestionableTagRenderingConfigJson
 } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
 import { TagConfigJson } from "../src/Models/ThemeConfig/Json/TagConfigJson"
 import { TagUtils } from "../src/Logic/Tags/TagUtils"
 import { TagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/TagRenderingConfigJson"
 import { Translatable } from "../src/Models/ThemeConfig/Json/Translatable"
 import * as questions from "../assets/layers/questions/questions.json"
+
 export class GenerateFavouritesLayer extends Script {
     private readonly layers: LayerConfigJson[] = []
 
@@ -202,7 +203,7 @@ export class GenerateFavouritesLayer extends Script {
             string,
             TagRenderingConfigJson[]
         >()
-        const path = "./src/assets/generated/layers/icons.json"
+        const path = "./public/assets/generated/layers/icons.json"
         if (existsSync(path)) {
             const config = <LayerConfigJson>JSON.parse(readFileSync(path, "utf8"))
             for (const tagRendering of config.tagRenderings) {

scripts/generateLayerOverview.ts

@@ -9,16 +9,12 @@ import {
     DoesImageExist,
     PrevalidateTheme,
     ValidateLayer,
-    ValidateThemeEnsemble,
+    ValidateThemeEnsemble
 } from "../src/Models/ThemeConfig/Conversion/Validation"
 import { Translation } from "../src/UI/i18n/Translation"
 import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
 import { PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme"
-import {
+import { Conversion, DesugaringContext, DesugaringStep } from "../src/Models/ThemeConfig/Conversion/Conversion"
-    Conversion,
-    DesugaringContext,
-    DesugaringStep,
-} from "../src/Models/ThemeConfig/Conversion/Conversion"
 import { Utils } from "../src/Utils"
 import Script from "./Script"
 import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
@@ -35,6 +31,7 @@ import { Translatable } from "../src/Models/ThemeConfig/Json/Translatable"
 import { ValidateThemeAndLayers } from "../src/Models/ThemeConfig/Conversion/ValidateThemeAndLayers"
 import { ExtractImages } from "../src/Models/ThemeConfig/Conversion/FixImages"
 import { TagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/TagRenderingConfigJson"
+import { LayerConfigDependencyGraph, LevelInfo } from "../src/Models/ThemeConfig/LayerConfigDependencyGraph"
 
 // This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files.
 // It spits out an overview of those to be used to load them
@@ -138,8 +135,171 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye
     }
 }
 
+class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
+    private readonly _dependencies: ReadonlyMap<string, string[]>
+    private readonly _states: Map<string, "clean" | "dirty" | "changed">
+    private readonly prepareLayer: PrepareLayer
+    private readonly _levels: LevelInfo[]
+    private readonly _loadedIds: Set<string> = new Set<string>()
+    private readonly _layerConfigJsons = new Map<string, LayerConfigJson>
+    private readonly _desugaringState: DesugaringContext
+
+    constructor(
+        layerConfigJsons: LayerConfigJson[],
+        dependencies: Map<string, string[]>,
+        levels: LevelInfo[],
+        states: Map<string, "clean" | "dirty" | "changed">,
+        sharedTagRenderings: QuestionableTagRenderingConfigJson[]) {
+        super("Builds all the layers, writes them to file", [], "LayerBuilder")
+        this._levels = levels
+        this._dependencies = dependencies
+        this._states = states
+        this._desugaringState = {
+            tagRenderings: LayerOverviewUtils.asDict(sharedTagRenderings),
+            tagRenderingOrder: sharedTagRenderings.map((tr) => tr.id),
+            sharedLayers: AllSharedLayers.getSharedLayersConfigs()
+        }
+        this.prepareLayer = new PrepareLayer(this._desugaringState)
+        for (const layerConfigJson of layerConfigJsons) {
+            this._layerConfigJsons.set(layerConfigJson.id, layerConfigJson)
+        }
+    }
+
+    public static targetPath(id: string): string {
+        return `${LayerOverviewUtils.layerPath}${id}.json`
+    }
+
+    public static sourcePath(id: string): string {
+        return `./assets/layers/${id}/${id}.json`
+    }
+
+    writeLayer(layer: LayerConfigJson) {
+        if (!existsSync(LayerOverviewUtils.layerPath)) {
+            mkdirSync(LayerOverviewUtils.layerPath)
+        }
+        writeFileSync(
+            LayerBuilder.targetPath(layer.id),
+            JSON.stringify(layer, null, "  "),
+            { encoding: "utf8" }
+        )
+    }
+
+
+    public buildLayer(id: string, context: ConversionContext, isLooping: boolean = false): LayerConfigJson {
+        if (id === "questions") {
+            return undefined
+        }
+        const deps = this._dependencies.get(id)
+        if (!isLooping) {
+            // Beware of the looping traps. Bring the leaf to the statue to teleport to "The Lab" (<ref>submachine 4</ref>)
+            const unbuilt = deps.filter(depId => !this._loadedIds.has(depId))
+            for (const unbuiltId of unbuilt) {
+                this.buildLayer(unbuiltId, context)
+            }
+        }
+
+        context = context.inOperation("building Layer " + id).enters("layer", id)
+
+        const config = this._layerConfigJsons.get(id)
+        const prepped = this.prepareLayer.convert(config, context)
+        this._loadedIds.add(id)
+        this._desugaringState.sharedLayers.set(id, prepped)
+        return prepped
+    }
+
+    private buildLooping(ids: string[], context: ConversionContext) {
+        const origIds: ReadonlyArray<string> = [...ids]
+
+        const deps = this._dependencies
+        const allDeps = Utils.Dedup([].concat(...ids.map(id => deps.get(id))))
+        const depsRecord = Utils.asRecord(Array.from(deps.keys()), k =>
+            deps.get(k).filter(dep => ids.indexOf(dep) >= 0))
+        const revDeps = Utils.TransposeMap(depsRecord)
+        for (const someDep of allDeps) {
+            if (ids.indexOf(someDep) >= 0) {
+                // BY definition, we _will_ need this dependency
+                // We add a small stub
+                this._desugaringState.sharedLayers.set(someDep, {
+                    id: someDep,
+                    pointRendering: [],
+                    tagRenderings: [],
+                    filter: [],
+                    source: "special:stub",
+                    allowMove: true
+                })
+                continue
+            }
+            // Make sure all are direct dependencies are loaded
+            if (!this._loadedIds.has(someDep)) {
+                this.buildLayer(someDep, context)
+            }
+        }
+        while (ids.length > 0) {
+            const first = ids.pop()
+            if (first === "questions") {
+                continue
+            }
+            const oldConfig = this._desugaringState.sharedLayers.get(first) ?? this._layerConfigJsons.get(first)
+            const newConfig = this.buildLayer(first, context.inOperation("resolving a looped dependency"), true)
+            const isDifferent = JSON.stringify(oldConfig) !== JSON.stringify(newConfig)
+
+            if (isDifferent) {
+                const toRunAgain = revDeps[first] ?? []
+                for (const id of toRunAgain) {
+                    if (ids.indexOf(id) < 0) {
+                        ids.push(id)
+                    }
+                }
+            }
+        }
+        for (const id of origIds) {
+            this.writeLayer(this._desugaringState.sharedLayers.get(id))
+        }
+        console.log("Done with the looping layers!")
+    }
+
+    public convert(o, context: ConversionContext):
+        Map<string, LayerConfigJson> {
+
+        for (const level of this._levels
+            ) {
+            if (level.loop) {
+                this.buildLooping(level.ids, context)
+                continue
+            }
+
+            for (let i = 0; i < level.ids.length; i++) {
+                const id = level.ids[i]
+                ScriptUtils.erasableLog(`Building level ${level.level}: validating layer ${i + 1}/${level.ids.length}: ${id}`)
+                if (id === "questions") {
+                    continue
+                }
+                if (this._states.get(id) === "clean") {
+                    const file = readFileSync(LayerBuilder.targetPath(id), "utf-8")
+                    if (file.length > 3) {
+                        try {
+                            const loaded = JSON.parse(file)
+                            this._desugaringState.sharedLayers.set(id, loaded)
+                            continue
+                        } catch (e) {
+                            console.error("Could not load generated layer file for ", id, " building it instead")
+                        }
+                    }
+                }
+                const prepped = this.buildLayer(id, context)
+                this.writeLayer(prepped)
+                LayerOverviewUtils.extractJavascriptCodeForLayer(prepped)
+            }
+        }
+        context.info("Recompiled " + this._loadedIds.size + " layers")
+        return this._desugaringState.sharedLayers
+    }
+
+
+}
+
 class LayerOverviewUtils extends Script {
-    public static readonly layerPath = "./src/assets/generated/layers/"
+    public static readonly layerPath = "./public/assets/generated/layers/"
     public static readonly themePath = "./public/assets/generated/themes/"
 
     constructor() {
@@ -190,7 +350,7 @@ class LayerOverviewUtils extends Script {
         return Translations.T(t).OnEveryLanguage((s) => parse_html(s).textContent).translations
     }
 
-    shouldBeUpdated(sourcefile: string | string[], targetfile: string): boolean {
+    public static shouldBeUpdated(sourcefile: string | string[], targetfile: string): boolean {
         if (!existsSync(targetfile)) {
             return true
         }
@@ -359,17 +519,6 @@ class LayerOverviewUtils extends Script {
         )
     }
 
-    writeLayer(layer: LayerConfigJson) {
-        if (!existsSync(LayerOverviewUtils.layerPath)) {
-            mkdirSync(LayerOverviewUtils.layerPath)
-        }
-        writeFileSync(
-            `${LayerOverviewUtils.layerPath}${layer.id}.json`,
-            JSON.stringify(layer, null, "  "),
-            { encoding: "utf8" }
-        )
-    }
-
     static asDict(
         trs: QuestionableTagRenderingConfigJson[]
     ): Map<string, QuestionableTagRenderingConfigJson> {
@@ -481,13 +630,6 @@ class LayerOverviewUtils extends Script {
                 ?.split(",") ?? []
         )
 
-        const layerWhitelist = new Set(
-            args
-                .find((a) => a.startsWith("--layers="))
-                ?.substring("--layers=".length)
-                ?.split(",") ?? []
-        )
-
         const forceReload = args.some((a) => a == "--force")
 
         const licensePaths = new Set<string>()
@@ -495,7 +637,7 @@ class LayerOverviewUtils extends Script {
             licensePaths.add(licenses[i].path)
         }
         const doesImageExist = new DoesImageExist(licensePaths, existsSync)
-        const sharedLayers = this.buildLayerIndex(doesImageExist, forceReload, layerWhitelist)
+        const sharedLayers = this.buildLayerIndex(doesImageExist)
 
         const priviliged = new Set<string>(Constants.priviliged_layers)
         sharedLayers.forEach((_, key) => {
@@ -582,9 +724,6 @@ class LayerOverviewUtils extends Script {
             )
         }
 
-        if (AllSharedLayers.getSharedLayersConfigs().size == 0) {
-            console.error("This was a bootstrapping-run. Run generate layeroverview again!")
-        }
     }
 
     private parseLayer(
@@ -606,81 +745,89 @@ class LayerOverviewUtils extends Script {
         return { ...result, context }
     }
 
+    private getAllLayerConfigs(): LayerConfigJson[] {
+        const allPaths = ScriptUtils.getLayerPaths()
+        const results: LayerConfigJson[] = []
+        for (let i = 0; i < allPaths.length; i++) {
+            const path = allPaths[i]
+            ScriptUtils.erasableLog(`Parsing layerConfig ${i + 1}/${allPaths.length}: ${path}                   `)
+            const data = JSON.parse(readFileSync(path, "utf8"))
+            results.push(data)
+        }
+
+
+        return results
+
+    }
+
     private buildLayerIndex(
-        doesImageExist: DoesImageExist,
+        doesImageExist: DoesImageExist
-        forceReload: boolean,
-        whitelist: Set<string>
     ): Map<string, LayerConfigJson> {
         // First, we expand and validate all builtin layers. These are written to src/assets/generated/layers
         // At the same time, an index of available layers is built.
-        console.log("------------- VALIDATING THE BUILTIN QUESTIONS ---------------")
+        const sharedQuestions = this.getSharedTagRenderings(doesImageExist)
-        const sharedTagRenderings = this.getSharedTagRenderings(doesImageExist)
+        const allLayerConfigs = this.getAllLayerConfigs()
-        console.log("   ---------- VALIDATING BUILTIN LAYERS ---------")
+        const sharedQuestionsDef = allLayerConfigs.find(l => l.id === "questions")
-        const state: DesugaringContext = {
+        sharedQuestionsDef.tagRenderings = sharedQuestions
-            tagRenderings: LayerOverviewUtils.asDict(sharedTagRenderings),
+
-            tagRenderingOrder: sharedTagRenderings.map((tr) => tr.id),
+
-            sharedLayers: AllSharedLayers.getSharedLayersConfigs(),
+        const dependencyGraph = LayerConfigDependencyGraph.buildDirectDependencies(allLayerConfigs)
-        }
+        const levels = LayerConfigDependencyGraph.buildLevels(dependencyGraph)
-        const sharedLayers = new Map<string, LayerConfigJson>()
+        const layerState = new Map<string, "clean" | "dirty" | "changed">()
-        const prepLayer = new PrepareLayer(state)
+        console.log("# BUILD PLAN\n\n")
-        const skippedLayers: string[] = []
+        for (const levelInfo of levels) {
-        const recompiledLayers: string[] = []
+            console.log(`## LEVEL ${levelInfo.level}${levelInfo.loop ? " (LOOP)" : ""}`)
-        let warningCount = 0
+            for (const id of levelInfo.ids) {
-        for (const sharedLayerPath of ScriptUtils.getLayerPaths()) {
+                const deps = dependencyGraph.get(id) ?? []
-            if (whitelist.size > 0) {
+                const dirtyDeps = deps.filter(dep => {
-                const idByPath = sharedLayerPath.split("/").at(-1).split(".")[0]
+                    const depState = layerState.get(dep)
-                if (!Constants.isPriviliged(idByPath) && !whitelist.has(idByPath)) {
+                    if (levelInfo.loop && depState === undefined) {
-                    continue
+                        const depIsClean =
-                }
+                            LayerOverviewUtils.shouldBeUpdated(
-            }
+                                LayerBuilder.sourcePath(dep),
-            {
+                                LayerBuilder.targetPath(dep))
-                const targetPath =
+                        if (depIsClean) {
-                    LayerOverviewUtils.layerPath +
+                            return false
-                    sharedLayerPath.substring(sharedLayerPath.lastIndexOf("/"))
+                        }
-                if (!forceReload && !this.shouldBeUpdated(sharedLayerPath, targetPath)) {
+                    }
-                    try {
+                    return depState !== "clean"
-                        const sharedLayer = JSON.parse(readFileSync(targetPath, "utf8"))
+                })
-                        sharedLayers.set(sharedLayer.id, sharedLayer)
+                if (dirtyDeps.length > 0) {
-                        skippedLayers.push(sharedLayer.id)
+                    layerState.set(id, "dirty")
-                        continue
+                } else {
-                    } catch (e) {
+
-                        throw "Could not parse " + targetPath + " : " + e
+
+                    const sourcePath = `./assets/layers/${id}/${id}.json`
+                    const targetPath = `./public/assets/generated/layers/${id}.json`
+
+                    if (id === "questions") {
+                        layerState.set(id, "clean")
+                    } else if (LayerOverviewUtils.shouldBeUpdated(sourcePath, targetPath)) {
+                        layerState.set(id, "changed")
+                    } else {
+                        layerState.set(id, "clean")
                     }
                 }
+                const state = layerState.get(id)
+                console.log(`- ${id} (${state}; ${dirtyDeps.map(dd => dd + "*").join(", ")})`)
             }
-
-            const parsed = this.parseLayer(doesImageExist, prepLayer, sharedLayerPath)
-            warningCount += parsed.context.getAll("warning").length
-            const fixed = parsed.raw
-            if (sharedLayers.has(fixed.id)) {
-                throw "There are multiple layers with the id " + fixed.id + ", " + sharedLayerPath
-            }
-            if (parsed.context.hasErrors()) {
-                throw "Some layers contain errors"
-            }
-
-            sharedLayers.set(fixed.id, fixed)
-            recompiledLayers.push(fixed.id)
-
-            this.writeLayer(fixed)
         }
 
-        console.log(
+
-            "Recompiled layers " +
+        const builder = new LayerBuilder(allLayerConfigs, dependencyGraph, levels, layerState, sharedQuestions)
-                recompiledLayers.join(", ") +
+        builder.writeLayer(sharedQuestionsDef)
-                " and skipped " +
+        const allLayers = builder.convertStrict({}, ConversionContext.construct([], []))
-                skippedLayers.length +
+        if (layerState.get("usersettings") !== "clean") {
-                " layers. Detected " +
+            // We always need the calculated tags of 'usersettings', so we export them separately if dirty
-                warningCount +
+
-                " warnings"
+            LayerOverviewUtils.extractJavascriptCodeForLayer(
-        )
+                allLayers.get("usersettings"),
-        // We always need the calculated tags of 'usersettings', so we export them separately
-        this.extractJavascriptCodeForLayer(
-            state.sharedLayers.get("usersettings"),
             "./src/Logic/State/UserSettingsMetaTagging.ts"
         )
+        }
+
+        return allLayers
 
-        return sharedLayers
     }
 
     /**
@@ -741,7 +888,7 @@ class LayerOverviewUtils extends Script {
         writeFileSync(targetDir + themeFile.id + ".ts", allCode.join("\n"))
     }
 
-    private extractJavascriptCodeForLayer(l: LayerConfigJson, targetPath?: string) {
+    public static extractJavascriptCodeForLayer(l: LayerConfigJson, targetPath?: string) {
         if (!l) {
             return // Probably a bootstrapping run
         }
@@ -858,7 +1005,7 @@ class LayerOverviewUtils extends Script {
                 LayerOverviewUtils.extractLayerIdsFrom(themeFile, false)
             ).map((id) => LayerOverviewUtils.layerPath + id + ".json")
 
-            if (!forceReload && !this.shouldBeUpdated([themePath, ...usedLayers], targetPath)) {
+            if (!forceReload && !LayerOverviewUtils.shouldBeUpdated([themePath, ...usedLayers], targetPath)) {
                 fixed.set(
                     themeFile.id,
                     JSON.parse(

scripts/initFiles.sh

@@ -5,12 +5,12 @@
 mkdir -p ./src/assets/generated/layers
 mkdir -p ./public/assets/generated/themes
 echo '{"layers": []}' > ./src/assets/generated/known_layers.json
-rm -f ./src/assets/generated/layers/*.json
+rm -f ./public/assets/generated/layers/*.json
 rm -f ./public/assets/generated/themes/*.json
-cp ./assets/layers/usersettings/usersettings.json ./src/assets/generated/layers/usersettings.json
+echo '{}' > ./public/assets/generated/layers/favourite.json
-echo '{}' > ./src/assets/generated/layers/favourite.json
+echo '{}' > ./public/assets/generated/layers/summary.json
-echo '{}' > ./src/assets/generated/layers/summary.json
+echo '{}' > ./public/assets/generated/layers/last_click.json
-echo '{}' > ./src/assets/generated/layers/last_click.json
+echo '{}' > ./public/assets/generated/layers/search.json
-echo '{}' > ./src/assets/generated/layers/search.json
+echo '[]' > ./public/assets/generated/theme_overview.json
-echo '[]' > ./src/assets/generated/theme_overview.json
+echo '{}' > ./public/assets/generated/layers/geocoded_image.json
-echo '{}' > ./src/assets/generated/layers/geocoded_image.json
+echo '{}' > ./public/assets/generated/layers/usersettings.json

scripts/prepare-build.sh

@@ -20,8 +20,6 @@ npm run download:editor-layer-index &&
 npm run prep:layeroverview &&
 npm run generate && # includes a single "refresh:layeroverview". Resetting the files is unnecessary as they are not in there in the first place
 npm run generate:mapcomplete-changes-theme  &&
-npm run refresh:layeroverview && #  a second time to propagate all calls
-npm run refresh:layeroverview &&  # a third time to fix some issues with the favourite layer all calls
 npm run generate:layouts
 
 if [ $? -ne 0 ]; then

src/Customizations/AllKnownLayouts.ts

@@ -1,5 +1,5 @@
 import ThemeConfig from "../Models/ThemeConfig/ThemeConfig"
-import favourite from "../assets/generated/layers/favourite.json"
+import favourite from "../../public/assets/generated/layers/favourite.json"
 import { ThemeConfigJson } from "../Models/ThemeConfig/Json/ThemeConfigJson"
 import { AllSharedLayers } from "./AllSharedLayers"
 import Constants from "../Models/Constants"

src/Logic/DetermineTheme.ts

@@ -9,7 +9,7 @@ import { PrepareTheme } from "../Models/ThemeConfig/Conversion/PrepareTheme"
 import licenses from "../assets/generated/license_info.json"
 import TagRenderingConfig from "../Models/ThemeConfig/TagRenderingConfig"
 import { FixImages } from "../Models/ThemeConfig/Conversion/FixImages"
-import questions from "../assets/generated/layers/questions.json"
+import questions from "../../public/assets/generated/layers/questions.json"
 import { DoesImageExist, PrevalidateTheme } from "../Models/ThemeConfig/Conversion/Validation"
 import { DesugaringContext } from "../Models/ThemeConfig/Conversion/Conversion"
 import { TagRenderingConfigJson } from "../Models/ThemeConfig/Json/TagRenderingConfigJson"

src/Logic/Search/GeocodingProvider.ts

@@ -2,7 +2,7 @@ import { BBox } from "../BBox"
 import { Feature, Geometry } from "geojson"
 import { DefaultPinIcon } from "../../Models/Constants"
 import { Store } from "../UIEventSource"
-import * as search from "../../assets/generated/layers/search.json"
+import * as search from "../../../public/assets/generated/layers/search.json"
 import LayerConfig from "../../Models/ThemeConfig/LayerConfig"
 import { LayerConfigJson } from "../../Models/ThemeConfig/Json/LayerConfigJson"
 import { GeoOperations } from "../GeoOperations"

src/Logic/State/UserRelatedState.ts

@@ -10,7 +10,7 @@ import translators from "../../assets/translators.json"
 import codeContributors from "../../assets/contributors.json"
 import LayerConfig from "../../Models/ThemeConfig/LayerConfig"
 import { LayerConfigJson } from "../../Models/ThemeConfig/Json/LayerConfigJson"
-import usersettings from "../../../src/assets/generated/layers/usersettings.json"
+import usersettings from "../../../public/assets/generated/layers/usersettings.json"
 import Locale from "../../UI/i18n/Locale"
 import LinkToWeblate from "../../UI/Base/LinkToWeblate"
 import FeatureSwitchState from "./FeatureSwitchState"

src/Logic/State/UserSettingsMetaTagging.ts

@@ -1,42 +1,14 @@
 import { Utils } from "../../Utils"
 /** This code is autogenerated - do not edit. Edit ./assets/layers/usersettings/usersettings.json instead */
 export class ThemeMetaTagging {
-    public static readonly themeName = "usersettings"
+   public static readonly themeName = "usersettings"
 
-    public metaTaggging_for_usersettings(feat: { properties: Record<string, string> }) {
+   public metaTaggging_for_usersettings(feat: {properties: Record<string, string>}) {
-        Utils.AddLazyProperty(feat.properties, "_mastodon_candidate_md", () =>
+      Utils.AddLazyProperty(feat.properties, '_mastodon_candidate_md', () => feat.properties._description.match(/\[[^\]]*\]\((.*(mastodon|en.osm.town).*)\).*/)?.at(1) ) 
-            feat.properties._description
+      Utils.AddLazyProperty(feat.properties, '_d', () => feat.properties._description?.replace(/&lt;/g,'<')?.replace(/&gt;/g,'>') ?? '' ) 
-                .match(/\[[^\]]*\]\((.*(mastodon|en.osm.town).*)\).*/)
+      Utils.AddLazyProperty(feat.properties, '_mastodon_candidate_a', () => (feat => {const e = document.createElement('div');e.innerHTML = feat.properties._d;return Array.from(e.getElementsByTagName("a")).filter(a => a.href.match(/mastodon|en.osm.town/) !== null)[0]?.href   }) (feat)  ) 
-                ?.at(1)
+      Utils.AddLazyProperty(feat.properties, '_mastodon_link', () => (feat => {const e = document.createElement('div');e.innerHTML = feat.properties._d;return Array.from(e.getElementsByTagName("a")).filter(a => a.getAttribute("rel")?.indexOf('me') >= 0)[0]?.href})(feat)  ) 
-        )
+      Utils.AddLazyProperty(feat.properties, '_mastodon_candidate', () => feat.properties._mastodon_candidate_md ?? feat.properties._mastodon_candidate_a ) 
-        Utils.AddLazyProperty(
+      feat.properties['__current_backgroun'] = 'initial_value'
-            feat.properties,
+   }
-            "_d",
+}
-            () => feat.properties._description?.replace(/&lt;/g, "<")?.replace(/&gt;/g, ">") ?? ""
-        )
-        Utils.AddLazyProperty(feat.properties, "_mastodon_candidate_a", () =>
-            ((feat) => {
-                const e = document.createElement("div")
-                e.innerHTML = feat.properties._d
-                return Array.from(e.getElementsByTagName("a")).filter(
-                    (a) => a.href.match(/mastodon|en.osm.town/) !== null
-                )[0]?.href
-            })(feat)
-        )
-        Utils.AddLazyProperty(feat.properties, "_mastodon_link", () =>
-            ((feat) => {
-                const e = document.createElement("div")
-                e.innerHTML = feat.properties._d
-                return Array.from(e.getElementsByTagName("a")).filter(
-                    (a) => a.getAttribute("rel")?.indexOf("me") >= 0
-                )[0]?.href
-            })(feat)
-        )
-        Utils.AddLazyProperty(
-            feat.properties,
-            "_mastodon_candidate",
-            () => feat.properties._mastodon_candidate_md ?? feat.properties._mastodon_candidate_a
-        )
-        feat.properties["__current_backgroun"] = "initial_value"
-    }
-}

src/Models/ThemeConfig/Conversion/ExpandFilter.ts

@@ -239,11 +239,11 @@ export class ExpandFilter extends DesugaringStep<LayerConfigJson> {
                     )
                 }
                 const layer = this._state.sharedLayers.get(split[0])
-                if (layer === undefined) {
+                if (!layer) {
                     context.err("Layer '" + split[0] + "' not found")
                 }
                 const expectedId = split[1]
-                const expandedFilter = (<(FilterConfigJson | string)[]>layer.filter).find(
+                const expandedFilter = (<(FilterConfigJson | string)[]>layer?.filter)?.find(
                     (f) => typeof f !== "string" && f.id === expectedId
                 )
                 if (expandedFilter === undefined) {

src/Models/ThemeConfig/Conversion/ExpandTagRendering.ts

@@ -380,15 +380,10 @@ export class ExpandTagRendering extends Conversion<
                                 Utils.NoNull(Array.from(state.sharedLayers.keys())),
                                 (s) => s
                             )
-                            if (state.sharedLayers.size === 0) {
+                            if (candidates.length === 0) {
-                                ctx.warn(
+                                ctx.err("While reusing a tagRendering: " + name + "; no candidates in layer " + layerName)
-                                    "BOOTSTRAPPING. Rerun generate layeroverview. While reusing tagrendering: " +
-                                        name +
-                                        ": layer " +
-                                        layerName +
-                                        " not found for now, but ignoring as this is a bootstrapping run. "
-                                )
                             } else {
+                                console.error("Bench was not found...")
                                 ctx.err(
                                     ": While reusing tagrendering: " +
                                         name +
@@ -400,10 +395,15 @@ export class ExpandTagRendering extends Conversion<
                             }
                             continue
                         }
+                        if (layer.source === "special:stub") {
+                            // We are dealing with a looping import, no error is necessary
+                            continue
+                        }
                         candidates = Utils.NoNull(layer.tagRenderings.map((tr) => tr["id"])).map(
                             (id) => layerName + "." + id
                         )
                     }
+
                     candidates = Utils.sortedByLevenshteinDistance(name, candidates, (i) => i)
                     ctx.err(
                         "The tagRendering with identifier " +

src/Models/ThemeConfig/Conversion/PrepareLayer.ts

@@ -1048,6 +1048,11 @@ export class PrepareLayer extends Fuse<LayerConfigJson> {
         if (json === undefined || json === null) {
             throw "Error: prepareLayer got null"
         }
+        if (json.source?.["osmTags"] !== undefined && json.source?.["osmTags"]?.["and"] === undefined) {
+            json = { ...json }
+            json.source = <any>{ ...(<object>json.source) }
+            json.source["osmTags"] = { "and": [json.source["osmTags"]] }
+        }
         return super.convert(json, context)
     }
 }

src/Models/ThemeConfig/Json/LayerConfigJson.ts

@@ -78,6 +78,7 @@ export interface LayerConfigJson {
         | undefined
         | "special"
         | "special:library"
+        | "special:stub" // only used when building looping imports
         | {
               /**
                * question: Which tags must be present on the feature to show it in this layer?

src/Models/ThemeViewState/WithSpecialLayers.ts

@@ -11,14 +11,14 @@ import MetaTagging from "../../Logic/MetaTagging"
 import FilteredLayer from "../FilteredLayer"
 import LayerConfig from "../ThemeConfig/LayerConfig"
 import { LayerConfigJson } from "../ThemeConfig/Json/LayerConfigJson"
-import last_click_layerconfig from "../../assets/generated/layers/last_click.json"
+import last_click_layerconfig from "../../../public/assets/generated/layers/last_click.json"
 import { GeoOperations } from "../../Logic/GeoOperations"
-import summaryLayer from "../../assets/generated/layers/summary.json"
+import summaryLayer from "../../../public/assets/generated/layers/summary.json"
 import { Store, UIEventSource } from "../../Logic/UIEventSource"
 import NearbyFeatureSource from "../../Logic/FeatureSource/Sources/NearbyFeatureSource"
 import {
     SummaryTileSource,
-    SummaryTileSourceRewriter,
+    SummaryTileSourceRewriter
 } from "../../Logic/FeatureSource/TiledFeatureSource/SummaryTileSource"
 import { ShowDataLayerOptions } from "../../UI/Map/ShowDataLayerOptions"
 
@@ -195,7 +195,6 @@ export class WithSpecialLayers extends WithChangesState {
                   | "range" // handled by UserMapFeatureSwitchState
                   | "selected_element" // handled by this.drawSelectedElement
               >
-        const empty = []
         /**
          * A listing which maps the layerId onto the featureSource
          */

src/UI/BigComponents/MenuDrawer.svelte

@@ -26,7 +26,7 @@
   import SelectedElementView from "./SelectedElementView.svelte"
   import LayerConfig from "../../Models/ThemeConfig/LayerConfig"
   import type { LayerConfigJson } from "../../Models/ThemeConfig/Json/LayerConfigJson"
-  import usersettings from "../../assets/generated/layers/usersettings.json"
+  import usersettings from "../../../public/assets/generated/layers/usersettings.json"
   import UserRelatedState from "../../Logic/State/UserRelatedState"
   import ArrowDownTray from "@babeard/svelte-heroicons/mini/ArrowDownTray"
   import DownloadPanel from "../DownloadFlow/DownloadPanel.svelte"

src/UI/Image/NearbyImages.svelte

@@ -19,7 +19,7 @@
   import { MapLibreAdaptor } from "../Map/MapLibreAdaptor"
   import ShowDataLayer from "../Map/ShowDataLayer"
   import StaticFeatureSource from "../../Logic/FeatureSource/Sources/StaticFeatureSource"
-  import * as geocoded_image from "../../assets/generated/layers/geocoded_image.json"
+  import * as geocoded_image from "../../../public/assets/generated/layers/geocoded_image.json"
   import type { LayerConfigJson } from "../../Models/ThemeConfig/Json/LayerConfigJson"
   import { onDestroy } from "svelte"
   import { BBox } from "../../Logic/BBox"