Merge master

src/Models/ThemeConfig/Conversion/AddContextToTranslations.ts

@@ -0,0 +1,157 @@
+import { DesugaringStep } from "./Conversion"
+import { Utils } from "../../../Utils"
+import Translations from "../../../UI/i18n/Translations"
+
+export class AddContextToTranslations<T> extends DesugaringStep<T> {
+    private readonly _prefix: string
+
+    constructor(prefix = "") {
+        super(
+            "Adds a '_context' to every object that is probably a translation",
+            ["_context"],
+            "AddContextToTranslation"
+        )
+        this._prefix = prefix
+    }
+
+    /**
+     * const theme = {
+     *   layers: [
+     *       {
+     *           builtin: ["abc"],
+     *           override: {
+     *               title:{
+     *                   en: "Some title"
+     *               }
+     *           }
+     *       }
+     *   ]
+     * }
+     * const rewritten = new AddContextToTranslations<any>("prefix:").convert(theme, "context").result
+     * const expected = {
+     *   layers: [
+     *       {
+     *           builtin: ["abc"],
+     *           override: {
+     *               title:{
+     *                  _context: "prefix:context.layers.0.override.title"
+     *                   en: "Some title"
+     *               }
+     *           }
+     *       }
+     *   ]
+     * }
+     * rewritten // => expected
+     *
+     * // should use the ID if one is present instead of the index
+     * const theme = {
+     *   layers: [
+     *       {
+     *           tagRenderings:[
+     *               {id: "some-tr",
+     *               question:{
+     *                   en:"Question?"
+     *               }
+     *               }
+     *           ]
+     *       }
+     *   ]
+     * }
+     * const rewritten = new AddContextToTranslations<any>("prefix:").convert(theme, "context").result
+     * const expected = {
+     *   layers: [
+     *       {
+     *           tagRenderings:[
+     *               {id: "some-tr",
+     *               question:{
+     *                  _context: "prefix:context.layers.0.tagRenderings.some-tr.question"
+     *                   en:"Question?"
+     *               }
+     *               }
+     *           ]
+     *       }
+     *   ]
+     * }
+     * rewritten // => expected
+     *
+     * // should preserve nulls
+     * const theme = {
+     *   layers: [
+     *       {
+     *           builtin: ["abc"],
+     *           override: {
+     *               name:null
+     *           }
+     *       }
+     *   ]
+     * }
+     * const rewritten = new AddContextToTranslations<any>("prefix:").convert(theme, "context").result
+     * const expected = {
+     *   layers: [
+     *       {
+     *           builtin: ["abc"],
+     *           override: {
+     *               name: null
+     *           }
+     *       }
+     *   ]
+     * }
+     * rewritten // => expected
+     *
+     *
+     * // Should ignore all if '#dont-translate' is set
+     * const theme = {
+     *  "#dont-translate": "*",
+     *   layers: [
+     *       {
+     *           builtin: ["abc"],
+     *           override: {
+     *               title:{
+     *                   en: "Some title"
+     *               }
+     *           }
+     *       }
+     *   ]
+     * }
+     * const rewritten = new AddContextToTranslations<any>("prefix:").convert(theme, "context").result
+     * rewritten // => theme
+     *
+     */
+    convert(
+        json: T,
+        context: string
+    ): { result: T; errors?: string[]; warnings?: string[]; information?: string[] } {
+        if (json["#dont-translate"] === "*") {
+            return { result: json }
+        }
+
+        const result = Utils.WalkJson(
+            json,
+            (leaf, path) => {
+                if (leaf === undefined || leaf === null) {
+                    return leaf
+                }
+                if (typeof leaf === "object") {
+                    // follow the path. If we encounter a number, check that there is no ID we can use instead
+                    let breadcrumb = json
+                    for (let i = 0; i < path.length; i++) {
+                        const pointer = path[i]
+                        breadcrumb = breadcrumb[pointer]
+                        if (pointer.match("[0-9]+") && breadcrumb["id"] !== undefined) {
+                            path[i] = breadcrumb["id"]
+                        }
+                    }
+
+                    return { ...leaf, _context: this._prefix + context + "." + path.join(".") }
+                } else {
+                    return leaf
+                }
+            },
+            (obj) => obj === undefined || obj === null || Translations.isProbablyATranslation(obj)
+        )
+
+        return {
+            result,
+        }
+    }
+}

src/Models/ThemeConfig/Conversion/Conversion.ts

@@ -0,0 +1,346 @@
+import { TagRenderingConfigJson } from "../Json/TagRenderingConfigJson"
+import { LayerConfigJson } from "../Json/LayerConfigJson"
+import { Utils } from "../../../Utils"
+
+export interface DesugaringContext {
+    tagRenderings: Map<string, TagRenderingConfigJson>
+    sharedLayers: Map<string, LayerConfigJson>
+    publicLayers?: Set<string>
+}
+
+export abstract class Conversion<TIn, TOut> {
+    public readonly modifiedAttributes: string[]
+    public readonly name: string
+    protected readonly doc: string
+
+    constructor(doc: string, modifiedAttributes: string[] = [], name: string) {
+        this.modifiedAttributes = modifiedAttributes
+        this.doc = doc + "\n\nModified attributes are\n" + modifiedAttributes.join(", ")
+        this.name = name
+    }
+
+    public static strict<T>(fixed: {
+        errors?: string[]
+        warnings?: string[]
+        information?: string[]
+        result?: T
+    }): T {
+        fixed.information?.forEach((i) => console.log("    ", i))
+        const yellow = (s) => "\x1b[33m" + s + "\x1b[0m"
+        const red = (s) => "\x1b[31m" + s + "\x1b[0m"
+        fixed.warnings?.forEach((w) => console.warn(red(`<!> `), yellow(w)))
+
+        if (fixed?.errors !== undefined && fixed?.errors?.length > 0) {
+            fixed.errors?.forEach((e) => console.error(red(`ERR ` + e)))
+            throw "Detected one or more errors, stopping now"
+        }
+
+        return fixed.result
+    }
+
+    public convertStrict(json: TIn, context: string): TOut {
+        const fixed = this.convert(json, context)
+        return DesugaringStep.strict(fixed)
+    }
+
+    public convertJoin(
+        json: TIn,
+        context: string,
+        errors: string[],
+        warnings?: string[],
+        information?: string[]
+    ): TOut {
+        const fixed = this.convert(json, context)
+        errors?.push(...(fixed.errors ?? []))
+        warnings?.push(...(fixed.warnings ?? []))
+        information?.push(...(fixed.information ?? []))
+        return fixed.result
+    }
+
+    public andThenF<X>(f: (tout: TOut) => X): Conversion<TIn, X> {
+        return new Pipe(this, new Pure(f))
+    }
+
+    abstract convert(
+        json: TIn,
+        context: string
+    ): { result: TOut; errors?: string[]; warnings?: string[]; information?: string[] }
+}
+
+export abstract class DesugaringStep<T> extends Conversion<T, T> {}
+
+class Pipe<TIn, TInter, TOut> extends Conversion<TIn, TOut> {
+    private readonly _step0: Conversion<TIn, TInter>
+    private readonly _step1: Conversion<TInter, TOut>
+
+    constructor(step0: Conversion<TIn, TInter>, step1: Conversion<TInter, TOut>) {
+        super("Merges two steps with different types", [], `Pipe(${step0.name}, ${step1.name})`)
+        this._step0 = step0
+        this._step1 = step1
+    }
+
+    convert(
+        json: TIn,
+        context: string
+    ): { result: TOut; errors?: string[]; warnings?: string[]; information?: string[] } {
+        const r0 = this._step0.convert(json, context)
+        const { result, errors, information, warnings } = r0
+        if (result === undefined && errors.length > 0) {
+            return {
+                ...r0,
+                result: undefined,
+            }
+        }
+
+        const r = this._step1.convert(result, context)
+        Utils.PushList(errors, r.errors)
+        Utils.PushList(warnings, r.warnings)
+        Utils.PushList(information, r.information)
+        return {
+            result: r.result,
+            errors,
+            warnings,
+            information,
+        }
+    }
+}
+
+class Pure<TIn, TOut> extends Conversion<TIn, TOut> {
+    private readonly _f: (t: TIn) => TOut
+
+    constructor(f: (t: TIn) => TOut) {
+        super("Wrapper around a pure function", [], "Pure")
+        this._f = f
+    }
+
+    convert(
+        json: TIn,
+        context: string
+    ): { result: TOut; errors?: string[]; warnings?: string[]; information?: string[] } {
+        return { result: this._f(json) }
+    }
+}
+
+export class Each<X, Y> extends Conversion<X[], Y[]> {
+    private readonly _step: Conversion<X, Y>
+
+    constructor(step: Conversion<X, Y>) {
+        super(
+            "Applies the given step on every element of the list",
+            [],
+            "OnEach(" + step.name + ")"
+        )
+        this._step = step
+    }
+
+    convert(
+        values: X[],
+        context: string
+    ): { result: Y[]; errors?: string[]; warnings?: string[]; information?: string[] } {
+        if (values === undefined || values === null) {
+            return { result: undefined }
+        }
+        const information: string[] = []
+        const warnings: string[] = []
+        const errors: string[] = []
+        const step = this._step
+        const result: Y[] = []
+        for (let i = 0; i < values.length; i++) {
+            const r = step.convert(values[i], context + "[" + i + "]")
+            Utils.PushList(information, r.information)
+            Utils.PushList(warnings, r.warnings)
+            Utils.PushList(errors, r.errors)
+            result.push(r.result)
+        }
+        return {
+            information,
+            errors,
+            warnings,
+            result,
+        }
+    }
+}
+
+export class On<P, T> extends DesugaringStep<T> {
+    private readonly key: string
+    private readonly step: (t: T) => Conversion<P, P>
+
+    constructor(key: string, step: Conversion<P, P> | ((t: T) => Conversion<P, P>)) {
+        super(
+            "Applies " + step.name + " onto property `" + key + "`",
+            [key],
+            `On(${key}, ${step.name})`
+        )
+        if (typeof step === "function") {
+            this.step = step
+        } else {
+            this.step = (_) => step
+        }
+        this.key = key
+    }
+
+    convert(
+        json: T,
+        context: string
+    ): { result: T; errors?: string[]; warnings?: string[]; information?: string[] } {
+        json = { ...json }
+        const step = this.step(json)
+        const key = this.key
+        const value: P = json[key]
+        if (value === undefined || value === null) {
+            return { result: json }
+        }
+        const r = step.convert(value, context + "." + key)
+        json[key] = r.result
+        return {
+            ...r,
+            result: json,
+        }
+    }
+}
+
+export class Pass<T> extends Conversion<T, T> {
+    constructor(message?: string) {
+        super(message ?? "Does nothing, often to swap out steps in testing", [], "Pass")
+    }
+
+    convert(
+        json: T,
+        context: string
+    ): { result: T; errors?: string[]; warnings?: string[]; information?: string[] } {
+        return {
+            result: json,
+        }
+    }
+}
+
+export class Concat<X, T> extends Conversion<X[], T[]> {
+    private readonly _step: Conversion<X, T[]>
+
+    constructor(step: Conversion<X, T[]>) {
+        super(
+            "Executes the given step, flattens the resulting list",
+            [],
+            "Concat(" + step.name + ")"
+        )
+        this._step = step
+    }
+
+    convert(
+        values: X[],
+        context: string
+    ): { result: T[]; errors?: string[]; warnings?: string[]; information?: string[] } {
+        if (values === undefined || values === null) {
+            // Move on - nothing to see here!
+            return {
+                result: undefined,
+            }
+        }
+        const r = new Each(this._step).convert(values, context)
+        const vals: T[][] = r.result
+
+        const flattened: T[] = [].concat(...vals)
+
+        return {
+            ...r,
+            result: flattened,
+        }
+    }
+}
+
+export class FirstOf<T, X> extends Conversion<T, X> {
+    private readonly _conversion: Conversion<T, X[]>
+
+    constructor(conversion: Conversion<T, X[]>) {
+        super(
+            "Picks the first result of the conversion step",
+            [],
+            "FirstOf(" + conversion.name + ")"
+        )
+        this._conversion = conversion
+    }
+
+    convert(
+        json: T,
+        context: string
+    ): { result: X; errors?: string[]; warnings?: string[]; information?: string[] } {
+        const reslt = this._conversion.convert(json, context)
+        return {
+            ...reslt,
+            result: reslt.result[0],
+        }
+    }
+}
+
+export class Fuse<T> extends DesugaringStep<T> {
+    private readonly steps: DesugaringStep<T>[]
+
+    constructor(doc: string, ...steps: DesugaringStep<T>[]) {
+        super(
+            (doc ?? "") +
+                "This fused pipeline of the following steps: " +
+                steps.map((s) => s.name).join(", "),
+            Utils.Dedup([].concat(...steps.map((step) => step.modifiedAttributes))),
+            "Fuse of " + steps.map((s) => s.name).join(", ")
+        )
+        this.steps = Utils.NoNull(steps)
+    }
+
+    convert(
+        json: T,
+        context: string
+    ): { result: T; errors: string[]; warnings: string[]; information: string[] } {
+        const errors = []
+        const warnings = []
+        const information = []
+        for (let i = 0; i < this.steps.length; i++) {
+            const step = this.steps[i]
+            try {
+                let r = step.convert(json, "While running step " + step.name + ": " + context)
+                if (r.result["tagRenderings"]?.some((tr) => tr === undefined)) {
+                    throw step.name + " introduced an undefined tagRendering"
+                }
+                errors.push(...(r.errors ?? []))
+                warnings.push(...(r.warnings ?? []))
+                information.push(...(r.information ?? []))
+                json = r.result
+                if (errors.length > 0) {
+                    break
+                }
+            } catch (e) {
+                console.error("Step " + step.name + " failed due to ", e, e.stack)
+                throw e
+            }
+        }
+        return {
+            result: json,
+            errors,
+            warnings,
+            information,
+        }
+    }
+}
+
+export class SetDefault<T> extends DesugaringStep<T> {
+    private readonly value: any
+    private readonly key: string
+    private readonly _overrideEmptyString: boolean
+
+    constructor(key: string, value: any, overrideEmptyString = false) {
+        super("Sets " + key + " to a default value if undefined", [], "SetDefault of " + key)
+        this.key = key
+        this.value = value
+        this._overrideEmptyString = overrideEmptyString
+    }
+
+    convert(json: T, context: string): { result: T } {
+        if (json[this.key] === undefined || (json[this.key] === "" && this._overrideEmptyString)) {
+            json = { ...json }
+            json[this.key] = this.value
+        }
+
+        return {
+            result: json,
+        }
+    }
+}

src/Models/ThemeConfig/Conversion/CreateNoteImportLayer.ts

@@ -0,0 +1,211 @@
+import { Conversion } from "./Conversion"
+import LayerConfig from "../LayerConfig"
+import { LayerConfigJson } from "../Json/LayerConfigJson"
+import Translations from "../../../UI/i18n/Translations"
+import PointRenderingConfigJson from "../Json/PointRenderingConfigJson"
+import { Translation, TypedTranslation } from "../../../UI/i18n/Translation"
+
+export default class CreateNoteImportLayer extends Conversion<LayerConfigJson, LayerConfigJson> {
+    /**
+     * A closed note is included if it is less then 'n'-days closed
+     * @private
+     */
+    private readonly _includeClosedNotesDays: number
+
+    constructor(includeClosedNotesDays = 0) {
+        super(
+            [
+                "Advanced conversion which deducts a layer showing all notes that are 'importable' (i.e. a note that contains a link to some MapComplete theme, with hash '#import').",
+                "The import buttons and matches will be based on the presets of the given theme",
+            ].join("\n\n"),
+            [],
+            "CreateNoteImportLayer"
+        )
+        this._includeClosedNotesDays = includeClosedNotesDays
+    }
+
+    convert(layerJson: LayerConfigJson, context: string): { result: LayerConfigJson } {
+        const t = Translations.t.importLayer
+
+        /**
+         * The note itself will contain `tags=k=v;k=v;k=v;...
+         * This must be matched with a regex.
+         * This is a simple JSON-object as how it'll be put into the layerConfigJson directly
+         */
+        const isShownIfAny: any[] = []
+        const layer = new LayerConfig(layerJson, "while constructing a note-import layer")
+        for (const preset of layer.presets) {
+            const mustMatchAll = []
+            for (const tag of preset.tags) {
+                const key = tag.key
+                const value = tag.value
+                const condition = "_tags~(^|.*;)" + key + "=" + value + "($|;.*)"
+                mustMatchAll.push(condition)
+            }
+            isShownIfAny.push({ and: mustMatchAll })
+        }
+
+        const pointRenderings = (layerJson.mapRendering ?? []).filter(
+            (r) => r !== null && r["location"] !== undefined
+        )
+        const firstRender = <PointRenderingConfigJson>pointRenderings[0]
+        if (firstRender === undefined) {
+            throw `Layer ${layerJson.id} does not have a pointRendering: ` + context
+        }
+        const title = layer.presets[0].title
+
+        const importButton = {}
+        {
+            const translations = trs(t.importButton, {
+                layerId: layer.id,
+                title: layer.presets[0].title,
+            })
+            for (const key in translations) {
+                if (key !== "_context") {
+                    importButton[key] = "{" + translations[key] + "}"
+                } else {
+                    importButton[key] = translations[key]
+                }
+            }
+        }
+
+        function embed(prefix, translation: Translation, postfix) {
+            const result = {}
+            for (const language in translation.translations) {
+                result[language] = prefix + translation.translations[language] + postfix
+            }
+            result["_context"] = translation.context
+            return result
+        }
+
+        function tr(translation: Translation) {
+            return { ...translation.translations, _context: translation.context }
+        }
+
+        function trs<T>(translation: TypedTranslation<T>, subs: T): Record<string, string> {
+            return { ...translation.Subs(subs).translations, _context: translation.context }
+        }
+
+        const result: LayerConfigJson = {
+            id: "note_import_" + layer.id,
+            // By disabling the name, the import-layers won't pollute the filter view "name": t.layerName.Subs({title: layer.title.render}).translations,
+            description: trs(t.description, { title: layer.title.render }),
+            source: {
+                osmTags: {
+                    and: ["id~*"],
+                },
+                geoJson:
+                    "https://api.openstreetmap.org/api/0.6/notes.json?limit=10000&closed=" +
+                    this._includeClosedNotesDays +
+                    "&bbox={x_min},{y_min},{x_max},{y_max}",
+                geoJsonZoomLevel: 10,
+                maxCacheAge: 0,
+            },
+            /* We need to set 'pass_all_features'
+             There are probably many note_import-layers, and we don't want the first one to gobble up all notes and then discard them...
+             */
+            passAllFeatures: true,
+            minzoom: Math.min(12, layerJson.minzoom - 2),
+            title: {
+                render: trs(t.popupTitle, { title }),
+            },
+            calculatedTags: [
+                "_first_comment=get(feat)('comments')[0].text.toLowerCase()",
+                "_trigger_index=(() => {const lines = feat.properties['_first_comment'].split('\\n'); const matchesMapCompleteURL = lines.map(l => l.match(\".*https://mapcomplete.osm.be/\\([a-zA-Z_-]+\\)\\(.html\\)?.*#import\")); const matchedIndexes = matchesMapCompleteURL.map((doesMatch, i) => [doesMatch !== null, i]).filter(v => v[0]).map(v => v[1]); return matchedIndexes[0] })()",
+                "_comments_count=get(feat)('comments').length",
+                "_intro=(() => {const lines = get(feat)('comments')[0].text.split('\\n'); lines.splice(get(feat)('_trigger_index')-1, lines.length); return lines.filter(l => l !== '').join('<br/>');})()",
+                "_tags=(() => {let lines = get(feat)('comments')[0].text.split('\\n').map(l => l.trim()); lines.splice(0, get(feat)('_trigger_index') + 1); lines = lines.filter(l => l != ''); return lines.join(';');})()",
+            ],
+            isShown: {
+                and: ["_trigger_index~*", { or: isShownIfAny }],
+            },
+            titleIcons: [
+                {
+                    render: "<a href='https://openstreetmap.org/note/{id}' target='_blank'><img src='./assets/svg/osm-logo-us.svg'></a>",
+                },
+            ],
+            tagRenderings: [
+                {
+                    id: "Intro",
+                    render: "{_intro}",
+                },
+                {
+                    id: "conversation",
+                    render: "{visualize_note_comments(comments,1)}",
+                    condition: "_comments_count>1",
+                },
+                {
+                    id: "import",
+                    render: importButton,
+                    condition: "closed_at=",
+                },
+                {
+                    id: "close_note_",
+                    render: embed(
+                        "{close_note(",
+                        t.notFound.Subs({ title }),
+                        ", ./assets/svg/close.svg, id, This feature does not exist, 18)}"
+                    ),
+                    condition: "closed_at=",
+                },
+                {
+                    id: "close_note_mapped",
+                    render: embed(
+                        "{close_note(",
+                        t.alreadyMapped.Subs({ title }),
+                        ", ./assets/svg/duplicate.svg, id, Already mapped, 18)}"
+                    ),
+                    condition: "closed_at=",
+                },
+                {
+                    id: "handled",
+                    render: tr(t.importHandled),
+                    condition: "closed_at~*",
+                },
+                {
+                    id: "comment",
+                    render: "{add_note_comment()}",
+                },
+                {
+                    id: "add_image",
+                    render: "{add_image_to_note()}",
+                },
+                {
+                    id: "nearby_images",
+                    render: tr(t.nearbyImagesIntro),
+                },
+                {
+                    id: "all_tags",
+                    render: "{all_tags()}",
+                    metacondition: {
+                        or: [
+                            "__featureSwitchIsDebugging=true",
+                            "mapcomplete-show_tags=full",
+                            "mapcomplete-show_debug=yes",
+                        ],
+                    },
+                },
+            ],
+            mapRendering: [
+                {
+                    location: ["point"],
+                    icon: {
+                        render: "circle:white;help:black",
+                        mappings: [
+                            {
+                                if: { or: ["closed_at~*", "_imported=yes"] },
+                                then: "circle:white;checkmark:black",
+                            },
+                        ],
+                    },
+                    iconSize: "40,40",
+                    anchor: "center",
+                },
+            ],
+        }
+
+        return {
+            result,
+        }
+    }
+}

src/Models/ThemeConfig/Conversion/FixImages.ts

@@ -0,0 +1,348 @@
+import { Conversion, DesugaringStep } from "./Conversion"
+import { LayoutConfigJson } from "../Json/LayoutConfigJson"
+import { Utils } from "../../../Utils"
+import metapaths from "../../../assets/schemas/layoutconfigmeta.json"
+import tagrenderingmetapaths from "../../../assets/schemas/questionabletagrenderingconfigmeta.json"
+import Translations from "../../../UI/i18n/Translations"
+
+import { parse as parse_html } from "node-html-parser"
+export class ExtractImages extends Conversion<
+    LayoutConfigJson,
+    { path: string; context: string }[]
+> {
+    private _isOfficial: boolean
+    private _sharedTagRenderings: Set<string>
+
+    private static readonly layoutMetaPaths = metapaths.filter((mp) => {
+        const typeHint = mp.hints.typehint
+        return (
+            ExtractImages.mightBeTagRendering(<any>mp) ||
+            (typeHint !== undefined &&
+                (typeHint === "image" ||
+                    typeHint === "icon" ||
+                    typeHint === "image[]" ||
+                    typeHint === "icon[]"))
+        )
+    })
+    private static readonly tagRenderingMetaPaths = tagrenderingmetapaths
+
+    constructor(isOfficial: boolean, sharedTagRenderings: Set<string>) {
+        super("Extract all images from a layoutConfig using the meta paths.", [], "ExctractImages")
+        this._isOfficial = isOfficial
+        this._sharedTagRenderings = sharedTagRenderings
+    }
+
+    public static mightBeTagRendering(metapath: { type?: string | string[] }): boolean {
+        if (!Array.isArray(metapath.type)) {
+            return false
+        }
+        return (
+            metapath.type?.some(
+                (t) =>
+                    t !== null &&
+                    (t["$ref"] == "#/definitions/TagRenderingConfigJson" ||
+                        t["$ref"] == "#/definitions/QuestionableTagRenderingConfigJson")
+            ) ?? false
+        )
+    }
+
+    /**
+     *  const images = new ExtractImages(true, new Map<string, any>()).convert(<any>{
+     *     "layers": [
+     *         {
+     *             tagRenderings: [
+     *                 {
+     *                     "mappings": [
+     *                         {
+     *                             "if": "bicycle_parking=stands",
+     *                             "then": {
+     *                                 "en": "Staple racks",
+     *                             },
+     *                             "icon": {
+     *                                 path: "./assets/layers/bike_parking/staple.svg",
+     *                                 class: "small"
+     *                             }
+     *                         },
+     *                         {
+     *                             "if": "bicycle_parking=stands",
+     *                             "then": {
+     *                                 "en": "Bollard",
+     *                             },
+     *                             "icon": "./assets/layers/bike_parking/bollard.svg",
+     *                         }
+     *                     ]
+     *                 }
+     *             ]
+     *         }
+     *     ]
+     * }, "test").result.map(i => i.path);
+     * images.length // => 2
+     * images.findIndex(img => img == "./assets/layers/bike_parking/staple.svg") >= 0 // => true
+     * images.findIndex(img => img == "./assets/layers/bike_parking/bollard.svg") >= 0 // => true
+     *
+     * // should not pickup rotation, should drop color
+     * const images = new ExtractImages(true, new Set<string>()).convert(<any>{"layers": [{mapRendering: [{"location": ["point", "centroid"],"icon": "pin:black",rotation: 180,iconSize: "40,40,center"}]}]
+     * }, "test").result
+     * images.length // => 1
+     * images[0].path // => "pin"
+     *
+     */
+    convert(
+        json: LayoutConfigJson,
+        context: string
+    ): { result: { path: string; context: string }[]; errors: string[]; warnings: string[] } {
+        const allFoundImages: { path: string; context: string }[] = []
+        const errors = []
+        const warnings = []
+        for (const metapath of ExtractImages.layoutMetaPaths) {
+            const mightBeTr = ExtractImages.mightBeTagRendering(<any>metapath)
+            const allRenderedValuesAreImages =
+                metapath.hints.typehint === "icon" || metapath.hints.typehint === "image"
+            const found = Utils.CollectPath(metapath.path, json)
+            if (mightBeTr) {
+                // We might have tagRenderingConfigs containing icons here
+                for (const el of found) {
+                    const path = el.path
+                    const foundImage = el.leaf
+                    if (typeof foundImage === "string") {
+                        if (!allRenderedValuesAreImages) {
+                            continue
+                        }
+
+                        if (foundImage == "") {
+                            warnings.push(context + "." + path.join(".") + " Found an empty image")
+                        }
+
+                        if (this._sharedTagRenderings?.has(foundImage)) {
+                            // This is not an image, but a shared tag rendering
+                            // At key positions for checking, they'll be expanded already, so we can safely ignore them here
+                            continue
+                        }
+
+                        allFoundImages.push({ path: foundImage, context: context + "." + path })
+                    } else {
+                        // This is a tagRendering.
+                        // Either every rendered value might be an icon
+                        // or -in the case of a normal tagrendering- only the 'icons' in the mappings have an icon (or exceptionally an '<img>' tag in the translation
+                        for (const trpath of ExtractImages.tagRenderingMetaPaths) {
+                            // Inspect all the rendered values
+                            const fromPath = Utils.CollectPath(trpath.path, foundImage)
+                            const isRendered = trpath.hints.typehint === "rendered"
+                            const isImage =
+                                trpath.hints.typehint === "icon" ||
+                                trpath.hints.typehint === "image"
+                            for (const img of fromPath) {
+                                if (allRenderedValuesAreImages && isRendered) {
+                                    // What we found is an image
+                                    if (img.leaf === "" || img.leaf["path"] == "") {
+                                        warnings.push(
+                                            context +
+                                                [...path, ...img.path].join(".") +
+                                                ": Found an empty image at "
+                                        )
+                                    } else if (typeof img.leaf !== "string") {
+                                        ;(this._isOfficial ? errors : warnings).push(
+                                            context +
+                                                "." +
+                                                img.path.join(".") +
+                                                ": found an image path that is not a string: " +
+                                                JSON.stringify(img.leaf)
+                                        )
+                                    } else {
+                                        allFoundImages.push({
+                                            path: img.leaf,
+                                            context: context + "." + path,
+                                        })
+                                    }
+                                }
+                                if (!allRenderedValuesAreImages && isImage) {
+                                    // Extract images from the translations
+                                    allFoundImages.push(
+                                        ...Translations.T(
+                                            img.leaf,
+                                            "extract_images from " + img.path.join(".")
+                                        )
+                                            .ExtractImages(false)
+                                            .map((path) => ({
+                                                path,
+                                                context: context + "." + path,
+                                            }))
+                                    )
+                                }
+                            }
+                        }
+                    }
+                }
+            } else {
+                for (const foundElement of found) {
+                    if (foundElement.leaf === "") {
+                        warnings.push(
+                            context + "." + foundElement.path.join(".") + " Found an empty image"
+                        )
+                        continue
+                    }
+                    if (typeof foundElement.leaf !== "string") {
+                        continue
+                    }
+                    allFoundImages.push({
+                        context: context + "." + foundElement.path.join("."),
+                        path: foundElement.leaf,
+                    })
+                }
+            }
+        }
+
+        const cleanedImages: { path: string; context: string }[] = []
+
+        for (const foundImage of allFoundImages) {
+            if (foundImage.path.startsWith("<") && foundImage.path.endsWith(">")) {
+                // These is probably html - we ignore
+                const doc = parse_html(foundImage.path)
+                const images = Array.from(doc.getElementsByTagName("img"))
+                const paths = images.map((i) => i.getAttribute("src"))
+                cleanedImages.push(
+                    ...paths.map((path) => ({ path, context: foundImage.context + " (in html)" }))
+                )
+                continue
+            }
+
+            // Split "circle:white;./assets/layers/.../something.svg" into ["circle", "./assets/layers/.../something.svg"]
+            const allPaths = Utils.NoNull(
+                Utils.NoEmpty(foundImage.path?.split(";")?.map((part) => part.split(":")[0]))
+            )
+            for (const path of allPaths) {
+                cleanedImages.push({ path, context: foundImage.context })
+            }
+        }
+
+        return { result: cleanedImages, errors, warnings }
+    }
+}
+
+export class FixImages extends DesugaringStep<LayoutConfigJson> {
+    private readonly _knownImages: Set<string>
+
+    constructor(knownImages: Set<string>) {
+        super(
+            "Walks over the entire theme and replaces images to the relative URL. Only works if the ID of the theme is an URL",
+            [],
+            "fixImages"
+        )
+        this._knownImages = knownImages
+    }
+
+    /**
+     * If the id is an URL to a json file, replaces "./" in images with the path to the json file
+     *
+     * const theme = {
+     *          "id": "https://raw.githubusercontent.com/seppesantens/MapComplete-Themes/main/VerkeerdeBordenDatabank/verkeerdeborden.json"
+     *         "layers": [
+     *             {
+     *                 "mapRendering": [
+     *                     {
+     *                         "icon": "./TS_bolt.svg",
+     *                         iconBadges: [{
+     *                             if: "id=yes",
+     *                             then: {
+     *                                 mappings: [
+     *                                     {
+     *                                         if: "id=yes",
+     *                                         then: "./Something.svg"
+     *                                     }
+     *                                 ]
+     *                             }
+     *                         }],
+     *                         "location": [
+     *                             "point",
+     *                             "centroid"
+     *                         ]
+     *                     }
+     *                 ]
+     *             }
+     *         ],
+     *     }
+     * const fixed = new FixImages(new Set<string>()).convert(<any> theme, "test").result
+     * fixed.layers[0]["mapRendering"][0].icon // => "https://raw.githubusercontent.com/seppesantens/MapComplete-Themes/main/VerkeerdeBordenDatabank/TS_bolt.svg"
+     * fixed.layers[0]["mapRendering"][0].iconBadges[0].then.mappings[0].then // => "https://raw.githubusercontent.com/seppesantens/MapComplete-Themes/main/VerkeerdeBordenDatabank/Something.svg"
+     */
+    convert(
+        json: LayoutConfigJson,
+        context: string
+    ): { result: LayoutConfigJson; warnings?: string[] } {
+        let url: URL
+        try {
+            url = new URL(json.id)
+        } catch (e) {
+            // Not a URL, we don't rewrite
+            return { result: json }
+        }
+
+        const warnings: string[] = []
+        const absolute = url.protocol + "//" + url.host
+        let relative = url.protocol + "//" + url.host + url.pathname
+        relative = relative.substring(0, relative.lastIndexOf("/"))
+        const self = this
+
+        if (relative.endsWith("assets/generated/themes")) {
+            warnings.push(
+                "Detected 'assets/generated/themes' as relative URL. I'm assuming that you are loading your file for the MC-repository, so I'm rewriting all image links as if they were absolute instead of relative"
+            )
+            relative = absolute
+        }
+
+        function replaceString(leaf: string) {
+            if (self._knownImages.has(leaf)) {
+                return leaf
+            }
+
+            if (typeof leaf !== "string") {
+                warnings.push(
+                    "Found a non-string object while replacing images: " + JSON.stringify(leaf)
+                )
+                return leaf
+            }
+
+            if (leaf.startsWith("./")) {
+                return relative + leaf.substring(1)
+            }
+            if (leaf.startsWith("/")) {
+                return absolute + leaf
+            }
+            return leaf
+        }
+
+        json = Utils.Clone(json)
+
+        for (const metapath of metapaths) {
+            if (metapath.hints.typehint !== "image" && metapath.hints.typehint !== "icon") {
+                continue
+            }
+            const mightBeTr = ExtractImages.mightBeTagRendering(<any>metapath)
+            Utils.WalkPath(metapath.path, json, (leaf, path) => {
+                if (typeof leaf === "string") {
+                    return replaceString(leaf)
+                }
+
+                if (mightBeTr) {
+                    // We might have reached a tagRenderingConfig containing icons
+                    // lets walk every rendered value and fix the images in there
+                    for (const trpath of tagrenderingmetapaths) {
+                        if (trpath.hints.typehint !== "rendered") {
+                            continue
+                        }
+                        Utils.WalkPath(trpath.path, leaf, (rendered) => {
+                            return replaceString(rendered)
+                        })
+                    }
+                }
+
+                return leaf
+            })
+        }
+
+        return {
+            warnings,
+            result: json,
+        }
+    }
+}

src/Models/ThemeConfig/Conversion/LegacyJsonConvert.ts

@@ -0,0 +1,257 @@
+import { LayoutConfigJson } from "../Json/LayoutConfigJson"
+import { Utils } from "../../../Utils"
+import LineRenderingConfigJson from "../Json/LineRenderingConfigJson"
+import { LayerConfigJson } from "../Json/LayerConfigJson"
+import { DesugaringStep, Each, Fuse, On } from "./Conversion"
+import PointRenderingConfigJson from "../Json/PointRenderingConfigJson"
+
+export class UpdateLegacyLayer extends DesugaringStep<
+    LayerConfigJson | string | { builtin; override }
+> {
+    constructor() {
+        super(
+            "Updates various attributes from the old data format to the new to provide backwards compatibility with the formats",
+            ["overpassTags", "source.osmtags", "tagRenderings[*].id", "mapRendering"],
+            "UpdateLegacyLayer"
+        )
+    }
+
+    convert(
+        json: LayerConfigJson,
+        context: string
+    ): { result: LayerConfigJson; errors: string[]; warnings: string[] } {
+        const warnings = []
+        if (typeof json === "string" || json["builtin"] !== undefined) {
+            // Reuse of an already existing layer; return as-is
+            return { result: json, errors: [], warnings: [] }
+        }
+        let config = { ...json }
+
+        if (config["overpassTags"]) {
+            config.source = config.source ?? {
+                osmTags: config["overpassTags"],
+            }
+            config.source["osmTags"] = config["overpassTags"]
+            delete config["overpassTags"]
+        }
+
+        for (const preset of config.presets ?? []) {
+            const preciseInput = preset["preciseInput"]
+            if (typeof preciseInput === "boolean") {
+                delete preset["preciseInput"]
+            } else if (preciseInput !== undefined) {
+                delete preciseInput["preferredBackground"]
+                console.log("Precise input:", preciseInput)
+                preset.snapToLayer = preciseInput.snapToLayer
+                delete preciseInput.snapToLayer
+                if (preciseInput.maxSnapDistance) {
+                    preset.maxSnapDistance = preciseInput.maxSnapDistance
+                    delete preciseInput.maxSnapDistance
+                }
+                if (Object.keys(preciseInput).length == 0) {
+                    delete preset["preciseInput"]
+                }
+            }
+
+            if (typeof preset.snapToLayer === "string") {
+                preset.snapToLayer = [preset.snapToLayer]
+            }
+        }
+
+        if (config.tagRenderings !== undefined) {
+            let i = 0
+            for (const tagRendering of config.tagRenderings) {
+                i++
+                if (
+                    typeof tagRendering === "string" ||
+                    tagRendering["builtin"] !== undefined ||
+                    tagRendering["rewrite"] !== undefined
+                ) {
+                    continue
+                }
+                if (tagRendering["id"] === undefined) {
+                    if (tagRendering["#"] !== undefined) {
+                        tagRendering["id"] = tagRendering["#"]
+                        delete tagRendering["#"]
+                    } else if (tagRendering["freeform"]?.key !== undefined) {
+                        tagRendering["id"] = config.id + "-" + tagRendering["freeform"]["key"]
+                    } else {
+                        tagRendering["id"] = "tr-" + i
+                    }
+                }
+            }
+        }
+
+        if (config.mapRendering === undefined) {
+            config.mapRendering = []
+            // This is a legacy format, lets create a pointRendering
+            let location: ("point" | "centroid")[] = ["point"]
+            let wayHandling: number = config["wayHandling"] ?? 0
+            if (wayHandling !== 0) {
+                location = ["point", "centroid"]
+            }
+            if (config["icon"] ?? config["label"] !== undefined) {
+                const pointConfig = {
+                    icon: config["icon"],
+                    iconBadges: config["iconOverlays"],
+                    label: config["label"],
+                    iconSize: config["iconSize"],
+                    location,
+                    rotation: config["rotation"],
+                }
+                config.mapRendering.push(pointConfig)
+            }
+
+            if (wayHandling !== 1) {
+                const lineRenderConfig = <LineRenderingConfigJson>{
+                    color: config["color"],
+                    width: config["width"],
+                    dashArray: config["dashArray"],
+                }
+                if (Object.keys(lineRenderConfig).length > 0) {
+                    config.mapRendering.push(lineRenderConfig)
+                }
+            }
+            if (config.mapRendering.length === 0) {
+                throw (
+                    "Could not convert the legacy theme into a new theme: no renderings defined for layer " +
+                    config.id
+                )
+            }
+        }
+
+        delete config["color"]
+        delete config["width"]
+        delete config["dashArray"]
+
+        delete config["icon"]
+        delete config["iconOverlays"]
+        delete config["label"]
+        delete config["iconSize"]
+        delete config["rotation"]
+        delete config["wayHandling"]
+        delete config["hideUnderlayingFeaturesMinPercentage"]
+
+        for (const mapRenderingElement of config.mapRendering ?? []) {
+            if (mapRenderingElement["iconOverlays"] !== undefined) {
+                mapRenderingElement["iconBadges"] = mapRenderingElement["iconOverlays"]
+            }
+            for (const overlay of mapRenderingElement["iconBadges"] ?? []) {
+                if (overlay["badge"] !== true) {
+                    warnings.push("Warning: non-overlay element for ", config.id)
+                }
+                delete overlay["badge"]
+            }
+        }
+
+        for (const rendering of config.mapRendering ?? []) {
+            if (!rendering["iconSize"]) {
+                continue
+            }
+            const pr = <PointRenderingConfigJson>rendering
+            let iconSize = pr.iconSize
+            console.log("Iconsize is", iconSize)
+
+            if (Object.keys(pr.iconSize).length === 1 && pr.iconSize["render"] !== undefined) {
+                iconSize = pr.iconSize["render"]
+            }
+
+            if (typeof iconSize === "string")
+                if (["bottom", "center", "top"].some((a) => (<string>iconSize).endsWith(a))) {
+                    const parts = iconSize.split(",").map((parts) => parts.toLowerCase().trim())
+                    pr.anchor = parts.pop()
+                    pr.iconSize = parts.join(",")
+                }
+        }
+
+        for (const rendering of config.mapRendering) {
+            for (const key in rendering) {
+                if (!rendering[key]) {
+                    continue
+                }
+                if (
+                    typeof rendering[key]["render"] === "string" &&
+                    Object.keys(rendering[key]).length === 1
+                ) {
+                    console.log("Rewrite: ", rendering[key])
+                    rendering[key] = rendering[key]["render"]
+                }
+            }
+        }
+
+        return {
+            result: config,
+            errors: [],
+            warnings,
+        }
+    }
+}
+
+class UpdateLegacyTheme extends DesugaringStep<LayoutConfigJson> {
+    constructor() {
+        super("Small fixes in the theme config", ["roamingRenderings"], "UpdateLegacyTheme")
+    }
+
+    convert(
+        json: LayoutConfigJson,
+        context: string
+    ): { result: LayoutConfigJson; errors: string[]; warnings: string[] } {
+        const oldThemeConfig = { ...json }
+
+        if (oldThemeConfig.socialImage === "") {
+            delete oldThemeConfig.socialImage
+        }
+
+        if (oldThemeConfig["roamingRenderings"] !== undefined) {
+            if (oldThemeConfig["roamingRenderings"].length == 0) {
+                delete oldThemeConfig["roamingRenderings"]
+            } else {
+                return {
+                    result: null,
+                    errors: [
+                        context +
+                            ": The theme contains roamingRenderings. These are not supported anymore",
+                    ],
+                    warnings: [],
+                }
+            }
+        }
+
+        oldThemeConfig.layers = Utils.NoNull(oldThemeConfig.layers)
+        delete oldThemeConfig["language"]
+        delete oldThemeConfig["version"]
+
+        if (oldThemeConfig["maintainer"] !== undefined) {
+            console.log(
+                "Maintainer: ",
+                oldThemeConfig["maintainer"],
+                "credits: ",
+                oldThemeConfig["credits"]
+            )
+            if (oldThemeConfig.credits === undefined) {
+                oldThemeConfig["credits"] = oldThemeConfig["maintainer"]
+                delete oldThemeConfig["maintainer"]
+            } else if (oldThemeConfig["maintainer"].toLowerCase().trim() === "mapcomplete") {
+                delete oldThemeConfig["maintainer"]
+            } else if (oldThemeConfig["maintainer"].toLowerCase().trim() === "") {
+                delete oldThemeConfig["maintainer"]
+            }
+        }
+
+        return {
+            errors: [],
+            warnings: [],
+            result: oldThemeConfig,
+        }
+    }
+}
+
+export class FixLegacyTheme extends Fuse<LayoutConfigJson> {
+    constructor() {
+        super(
+            "Fixes a legacy theme to the modern JSON format geared to humans. Syntactic sugars are kept (i.e. no tagRenderings are expandend, no dependencies are automatically gathered)",
+            new UpdateLegacyTheme(),
+            new On("layers", new Each(new UpdateLegacyLayer()))
+        )
+    }
+}

src/Models/ThemeConfig/Conversion/PrepareTheme.ts

@@ -0,0 +1,646 @@
+import {
+    Concat,
+    Conversion,
+    DesugaringContext,
+    DesugaringStep,
+    Each,
+    Fuse,
+    On,
+    Pass,
+    SetDefault,
+} from "./Conversion"
+import { LayoutConfigJson } from "../Json/LayoutConfigJson"
+import { PrepareLayer } from "./PrepareLayer"
+import { LayerConfigJson } from "../Json/LayerConfigJson"
+import { Utils } from "../../../Utils"
+import Constants from "../../Constants"
+import CreateNoteImportLayer from "./CreateNoteImportLayer"
+import LayerConfig from "../LayerConfig"
+import { TagRenderingConfigJson } from "../Json/TagRenderingConfigJson"
+import DependencyCalculator from "../DependencyCalculator"
+import { AddContextToTranslations } from "./AddContextToTranslations"
+import ValidationUtils from "./ValidationUtils"
+
+class SubstituteLayer extends Conversion<string | LayerConfigJson, LayerConfigJson[]> {
+    private readonly _state: DesugaringContext
+
+    constructor(state: DesugaringContext) {
+        super(
+            "Converts the identifier of a builtin layer into the actual layer, or converts a 'builtin' syntax with override in the fully expanded form",
+            [],
+            "SubstituteLayer"
+        )
+        this._state = state
+    }
+
+    convert(
+        json: string | LayerConfigJson,
+        context: string
+    ): { result: LayerConfigJson[]; errors: string[]; information?: string[] } {
+        const errors = []
+        const information = []
+        const state = this._state
+
+        function reportNotFound(name: string) {
+            const knownLayers = Array.from(state.sharedLayers.keys())
+            const withDistance = knownLayers.map((lname) => [
+                lname,
+                Utils.levenshteinDistance(name, lname),
+            ])
+            withDistance.sort((a, b) => a[1] - b[1])
+            const ids = withDistance.map((n) => n[0])
+            // Known builtin layers are "+.join(",")+"\n    For more information, see "
+            errors.push(`${context}: The layer with name ${name} was not found as a builtin layer. Perhaps you meant ${ids[0]}, ${ids[1]} or ${ids[2]}?
+ For an overview of all available layers, refer to https://github.com/pietervdvn/MapComplete/blob/develop/Docs/BuiltinLayers.md`)
+        }
+
+        if (typeof json === "string") {
+            const found = state.sharedLayers.get(json)
+            if (found === undefined) {
+                reportNotFound(json)
+                return {
+                    result: null,
+                    errors,
+                }
+            }
+            return {
+                result: [found],
+                errors,
+            }
+        }
+
+        if (json["builtin"] !== undefined) {
+            let names = json["builtin"]
+            if (typeof names === "string") {
+                names = [names]
+            }
+            const layers = []
+
+            for (const name of names) {
+                const found = Utils.Clone(state.sharedLayers.get(name))
+                if (found === undefined) {
+                    reportNotFound(name)
+                    continue
+                }
+                if (
+                    json["override"]["tagRenderings"] !== undefined &&
+                    (found["tagRenderings"] ?? []).length > 0
+                ) {
+                    errors.push(
+                        `At ${context}: when overriding a layer, an override is not allowed to override into tagRenderings. Use "+tagRenderings" or "tagRenderings+" instead to prepend or append some questions.`
+                    )
+                }
+                try {
+                    Utils.Merge(json["override"], found)
+                    layers.push(found)
+                } catch (e) {
+                    errors.push(
+                        `At ${context}: could not apply an override due to: ${e}.\nThe override is: ${JSON.stringify(
+                            json["override"]
+                        )}`
+                    )
+                }
+
+                if (json["hideTagRenderingsWithLabels"]) {
+                    const hideLabels: Set<string> = new Set(json["hideTagRenderingsWithLabels"])
+                    // These labels caused at least one deletion
+                    const usedLabels: Set<string> = new Set<string>()
+                    const filtered = []
+                    for (const tr of found.tagRenderings) {
+                        const labels = tr["labels"]
+                        if (labels !== undefined) {
+                            const forbiddenLabel = labels.findIndex((l) => hideLabels.has(l))
+                            if (forbiddenLabel >= 0) {
+                                usedLabels.add(labels[forbiddenLabel])
+                                information.push(
+                                    context +
+                                        ": Dropping tagRendering " +
+                                        tr["id"] +
+                                        " as it has a forbidden label: " +
+                                        labels[forbiddenLabel]
+                                )
+                                continue
+                            }
+                        }
+
+                        if (hideLabels.has(tr["id"])) {
+                            usedLabels.add(tr["id"])
+                            information.push(
+                                context +
+                                    ": Dropping tagRendering " +
+                                    tr["id"] +
+                                    " as its id is a forbidden label"
+                            )
+                            continue
+                        }
+
+                        if (hideLabels.has(tr["group"])) {
+                            usedLabels.add(tr["group"])
+                            information.push(
+                                context +
+                                    ": Dropping tagRendering " +
+                                    tr["id"] +
+                                    " as its group `" +
+                                    tr["group"] +
+                                    "` is a forbidden label"
+                            )
+                            continue
+                        }
+
+                        filtered.push(tr)
+                    }
+                    const unused = Array.from(hideLabels).filter((l) => !usedLabels.has(l))
+                    if (unused.length > 0) {
+                        errors.push(
+                            "This theme specifies that certain tagrenderings have to be removed based on forbidden layers. One or more of these layers did not match any tagRenderings and caused no deletions: " +
+                                unused.join(", ") +
+                                "\n   This means that this label can be removed or that the original tagRendering that should be deleted does not have this label anymore"
+                        )
+                    }
+                    found.tagRenderings = filtered
+                }
+            }
+            return {
+                result: layers,
+                errors,
+                information,
+            }
+        }
+
+        return {
+            result: [json],
+            errors,
+        }
+    }
+}
+
+class AddDefaultLayers extends DesugaringStep<LayoutConfigJson> {
+    private readonly _state: DesugaringContext
+
+    constructor(state: DesugaringContext) {
+        super(
+            "Adds the default layers, namely: " + Constants.added_by_default.join(", "),
+            ["layers"],
+            "AddDefaultLayers"
+        )
+        this._state = state
+    }
+
+    convert(
+        json: LayoutConfigJson,
+        context: string
+    ): { result: LayoutConfigJson; errors: string[]; warnings: string[] } {
+        const errors = []
+        const warnings = []
+        const state = this._state
+        json.layers = [...json.layers]
+        const alreadyLoaded = new Set(json.layers.map((l) => l["id"]))
+
+        for (const layerName of Constants.added_by_default) {
+            const v = state.sharedLayers.get(layerName)
+            if (v === undefined) {
+                errors.push("Default layer " + layerName + " not found")
+                continue
+            }
+            if (alreadyLoaded.has(v.id)) {
+                warnings.push(
+                    "Layout " +
+                        context +
+                        " already has a layer with name " +
+                        v.id +
+                        "; skipping inclusion of this builtin layer"
+                )
+                continue
+            }
+            json.layers.push(v)
+        }
+
+        return {
+            result: json,
+            errors,
+            warnings,
+        }
+    }
+}
+
+class AddImportLayers extends DesugaringStep<LayoutConfigJson> {
+    constructor() {
+        super(
+            "For every layer in the 'layers'-list, create a new layer which'll import notes. (Note that priviliged layers and layers which have a geojson-source set are ignored)",
+            ["layers"],
+            "AddImportLayers"
+        )
+    }
+
+    convert(
+        json: LayoutConfigJson,
+        context: string
+    ): { result: LayoutConfigJson; errors?: string[]; warnings?: string[] } {
+        if (!(json.enableNoteImports ?? true)) {
+            return {
+                warnings: [
+                    "Not creating a note import layers for theme " +
+                        json.id +
+                        " as they are disabled",
+                ],
+                result: json,
+            }
+        }
+        const errors = []
+
+        json = { ...json }
+        const allLayers: LayerConfigJson[] = <LayerConfigJson[]>json.layers
+        json.layers = [...json.layers]
+
+        const creator = new CreateNoteImportLayer()
+        for (let i1 = 0; i1 < allLayers.length; i1++) {
+            const layer = allLayers[i1]
+            if (layer.source === undefined) {
+                // Priviliged layers are skipped
+                continue
+            }
+
+            if (layer.source["geoJson"] !== undefined) {
+                // Layer which don't get their data from OSM are skipped
+                continue
+            }
+
+            if (layer.title === undefined || layer.name === undefined) {
+                // Anonymous layers and layers without popup are skipped
+                continue
+            }
+
+            if (layer.presets === undefined || layer.presets.length == 0) {
+                // A preset is needed to be able to generate a new point
+                continue
+            }
+
+            try {
+                const importLayerResult = creator.convert(
+                    layer,
+                    context + ".(noteimportlayer)[" + i1 + "]"
+                )
+                if (importLayerResult.result !== undefined) {
+                    json.layers.push(importLayerResult.result)
+                }
+            } catch (e) {
+                errors.push("Could not generate an import-layer for " + layer.id + " due to " + e)
+            }
+        }
+
+        return {
+            errors,
+            result: json,
+        }
+    }
+}
+
+class AddContextToTranslationsInLayout extends DesugaringStep<LayoutConfigJson> {
+    constructor() {
+        super(
+            "Adds context to translations, including the prefix 'themes:json.id'; this is to make sure terms in an 'overrides' or inline layer are linkable too",
+            ["_context"],
+            "AddContextToTranlationsInLayout"
+        )
+    }
+
+    convert(
+        json: LayoutConfigJson,
+        context: string
+    ): {
+        result: LayoutConfigJson
+        errors?: string[]
+        warnings?: string[]
+        information?: string[]
+    } {
+        const conversion = new AddContextToTranslations<LayoutConfigJson>("themes:")
+        return conversion.convert(json, json.id)
+    }
+}
+
+class ApplyOverrideAll extends DesugaringStep<LayoutConfigJson> {
+    constructor() {
+        super(
+            "Applies 'overrideAll' onto every 'layer'. The 'overrideAll'-field is removed afterwards",
+            ["overrideAll", "layers"],
+            "ApplyOverrideAll"
+        )
+    }
+
+    convert(
+        json: LayoutConfigJson,
+        context: string
+    ): { result: LayoutConfigJson; errors: string[]; warnings: string[] } {
+        const overrideAll = json.overrideAll
+        if (overrideAll === undefined) {
+            return { result: json, warnings: [], errors: [] }
+        }
+
+        json = { ...json }
+
+        delete json.overrideAll
+        const newLayers = []
+        for (let layer of json.layers) {
+            layer = Utils.Clone(<LayerConfigJson>layer)
+            Utils.Merge(overrideAll, layer)
+            newLayers.push(layer)
+        }
+        json.layers = newLayers
+
+        return { result: json, warnings: [], errors: [] }
+    }
+}
+
+class AddDependencyLayersToTheme extends DesugaringStep<LayoutConfigJson> {
+    private readonly _state: DesugaringContext
+
+    constructor(state: DesugaringContext) {
+        super(
+            `If a layer has a dependency on another layer, these layers are added automatically on the theme. (For example: defibrillator depends on 'walls_and_buildings' to snap onto. This layer is added automatically)
+
+            Note that these layers are added _at the start_ of the layer list, meaning that they will see _every_ feature.
+            Furthermore, \`passAllFeatures\` will be set, so that they won't steal away features from further layers.
+            Some layers (e.g. \`all_buildings_and_walls\' or \'streets_with_a_name\') are invisible, so by default, \'force_load\' is set too.
+            `,
+            ["layers"],
+            "AddDependencyLayersToTheme"
+        )
+        this._state = state
+    }
+
+    private static CalculateDependencies(
+        alreadyLoaded: LayerConfigJson[],
+        allKnownLayers: Map<string, LayerConfigJson>,
+        themeId: string
+    ): { config: LayerConfigJson; reason: string }[] {
+        const dependenciesToAdd: { config: LayerConfigJson; reason: string }[] = []
+        const loadedLayerIds: Set<string> = new Set<string>(alreadyLoaded.map((l) => l.id))
+
+        // Verify cross-dependencies
+        let unmetDependencies: {
+            neededLayer: string
+            neededBy: string
+            reason: string
+            context?: string
+        }[] = []
+        do {
+            const dependencies: {
+                neededLayer: string
+                reason: string
+                context?: string
+                neededBy: string
+            }[] = []
+
+            for (const layerConfig of alreadyLoaded) {
+                try {
+                    const layerDeps = DependencyCalculator.getLayerDependencies(
+                        new LayerConfig(layerConfig, themeId + "(dependencies)")
+                    )
+                    dependencies.push(...layerDeps)
+                } catch (e) {
+                    console.error(e)
+                    throw (
+                        "Detecting layer dependencies for " + layerConfig.id + " failed due to " + e
+                    )
+                }
+            }
+
+            for (const dependency of dependencies) {
+                if (loadedLayerIds.has(dependency.neededLayer)) {
+                    // We mark the needed layer as 'mustLoad'
+                    alreadyLoaded.find((l) => l.id === dependency.neededLayer).forceLoad = true
+                }
+            }
+
+            // During the generate script, builtin layers are verified but not loaded - so we have to add them manually here
+            // Their existence is checked elsewhere, so this is fine
+            unmetDependencies = dependencies.filter((dep) => !loadedLayerIds.has(dep.neededLayer))
+            for (const unmetDependency of unmetDependencies) {
+                if (loadedLayerIds.has(unmetDependency.neededLayer)) {
+                    continue
+                }
+                const dep = Utils.Clone(allKnownLayers.get(unmetDependency.neededLayer))
+                const reason =
+                    "This layer is needed by " +
+                    unmetDependency.neededBy +
+                    " because " +
+                    unmetDependency.reason +
+                    " (at " +
+                    unmetDependency.context +
+                    ")"
+                if (dep === undefined) {
+                    const message = [
+                        "Loading a dependency failed: layer " +
+                            unmetDependency.neededLayer +
+                            " is not found, neither as layer of " +
+                            themeId +
+                            " nor as builtin layer.",
+                        reason,
+                        "Loaded layers are: " + alreadyLoaded.map((l) => l.id).join(","),
+                    ]
+                    throw message.join("\n\t")
+                }
+
+                dep.forceLoad = true
+                dep.passAllFeatures = true
+                dep.description = reason
+                dependenciesToAdd.unshift({
+                    config: dep,
+                    reason,
+                })
+                loadedLayerIds.add(dep.id)
+                unmetDependencies = unmetDependencies.filter(
+                    (d) => d.neededLayer !== unmetDependency.neededLayer
+                )
+            }
+        } while (unmetDependencies.length > 0)
+
+        return dependenciesToAdd
+    }
+
+    convert(
+        theme: LayoutConfigJson,
+        context: string
+    ): { result: LayoutConfigJson; information: string[] } {
+        const state = this._state
+        const allKnownLayers: Map<string, LayerConfigJson> = state.sharedLayers
+        const knownTagRenderings: Map<string, TagRenderingConfigJson> = state.tagRenderings
+        const information = []
+        const layers: LayerConfigJson[] = <LayerConfigJson[]>theme.layers // Layers should be expanded at this point
+
+        knownTagRenderings.forEach((value, key) => {
+            value.id = key
+        })
+
+        const dependencies = AddDependencyLayersToTheme.CalculateDependencies(
+            layers,
+            allKnownLayers,
+            theme.id
+        )
+        for (const dependency of dependencies) {
+        }
+        if (dependencies.length > 0) {
+            for (const dependency of dependencies) {
+                information.push(
+                    context +
+                        ": added " +
+                        dependency.config.id +
+                        " to the theme. " +
+                        dependency.reason
+                )
+            }
+        }
+        layers.unshift(...dependencies.map((l) => l.config))
+
+        return {
+            result: {
+                ...theme,
+                layers: layers,
+            },
+            information,
+        }
+    }
+}
+
+class PreparePersonalTheme extends DesugaringStep<LayoutConfigJson> {
+    private readonly _state: DesugaringContext
+
+    constructor(state: DesugaringContext) {
+        super("Adds every public layer to the personal theme", ["layers"], "PreparePersonalTheme")
+        this._state = state
+    }
+
+    convert(
+        json: LayoutConfigJson,
+        context: string
+    ): {
+        result: LayoutConfigJson
+        errors?: string[]
+        warnings?: string[]
+        information?: string[]
+    } {
+        if (json.id !== "personal") {
+            return { result: json }
+        }
+
+        // The only thing this _really_ does, is adding the layer-ids into 'layers'
+        // All other preparations are done by the 'override-all'-block in personal.json
+
+        json.layers = Array.from(this._state.sharedLayers.keys())
+            .filter((l) => this._state.sharedLayers.get(l).source !== null)
+            .filter((l) => this._state.publicLayers.has(l))
+        return {
+            result: json,
+            information: ["The personal theme has " + json.layers.length + " public layers"],
+        }
+    }
+}
+
+class WarnForUnsubstitutedLayersInTheme extends DesugaringStep<LayoutConfigJson> {
+    constructor() {
+        super(
+            "Generates a warning if a theme uses an unsubstituted layer",
+            ["layers"],
+            "WarnForUnsubstitutedLayersInTheme"
+        )
+    }
+
+    convert(
+        json: LayoutConfigJson,
+        context: string
+    ): {
+        result: LayoutConfigJson
+        errors?: string[]
+        warnings?: string[]
+        information?: string[]
+    } {
+        if (json.hideFromOverview === true) {
+            return { result: json }
+        }
+        const warnings = []
+        for (const layer of json.layers) {
+            if (typeof layer === "string") {
+                continue
+            }
+            if (layer["builtin"] !== undefined) {
+                continue
+            }
+            if (layer["source"]["geojson"] !== undefined) {
+                // We turn a blind eye for import layers
+                continue
+            }
+
+            const wrn =
+                "The theme " +
+                json.id +
+                " has an inline layer: " +
+                layer["id"] +
+                ". This is discouraged."
+            warnings.push(wrn)
+        }
+        return {
+            result: json,
+            warnings,
+        }
+    }
+}
+
+export class PrepareTheme extends Fuse<LayoutConfigJson> {
+    private state: DesugaringContext
+    constructor(
+        state: DesugaringContext,
+        options?: {
+            skipDefaultLayers: false | boolean
+        }
+    ) {
+        super(
+            "Fully prepares and expands a theme",
+
+            new AddContextToTranslationsInLayout(),
+            new PreparePersonalTheme(state),
+            new WarnForUnsubstitutedLayersInTheme(),
+            new On("layers", new Concat(new SubstituteLayer(state))),
+            new SetDefault("socialImage", "assets/SocialImage.png", true),
+            // We expand all tagrenderings first...
+            new On("layers", new Each(new PrepareLayer(state))),
+            // Then we apply the override all
+            new ApplyOverrideAll(),
+            // And then we prepare all the layers _again_ in case that an override all contained unexpanded tagrenderings!
+            new On("layers", new Each(new PrepareLayer(state))),
+            options?.skipDefaultLayers
+                ? new Pass("AddDefaultLayers is disabled due to the set flag")
+                : new AddDefaultLayers(state),
+            new AddDependencyLayersToTheme(state),
+            new AddImportLayers()
+        )
+        this.state = state
+    }
+
+    convert(
+        json: LayoutConfigJson,
+        context: string
+    ): { result: LayoutConfigJson; errors: string[]; warnings: string[]; information: string[] } {
+        const result = super.convert(json, context)
+        if (this.state.publicLayers.size === 0) {
+            // THis is a bootstrapping run, no need to already set this flag
+            return result
+        }
+
+        const needsNodeDatabase = result.result.layers?.some((l: LayerConfigJson) =>
+            l.tagRenderings?.some((tr: TagRenderingConfigJson) =>
+                ValidationUtils.getSpecialVisualisations(tr)?.some(
+                    (special) => special.needsNodeDatabase
+                )
+            )
+        )
+        if (needsNodeDatabase) {
+            result.information.push(
+                context +
+                    ": setting 'enableNodeDatabase' as this theme uses a special visualisation which needs to keep track of _all_ nodes"
+            )
+            result.result.enableNodeDatabase = true
+        }
+
+        return result
+    }
+}

src/Models/ThemeConfig/Conversion/ValidationUtils.ts

@@ -0,0 +1,64 @@
+import { TagRenderingConfigJson } from "../Json/TagRenderingConfigJson"
+import { Utils } from "../../../Utils"
+import SpecialVisualizations from "../../../UI/SpecialVisualizations"
+import { RenderingSpecification, SpecialVisualization } from "../../../UI/SpecialVisualization"
+import { LayerConfigJson } from "../Json/LayerConfigJson"
+
+export default class ValidationUtils {
+    public static hasSpecialVisualisation(
+        layer: LayerConfigJson,
+        specialVisualisation: string
+    ): boolean {
+        return (
+            layer.tagRenderings?.some((tagRendering) => {
+                if (tagRendering === undefined) {
+                    return false
+                }
+
+                const spec = ValidationUtils.getSpecialVisualisations(
+                    <TagRenderingConfigJson>tagRendering
+                )
+                return spec.some((vis) => vis.funcName === specialVisualisation)
+            }) ?? false
+        )
+    }
+
+    /**
+     * Gives all the (function names of) used special visualisations
+     * @param renderingConfig
+     */
+    public static getSpecialVisualisations(
+        renderingConfig: TagRenderingConfigJson
+    ): SpecialVisualization[] {
+        return ValidationUtils.getSpecialVisualsationsWithArgs(renderingConfig).map(
+            (spec) => spec["func"]
+        )
+    }
+
+    public static getSpecialVisualsationsWithArgs(
+        renderingConfig: TagRenderingConfigJson
+    ): RenderingSpecification[] {
+        const translations: any[] = Utils.NoNull([
+            renderingConfig.render,
+            ...(renderingConfig.mappings ?? []).map((m) => m.then),
+        ])
+        const all: RenderingSpecification[] = []
+        for (let translation of translations) {
+            if (typeof translation == "string") {
+                translation = { "*": translation }
+            }
+
+            for (const key in translation) {
+                if (!translation.hasOwnProperty(key)) {
+                    continue
+                }
+
+                const template = translation[key]
+                const parts = SpecialVisualizations.constructSpecification(template)
+                const specials = parts.filter((p) => typeof p !== "string")
+                all.push(...specials)
+            }
+        }
+        return all
+    }
+}