MapComplete/scripts/generateLayerOverview.ts

import ScriptUtils from "./ScriptUtils"
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "fs"
import { ThemeConfigJson } from "../src/Models/ThemeConfig/Json/ThemeConfigJson"
import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
import Constants from "../src/Models/Constants"
import {
    DetectDuplicateFilters,
    DoesImageExist,
    PrevalidateTheme,
    ValidateLayer,
    ValidateThemeEnsemble,
} from "../src/Models/ThemeConfig/Conversion/Validation"
import { Translation } from "../src/UI/i18n/Translation"
import { OrderLayer, PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
import { OrderTheme, PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme"
import {
    Conversion,
    DesugaringContext,
    DesugaringStep,
    Each,
    Fuse,
    On,
} from "../src/Models/ThemeConfig/Conversion/Conversion"
import { Utils } from "../src/Utils"
import Script from "./Script"
import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
import { parse as parse_html } from "node-html-parser"
import { ExtraFunctions } from "../src/Logic/ExtraFunctions"
import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import LayerConfig from "../src/Models/ThemeConfig/LayerConfig"
import PointRenderingConfig from "../src/Models/ThemeConfig/PointRenderingConfig"
import { ConversionContext } from "../src/Models/ThemeConfig/Conversion/ConversionContext"
import { GenerateFavouritesLayer } from "./generateFavouritesLayer"
import ThemeConfig, { MinimalThemeInformation } from "../src/Models/ThemeConfig/ThemeConfig"
import Translations from "../src/UI/i18n/Translations"
import { Translatable } from "../src/Models/ThemeConfig/Json/Translatable"
import { ValidateThemeAndLayers } from "../src/Models/ThemeConfig/Conversion/ValidateThemeAndLayers"
import { ExtractImages } from "../src/Models/ThemeConfig/Conversion/FixImages"
import { TagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/TagRenderingConfigJson"
import { Lists } from "../src/Utils/Lists"
import {
    LayerConfigDependencyGraph,
    LevelInfo,
} from "../src/Models/ThemeConfig/LayerConfigDependencyGraph"
import { AddContextToTranslations } from "../src/Models/ThemeConfig/Conversion/AddContextToTranslations"

// This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files.
// It spits out an overview of those to be used to load them

class ParseLayer extends Conversion<
    string,
    {
        parsed: LayerConfig
        raw: LayerConfigJson
    }
> {
    private readonly _prepareLayer: PrepareLayer
    private readonly _doesImageExist: DoesImageExist

    constructor(prepareLayer: PrepareLayer, doesImageExist: DoesImageExist) {
        super("ParseLayer", "Parsed a layer from file, validates it")
        this._prepareLayer = prepareLayer
        this._doesImageExist = doesImageExist
    }

    convert(
        path: string,
        context: ConversionContext
    ): {
        parsed: LayerConfig
        raw: LayerConfigJson
    } {
        let parsed
        let fileContents
        try {
            fileContents = readFileSync(path, "utf8")
        } catch (e) {
            context.err("Could not read file " + path + " due to " + e)
            return undefined
        }
        try {
            parsed = JSON.parse(fileContents)
        } catch (e) {
            context.err("Could not parse file as JSON: " + e)
            return undefined
        }
        if (parsed === undefined) {
            context.err("yielded undefined")
            return undefined
        }
        const fixed = this._prepareLayer.convert(parsed, context.inOperation("PrepareLayer"))

        if (!fixed.source && fixed.presets?.length < 1) {
            context
                .enter("source")
                .err(
                    "No source is configured. (Tags might be automatically derived if presets are given)"
                )
            return undefined
        }

        if (
            fixed.source &&
            typeof fixed.source !== "string" &&
            fixed.source?.["osmTags"] &&
            fixed.source?.["osmTags"]["and"] === undefined
        ) {
            fixed.source["osmTags"] = { and: [fixed.source["osmTags"]] }
        }

        const validator = new ValidateLayer(path, true, this._doesImageExist)
        return validator.convert(fixed, context.inOperation("ValidateLayer"))
    }
}

class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: LayerConfig }> {
    static singleton = new AddIconSummary()

    constructor() {
        super(
            "AddIconSummary",
            "Adds an icon summary ('_layerIcon') for quick reference. This previews how the layer should be shown in e.g. the filter menu"
        )
    }

    convert(json: { raw: LayerConfigJson; parsed: LayerConfig }) {
        // Add a summary of the icon
        const fixed = json.raw
        const layerConfig = json.parsed
        const pointRendering: PointRenderingConfig = layerConfig.mapRendering.find((pr) =>
            pr.location.has("point")
        )
        const defaultTags = layerConfig.baseTags
        fixed["_layerIcon"] = Lists.noNull(
            (pointRendering?.marker ?? []).map((i) => {
                const icon = i.icon?.GetRenderValue(defaultTags)?.txt
                if (!icon) {
                    return undefined
                }
                const result = { icon }
                const c = i.color?.GetRenderValue(defaultTags)?.txt
                if (c) {
                    result["color"] = c
                }
                return result
            })
        )
        return { raw: fixed, parsed: layerConfig }
    }
}

class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
    private readonly _dependencies: ReadonlyMap<string, string[]>
    private readonly _states: Map<string, "clean" | "dirty" | "changed">
    private readonly prepareLayer: PrepareLayer
    private readonly _levels: LevelInfo[]
    private readonly _loadedIds: Set<string> = new Set<string>()
    private readonly _layerConfigJsons = new Map<string, LayerConfigJson>()
    private readonly _desugaringState: DesugaringContext
    private readonly _labelBlacklist: ReadonlySet<string>

    constructor(
        layerConfigJsons: LayerConfigJson[],
        dependencies: Map<string, string[]>,
        levels: LevelInfo[],
        states: Map<string, "clean" | "dirty" | "changed">,
        sharedTagRenderings: QuestionableTagRenderingConfigJson[],
        labelBlacklist: ReadonlySet<string>
    ) {
        super("LayerBuilder", "Builds all the layers, writes them to file")
        this._levels = levels
        this._dependencies = dependencies
        this._states = states
        this._labelBlacklist = labelBlacklist
        this._desugaringState = {
            tagRenderings: LayerOverviewUtils.asDict(sharedTagRenderings),
            tagRenderingOrder: sharedTagRenderings.map((tr) => tr.id),
            sharedLayers: AllSharedLayers.getSharedLayersConfigs(),
        }
        this.prepareLayer = new PrepareLayer(this._desugaringState)
        for (const layerConfigJson of layerConfigJsons) {
            this._layerConfigJsons.set(layerConfigJson.id, layerConfigJson)
        }
    }

    public static targetPath(id: string): string {
        return `${LayerOverviewUtils.layerPath}${id}.json`
    }

    public static sourcePath(id: string): string {
        return `./assets/layers/${id}/${id}.json`
    }

    writeLayer(layer: LayerConfigJson) {
        if (layer.labels?.some((l) => this._labelBlacklist.has(l))) {
            console.log("Not writing layer " + layer.id + ", censored")
            return
        }
        layer = new CensorLayer(this._labelBlacklist).convertStrict(layer)
        if (!existsSync(LayerOverviewUtils.layerPath)) {
            mkdirSync(LayerOverviewUtils.layerPath)
        }
        writeFileSync(LayerBuilder.targetPath(layer.id), JSON.stringify(layer, null, "  "), {
            encoding: "utf8",
        })
    }

    public buildLayer(
        id: string,
        context: ConversionContext,
        isLooping: boolean = false
    ): LayerConfigJson {
        if (id === "questions") {
            return undefined
        }
        const deps = this._dependencies.get(id)
        if (!isLooping) {
            // Beware of the looping traps. Bring the leaf to the statue to teleport to "The Lab" (<ref>submachine 4</ref>)
            const unbuilt = deps.filter((depId) => !this._loadedIds.has(depId))
            for (const unbuiltId of unbuilt) {
                this.buildLayer(unbuiltId, context)
            }
        }

        context = context.inOperation("building Layer " + id).enters("layer:", id)

        const config = this._layerConfigJsons.get(id)
        if (config.id !== id) {
            context.err("Invalid ID: expected", id, "but got", id)
        }
        const prepped = this.prepareLayer.convert(config, context)
        const withContext = new AddContextToTranslations<LayerConfigJson>("layers:").convertStrict(
            prepped,
            ConversionContext.construct([prepped.id], ["AddContextToTranslations"])
        )
        this._loadedIds.add(id)
        this._desugaringState.sharedLayers.set(id, withContext)
        return withContext
    }

    private buildLooping(ids: string[], context: ConversionContext) {
        const origIds: ReadonlyArray<string> = [...ids]

        const deps = this._dependencies
        const allDeps = Lists.dedup([].concat(...ids.map((id) => deps.get(id))))
        const depsRecord = Utils.asRecord(Array.from(deps.keys()), (k) =>
            deps.get(k).filter((dep) => ids.indexOf(dep) >= 0)
        )
        const revDeps = Utils.TransposeMap(depsRecord)
        for (const someDep of allDeps) {
            if (ids.indexOf(someDep) >= 0) {
                // BY definition, we _will_ need this dependency
                // We add a small stub
                this._desugaringState.sharedLayers.set(someDep, {
                    id: someDep,
                    pointRendering: [],
                    tagRenderings: [],
                    filter: [],
                    source: "special:stub",
                    allowMove: true,
                })
                continue
            }
            // Make sure all are direct dependencies are loaded
            if (!this._loadedIds.has(someDep)) {
                this.buildLayer(someDep, context)
            }
        }
        while (ids.length > 0) {
            const first = ids.pop()
            if (first === "questions") {
                continue
            }
            const oldConfig =
                this._desugaringState.sharedLayers.get(first) ?? this._layerConfigJsons.get(first)
            const newConfig = this.buildLayer(
                first,
                context.inOperation("resolving a looped dependency"),
                true
            )
            const isDifferent = JSON.stringify(oldConfig) !== JSON.stringify(newConfig)

            if (isDifferent) {
                const toRunAgain = revDeps[first] ?? []
                for (const id of toRunAgain) {
                    if (ids.indexOf(id) < 0) {
                        ids.push(id)
                    }
                }
            }
        }
        for (const id of origIds) {
            this.writeLayer(this._desugaringState.sharedLayers.get(id))
        }
        console.log("Done with the looping layers!")
    }

    public convert(o, context: ConversionContext): Map<string, LayerConfigJson> {
        for (const level of this._levels) {
            if (level.loop) {
                this.buildLooping(level.ids, context)
                continue
            }

            for (let i = 0; i < level.ids.length; i++) {
                const id = level.ids[i]
                ScriptUtils.erasableLog(
                    `Building level ${i}: validating layer ${i + 1}/${level.ids.length}: ${id}`
                )
                if (id === "questions") {
                    continue
                }
                if (this._states.get(id) === "clean") {
                    const file = readFileSync(LayerBuilder.targetPath(id), "utf-8")
                    if (file.length > 3) {
                        try {
                            const loaded = JSON.parse(file)
                            this._desugaringState.sharedLayers.set(id, loaded)
                            continue
                        } catch (e) {
                            console.error(
                                "Could not load generated layer file for ",
                                id,
                                " building it instead"
                            )
                        }
                    }
                }
                const prepped = this.buildLayer(id, context)
                this.writeLayer(prepped)
                LayerOverviewUtils.extractJavascriptCodeForLayer(prepped)
            }
        }
        context.info("Recompiled " + this._loadedIds.size + " layers")
        return this._desugaringState.sharedLayers
    }
}

class ReorderFiles extends Script {
    constructor() {
        super("Reorders the attributes in the layers and theme files")
    }

    private lintAll<T>(items: { parsed: T; path: string }[], reorder: DesugaringStep<T>) {
        for (const item of items) {
            const l = reorder.convertStrict(
                item.parsed,
                ConversionContext.construct([item.path], ["reorder"])
            )
            const content = JSON.stringify(l, null, "  ")
            const contentOld = JSON.stringify(item.parsed, null, "  ")
            if (contentOld === content) {
                continue
            }
            const orig = readFileSync(item.path, "utf-8")
            const ending = orig.endsWith("\n") ? "\n" : ""
            writeFileSync(item.path, content + ending, "utf-8")
        }
    }

    async main() {
        console.log("Reordering layers and themes")
        const orderL = new OrderLayer()
        const layers = ScriptUtils.getLayerFiles()
        this.lintAll(layers, orderL)

        const orderT = new OrderTheme()
        const themes = ScriptUtils.getThemeFiles()
        this.lintAll(themes, orderT)
    }
}

class CensorLayer extends DesugaringStep<LayerConfigJson> {
    private readonly _excludedLabels: ReadonlySet<string>

    constructor(excludedLabels: ReadonlySet<string>) {
        super("CensorLayer", "Removes unwanted layers for specific builds (mostly play store)")
        this._excludedLabels = excludedLabels
    }

    convert(json: LayerConfigJson, context: ConversionContext): LayerConfigJson {
        json = { ...json }
        json.tagRenderings = json.tagRenderings?.filter((trs) => {
            const tr = <QuestionableTagRenderingConfigJson>trs
            const keep = !(tr.labels ?? [])?.some((l) => this._excludedLabels.has(l))
            if (!keep) {
                const forbidden = (tr.labels ?? [])?.filter((l) => this._excludedLabels.has(l))
                context.info(
                    "Dropping tagRendering " +
                        tr.id +
                        " from layer " +
                        json.id +
                        " due to forbidden label: " +
                        forbidden.join(", ")
                )
            }
            return keep
        })
        return json
    }
}

class CensorTheme extends Fuse<ThemeConfigJson & { layers: LayerConfigJson[] }> {
    private readonly _excludedLabels: ReadonlySet<string>

    constructor(excludedLabels: ReadonlySet<string>) {
        super(
            "Removes unwanted layers for specific builds (mostly play store)",
            new On("layers", new Each(new CensorLayer(excludedLabels)))
        )
        this._excludedLabels = excludedLabels
    }

    convert(
        json: ThemeConfigJson & { layers: LayerConfigJson[] },
        context: ConversionContext
    ): ThemeConfigJson & { layers: LayerConfigJson[] } {
        json = { ...json }
        const newLayers: LayerConfigJson[] = []
        for (const layer of <LayerConfigJson[]>json.layers) {
            if (layer.labels?.some((label) => this._excludedLabels.has(label))) {
                context.info(
                    "Dropping layer " +
                        layer.id +
                        " from theme " +
                        json.id +
                        " due to forbidden label: " +
                        layer.labels?.filter((l) => this._excludedLabels.has(l)).join(", ")
                )
                continue
            }
            newLayers.push(layer)
        }
        json.layers = newLayers
        super.convert(json, context)
        return json
    }
}

class LayerOverviewUtils extends Script {
    public static readonly layerPath = "./public/assets/generated/layers/"
    public static readonly themePath = "./public/assets/generated/themes/"

    constructor() {
        super(
            "Reviews and generates the compiled themes. Arguments: '[--exclude-labels=label0,label1] --themes=theme0,theme1'"
        )
    }

    private static publicLayerIdsFrom(themefiles: ThemeConfigJson[]): Set<string> {
        const publicThemes = [].concat(...themefiles.filter((th) => !th.hideFromOverview))

        return new Set([].concat(...publicThemes.map((th) => this.extractLayerIdsFrom(th))))
    }

    private static extractLayerIdsFrom(
        themeFile: ThemeConfigJson,
        includeInlineLayers = true
    ): string[] {
        const publicLayerIds: string[] = []
        if (!Array.isArray(themeFile.layers)) {
            throw (
                "Cannot iterate over 'layers' of " +
                themeFile.id +
                "; it is a " +
                typeof themeFile.layers
            )
        }
        for (const publicLayer of themeFile.layers) {
            if (typeof publicLayer === "string") {
                publicLayerIds.push(publicLayer)
                continue
            }
            if (publicLayer["builtin"] !== undefined) {
                const bi: string | string[] = publicLayer["builtin"]
                if (typeof bi === "string") {
                    publicLayerIds.push(bi)
                } else {
                    bi.forEach((id) => publicLayerIds.push(id))
                }
                continue
            }
            if (includeInlineLayers) {
                publicLayerIds.push(publicLayer["id"])
            }
        }
        return publicLayerIds
    }

    public static cleanTranslation(t: string | Record<string, string> | Translation): Translatable {
        return Translations.T(t).OnEveryLanguage((s) => parse_html(s).textContent).translations
    }

    public static shouldBeUpdated(sourcefile: string | string[], targetfile: string): boolean {
        if (!existsSync(targetfile)) {
            return true
        }
        const targetModified = statSync(targetfile).mtime
        if (typeof sourcefile === "string") {
            sourcefile = [sourcefile]
        }

        for (const path of sourcefile) {
            if (!existsSync(path)) {
                return true
            }
            const hasChange = statSync(path).mtime > targetModified
            if (hasChange) {
                return true
            }
        }
        return false
    }

    static mergeKeywords(
        into: Record<string, string[]>,
        source: Readonly<Record<string, string[]>>
    ) {
        for (const key in source) {
            if (into[key]) {
                into[key].push(...source[key])
            } else {
                into[key] = source[key]
            }
        }
    }

    private layerKeywords(l: LayerConfigJson): Record<string, string[]> {
        const keywords: Record<string, string[]> = {}

        function addWord(language: string, word: string | string[]) {
            if (Array.isArray(word)) {
                word.forEach((w) => addWord(language, w))
                return
            }

            word = Utils.SubstituteKeys(word, {})?.trim()
            if (!word) {
                return
            }
            if (!keywords[language]) {
                keywords[language] = []
            }
            keywords[language].push(word)
        }

        function addWords(
            tr: string | Record<string, string> | Record<string, string[]> | TagRenderingConfigJson
        ) {
            if (!tr) {
                return
            }
            if (typeof tr === "string") {
                addWord("*", tr)
                return
            }
            if (tr["render"] !== undefined || tr["mappings"] !== undefined) {
                tr = <TagRenderingConfigJson>tr
                addWords(<Translatable>tr.render)
                for (const mapping of tr.mappings ?? []) {
                    if (typeof mapping === "string") {
                        addWords(mapping)
                        continue
                    }
                    addWords(mapping.then)
                }
                return
            }
            for (const lang in tr) {
                addWord(lang, tr[lang])
            }
        }

        addWord("*", l.id)
        addWords(l.title)
        addWords(l.description)
        addWords(l.searchTerms)
        return keywords
    }

    writeSmallOverview(
        themes: {
            id: string
            title: Translatable
            shortDescription: Translatable
            icon: string
            hideFromOverview: boolean
            mustHaveLanguage: boolean
            layers: (
                | LayerConfigJson
                | string
                | {
                      builtin
                  }
            )[]
        }[],
        sharedLayers: Map<string, LayerConfigJson>
    ) {
        const layerKeywords: Record<string, Record<string, string[]>> = {}

        sharedLayers.forEach((layer, id) => {
            layerKeywords[id] = this.layerKeywords(layer)
        })

        const perId = new Map<string, MinimalThemeInformation>()
        for (const theme of themes) {
            const keywords: Record<string, string[]> = {}
            for (const layer of theme.layers ?? []) {
                const l = <LayerConfigJson>layer
                if (sharedLayers.has(l.id)) {
                    continue
                }
                LayerOverviewUtils.mergeKeywords(keywords, this.layerKeywords(l))
            }

            const data = <MinimalThemeInformation>{
                id: theme.id,
                title: theme.title,
                shortDescription: LayerOverviewUtils.cleanTranslation(theme.shortDescription),
                icon: theme.icon,
                hideFromOverview: theme.hideFromOverview,
                mustHaveLanguage: theme.mustHaveLanguage,
                keywords,
                layers: (<LayerConfigJson[]>theme.layers)
                    .filter((l) => sharedLayers.has(l.id))
                    .filter((l) => l.minzoom < 17)
                    .map((l) => l.id),
            }
            perId.set(data.id, data)
        }

        const sorted = Constants.themeOrder.map((id) => {
            if (!perId.has(id)) {
                throw "Ordered theme '" + id + "' not found"
            }
            return perId.get(id)
        })

        perId.forEach((value) => {
            if (Constants.themeOrder.indexOf(value.id) >= 0) {
                return // actually a continue
            }
            sorted.push(value)
        })

        writeFileSync(
            "./src/assets/generated/theme_overview.json",
            JSON.stringify(
                {
                    "#": "Generated by generateLayerOverview",
                    "#version": new Date().toISOString(),
                    layers: layerKeywords,
                    themes: sorted,
                },
                null,
                "  "
            ),
            { encoding: "utf8" }
        )
    }

    writeTheme(theme: ThemeConfigJson) {
        if (!existsSync(LayerOverviewUtils.themePath)) {
            mkdirSync(LayerOverviewUtils.themePath)
        }

        writeFileSync(
            `${LayerOverviewUtils.themePath}${theme.id}.json`,
            JSON.stringify(theme, null, "  "),
            { encoding: "utf8" }
        )
    }

    static asDict(
        trs: QuestionableTagRenderingConfigJson[]
    ): Map<string, QuestionableTagRenderingConfigJson> {
        const d = new Map<string, QuestionableTagRenderingConfigJson>()
        for (const tr of trs) {
            d.set(tr.id, tr)
        }
        return d
    }

    getSharedTagRenderings(doesImageExist: DoesImageExist): QuestionableTagRenderingConfigJson[]
    getSharedTagRenderings(
        doesImageExist: DoesImageExist,
        bootstrapTagRenderings: Map<string, QuestionableTagRenderingConfigJson>,
        bootstrapTagRenderingsOrder: string[]
    ): QuestionableTagRenderingConfigJson[]
    getSharedTagRenderings(
        doesImageExist: DoesImageExist,
        bootstrapTagRenderings: Map<string, QuestionableTagRenderingConfigJson> = null,
        bootstrapTagRenderingsOrder: string[] = []
    ): QuestionableTagRenderingConfigJson[] {
        const prepareLayer = new PrepareLayer(
            {
                tagRenderings: bootstrapTagRenderings,
                tagRenderingOrder: bootstrapTagRenderingsOrder,
                sharedLayers: null,
                publicLayers: null,
            },
            {
                addTagRenderingsToContext: true,
            }
        )

        const path = "assets/layers/questions/questions.json"
        const sharedQuestionsRaw: LayerConfigJson = this.parseLayer(
            doesImageExist,
            prepareLayer,
            path
        ).raw
        const sharedQuestions: LayerConfigJson = new AddContextToTranslations<LayerConfigJson>(
            ""
        ).convertStrict(sharedQuestionsRaw, ConversionContext.construct(["layers:questions"], []))
        const dict = new Map<string, QuestionableTagRenderingConfigJson>()

        for (const tr of sharedQuestions.tagRenderings) {
            const tagRendering = <QuestionableTagRenderingConfigJson>tr
            tagRendering._definedIn = ["questions", tr["id"]]
            dict.set(tagRendering["id"], tagRendering)
        }

        if (dict.size === bootstrapTagRenderings?.size) {
            return <QuestionableTagRenderingConfigJson[]>sharedQuestions.tagRenderings
        }

        return this.getSharedTagRenderings(
            doesImageExist,
            dict,
            sharedQuestions.tagRenderings.map((tr) => tr["id"])
        )
    }

    checkAllSvgs() {
        const allSvgs = ScriptUtils.readDirRecSync("./src/assets")
            .filter((path) => path.endsWith(".svg"))
            .filter((path) => !path.startsWith("./src/assets/generated"))
        let errCount = 0
        const exempt = [
            "src/assets/SocialImageTemplate.svg",
            "src/assets/SocialImageTemplateWide.svg",
            "src/assets/SocialImageBanner.svg",
            "src/assets/SocialImageRepo.svg",
            "src/assets/svg/osm-logo.svg",
            "src/assets/templates/*",
        ]
        for (const path of allSvgs) {
            if (
                exempt.some((p) => {
                    if (p.endsWith("*") && path.startsWith("./" + p.substring(0, p.length - 1))) {
                        return true
                    }
                    return "./" + p === path
                })
            ) {
                continue
            }

            const contents = readFileSync(path, { encoding: "utf8" })
            if (contents.indexOf("data:image/png;") >= 0) {
                console.warn("The SVG at " + path + " is a fake SVG: it contains PNG data!")
                errCount++
                if (path.startsWith("./src/assets/svg")) {
                    throw "A core SVG is actually a PNG. Don't do this!"
                }
            }
            if (contents.indexOf("<text") > 0) {
                console.warn(
                    "The SVG at " +
                        path +
                        " contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path"
                )
                errCount++
            }
        }
        if (errCount > 0) {
            throw `There are ${errCount} invalid svgs`
        }
    }

    async main(args: string[]) {
        console.log("Generating layer overview...")
        const themeWhitelist = new Set(
            args
                .find((a) => a.startsWith("--themes="))
                ?.substring("--themes=".length)
                ?.split(",") ?? []
        )
        const labelBlacklist = new Set(
            args
                .find((a) => a.startsWith("--exclude-labels="))
                ?.substring("--exclude-labels=".length)
                ?.split(",") ?? []
        )

        const forceReload = args.some((a) => a == "--force") || labelBlacklist.size > 0
        const printAssets = args.some((a) => a === "--print-needed-assets")
        console.log("Arguments are:", { labelBlacklist, themeWhitelist, forceReload })
        const doesImageExist = DoesImageExist.constructWithLicenses(existsSync)
        const sharedLayers = this.buildLayerIndex(doesImageExist, labelBlacklist)

        const priviliged = new Set<string>(Constants.priviliged_layers)
        sharedLayers.forEach((_, key) => {
            priviliged.delete(key)
        })

        // These get a free pass
        priviliged.delete("summary")
        priviliged.delete("last_click")
        priviliged.delete("search")

        const isBoostrapping = AllSharedLayers.sharedLayers.size == 0
        if (!isBoostrapping && priviliged.size > 0) {
            throw (
                "Priviliged layer " +
                Array.from(priviliged).join(", ") +
                " has no definition file, create it at `src/assets/layers/<layername>/<layername.json>"
            )
        }
        const recompiledThemes: string[] = []
        const sharedThemes = this.buildThemeIndex(
            sharedLayers,
            recompiledThemes,
            forceReload,
            themeWhitelist,
            labelBlacklist
        )

        new ValidateThemeEnsemble().convertStrict(
            Array.from(sharedThemes.values()).map((th) => new ThemeConfig(th, true))
        )

        if (recompiledThemes.length > 0) {
            writeFileSync(
                "./src/assets/generated/known_layers.json",
                JSON.stringify({
                    layers: Array.from(sharedLayers.values()).filter(
                        (l) => !(l["#no-index"] === "yes")
                    ),
                })
            )
        }

        const mcChangesPath = "./assets/themes/mapcomplete-changes/mapcomplete-changes.json"
        if (
            (recompiledThemes.length > 0 &&
                !(
                    recompiledThemes.length === 1 && recompiledThemes[0] === "mapcomplete-changes"
                )) ||
            args.indexOf("--generate-change-map") >= 0 ||
            !existsSync(mcChangesPath)
        ) {
            // mapcomplete-changes shows an icon for each corresponding mapcomplete-theme
            const iconsPerTheme = Array.from(sharedThemes.values()).map((th) => ({
                if: "theme=" + th.id,
                then: th.icon,
            }))
            const proto: ThemeConfigJson = JSON.parse(
                readFileSync("./assets/themes/mapcomplete-changes/mapcomplete-changes.proto.json", {
                    encoding: "utf8",
                })
            )
            const protolayer = <LayerConfigJson>(
                proto.layers.filter((l) => l["id"] === "mapcomplete-changes")[0]
            )
            const rendering = protolayer.pointRendering[0]
            rendering.marker[0].icon["mappings"] = iconsPerTheme
            writeFileSync(mcChangesPath, JSON.stringify(proto, null, "  "))
        }

        this.checkAllSvgs()

        new DetectDuplicateFilters().convertStrict(
            {
                layers: ScriptUtils.getLayerFiles().map((f) => f.parsed),
                themes: ScriptUtils.getThemeFiles().map((f) => f.parsed),
            },
            ConversionContext.construct([], [])
        )

        for (const [_, theme] of sharedThemes) {
            theme.layers = theme.layers.filter(
                (l) => Constants.added_by_default.indexOf(l["id"]) < 0
            )
        }
        if (printAssets) {
            const images = Lists.dedup(
                Array.from(sharedThemes.values()).flatMap((th) => th._usedImages ?? [])
            )
            writeFileSync("needed_assets.csv", images.join("\n"))
            console.log("Written needed_assets.csv")
        }
    }

    private parseLayer(
        doesImageExist: DoesImageExist,
        prepLayer: PrepareLayer,
        sharedLayerPath: string
    ): {
        raw: LayerConfigJson
        parsed: LayerConfig
        context: ConversionContext
    } {
        const parser = new ParseLayer(prepLayer, doesImageExist)
        const context = ConversionContext.construct([sharedLayerPath], ["ParseLayer"])
        const parsed = parser.convertStrict(sharedLayerPath, context)
        const result = AddIconSummary.singleton.convertStrict(
            parsed,
            context.inOperation("AddIconSummary")
        )
        return { ...result, context }
    }

    private getAllLayerConfigs(): LayerConfigJson[] {
        const allPaths = ScriptUtils.getLayerPaths()
        const results: LayerConfigJson[] = []
        for (let i = 0; i < allPaths.length; i++) {
            const path = allPaths[i]
            ScriptUtils.erasableLog(
                `Parsing layerConfig ${i + 1}/${allPaths.length}: ${path}                   `
            )
            const expectedId = path.match(/.*\/([a-z_0-9]+).json/)[1]
            try {
                const data = JSON.parse(readFileSync(path, "utf8"))
                results.push(data)
                if (data.id !== expectedId) {
                    throw (
                        "Wrong ID or location for file " +
                        path +
                        "; expected " +
                        expectedId +
                        " but got " +
                        data.id
                    )
                }
            } catch (e) {
                throw "Could not parse layer file " + path + " due to " + e
            }
        }

        return results
    }

    private buildLayerIndex(
        doesImageExist: DoesImageExist,
        labelBlacklist: Set<string>
    ): Map<string, LayerConfigJson> {
        // First, we expand and validate all builtin layers. These are written to src/assets/generated/layers
        // At the same time, an index of available layers is built.
        const sharedQuestions = this.getSharedTagRenderings(doesImageExist)
        const allLayerConfigs = this.getAllLayerConfigs()
        const sharedQuestionsDef = allLayerConfigs.find((l) => l.id === "questions")
        sharedQuestionsDef.tagRenderings = sharedQuestions

        const dependencyGraph = LayerConfigDependencyGraph.buildDirectDependencies(allLayerConfigs)
        const levels = LayerConfigDependencyGraph.buildLevels(dependencyGraph)
        const layerState = new Map<string, "clean" | "dirty" | "changed">()
        console.log("# BUILD PLAN\n\n")
        for (const levelInfo of levels) {
            if (levelInfo.loop) {
                console.log(`(LOOP)`)
            }
            let allClean = true
            for (const id of levelInfo.ids) {
                const deps = dependencyGraph.get(id) ?? []
                const dirtyDeps = deps.filter((dep) => {
                    const depState = layerState.get(dep)
                    if (levelInfo.loop && depState === undefined) {
                        const depIsClean = LayerOverviewUtils.shouldBeUpdated(
                            LayerBuilder.sourcePath(dep),
                            LayerBuilder.targetPath(dep)
                        )
                        if (depIsClean) {
                            return false
                        }
                    }
                    return depState !== "clean"
                })
                if (dirtyDeps.length > 0) {
                    layerState.set(id, "dirty")
                } else {
                    const sourcePath = `./assets/layers/${id}/${id}.json`
                    const targetPath = `./public/assets/generated/layers/${id}.json`

                    if (LayerOverviewUtils.shouldBeUpdated(sourcePath, targetPath)) {
                        layerState.set(id, "changed")
                    } else {
                        layerState.set(id, "clean")
                    }
                }
                const state = layerState.get(id)
                if (state !== "clean") {
                    allClean = false
                    console.log(`- ${id} (${state}; ${dirtyDeps.map((dd) => dd + "*").join(", ")})`)
                }
            }
            if (!allClean) {
                console.log("\n")
            }
        }

        const builder = new LayerBuilder(
            allLayerConfigs,
            dependencyGraph,
            levels,
            layerState,
            sharedQuestions,
            labelBlacklist
        )
        builder.writeLayer(sharedQuestionsDef)
        const allLayers = builder.convertStrict(
            {},
            ConversionContext.construct([], ["Building the layer index"])
        )
        if (layerState.get("usersettings") !== "clean") {
            // We always need the calculated tags of 'usersettings', so we export them separately if dirty

            LayerOverviewUtils.extractJavascriptCodeForLayer(
                allLayers.get("usersettings"),
                "./src/Logic/State/UserSettingsMetaTagging.ts"
            )
        }

        return allLayers
    }

    /**
     * Given: a fully expanded themeConfigJson
     *
     * Will extract a dictionary of the special code and write it into a javascript file which can be imported.
     * This removes the need for _eval_, allowing for a correct CSP
     * @param themeFile
     * @private
     */
    private extractJavascriptCode(themeFile: ThemeConfigJson) {
        const allCode = [
            "import {Feature} from 'geojson'",
            'import { ExtraFuncType } from "../../../Logic/ExtraFunctions";',
            'import { Utils } from "../../../Utils"',
            "export class ThemeMetaTagging {",
            "   public static readonly themeName = " + JSON.stringify(themeFile.id),
            "",
        ]
        for (const layer of themeFile.layers) {
            const l = <LayerConfigJson>layer
            const id = l.id.replace(/[^a-zA-Z0-9_]/g, "_")
            const code = l.calculatedTags ?? []

            allCode.push(
                "   public metaTaggging_for_" +
                    id +
                    "(feat: Feature, helperFunctions: Record<ExtraFuncType, (feature: Feature) => Function>) {"
            )
            if (code?.length > 0) {
                allCode.push(
                    "      const {" + ExtraFunctions.types.join(", ") + "} = helperFunctions"
                )
            }
            for (const line of code) {
                const firstEq = line.indexOf("=")
                let attributeName = line.substring(0, firstEq).trim()
                const expression = line.substring(firstEq + 1)
                const isStrict = attributeName.endsWith(":")
                if (!isStrict) {
                    allCode.push(
                        "      Utils.AddLazyProperty(feat.properties, '" +
                            attributeName +
                            "', () => " +
                            expression +
                            " ) "
                    )
                } else {
                    attributeName = attributeName.substring(0, attributeName.length - 1).trim()
                    allCode.push("      feat.properties['" + attributeName + "'] = " + expression)
                }
            }
            allCode.push("   }")
        }

        const targetDir = "./src/assets/generated/metatagging/"
        if (!existsSync(targetDir)) {
            mkdirSync(targetDir)
        }
        allCode.push("}")

        writeFileSync(targetDir + themeFile.id + ".ts", allCode.join("\n"))
    }

    public static extractJavascriptCodeForLayer(l: LayerConfigJson, targetPath?: string) {
        if (!l) {
            return // Probably a bootstrapping run
        }
        let importPath = "../../../"
        if (targetPath) {
            const l = targetPath.split("/")
            if (l.length == 1) {
                importPath = "./"
            } else {
                importPath = ""
                for (let i = 0; i < l.length - 3; i++) {
                    importPath += "../"
                }
            }
        }
        const allCode = [
            `import { Utils } from "${importPath}Utils"`,
            `/** This code is autogenerated - do not edit. Edit ./assets/layers/${l?.id}/${l?.id}.json instead */`,
            "export class ThemeMetaTagging {",
            "   public static readonly themeName = " + JSON.stringify(l.id),
            "",
        ]
        const code = l.calculatedTags ?? []

        allCode.push(
            "   public metaTaggging_for_" + l.id + "(feat: {properties: Record<string, string>}) {"
        )
        for (const line of code) {
            const firstEq = line.indexOf("=")
            let attributeName = line.substring(0, firstEq).trim()
            const expression = line.substring(firstEq + 1)
            const isStrict = attributeName.endsWith(":")
            if (!isStrict) {
                allCode.push(
                    "      Utils.AddLazyProperty(feat.properties, '" +
                        attributeName +
                        "', () => " +
                        expression +
                        " ) "
                )
            } else {
                attributeName = attributeName.substring(0, attributeName.length - 2).trim()
                allCode.push("      feat.properties['" + attributeName + "'] = " + expression)
            }
        }
        allCode.push("   }")
        allCode.push("}")

        const targetDir = "./src/assets/generated/metatagging/"
        if (!targetPath) {
            if (!existsSync(targetDir)) {
                mkdirSync(targetDir)
            }
        }

        writeFileSync(targetPath ?? targetDir + "layer_" + l.id + ".ts", allCode.join("\n"))
    }

    private buildThemeIndex(
        sharedLayers: Map<string, LayerConfigJson>,
        recompiledThemes: string[],
        forceReload: boolean,
        whitelist: ReadonlySet<string>,
        labelBlacklist: ReadonlySet<string>
    ): Map<string, ThemeConfigJson> {
        console.log("   ---------- VALIDATING BUILTIN THEMES ---------")
        const themeFiles = ScriptUtils.getThemeFiles()
        const fixed = new Map<string, ThemeConfigJson>()

        const publicLayers = LayerOverviewUtils.publicLayerIdsFrom(
            themeFiles.map((th) => th.parsed)
        )

        const trs = this.getSharedTagRenderings(DoesImageExist.constructWithLicenses(existsSync))

        const convertState: DesugaringContext = {
            sharedLayers,
            tagRenderings: LayerOverviewUtils.asDict(trs),
            tagRenderingOrder: trs.map((tr) => tr.id),
            publicLayers,
        }
        const knownTagRenderings = new Set<string>()
        convertState.tagRenderings.forEach((_, key) => knownTagRenderings.add(key))
        sharedLayers.forEach((layer) => {
            for (const tagRendering of layer.tagRenderings ?? []) {
                if (tagRendering["id"]) {
                    knownTagRenderings.add(layer.id + "." + tagRendering["id"])
                }
                if (tagRendering["labels"]) {
                    for (const label of tagRendering["labels"]) {
                        knownTagRenderings.add(layer.id + "." + label)
                    }
                }
            }
        })

        const skippedThemes: string[] = []
        const censorTheme = new CensorTheme(labelBlacklist)
        for (let i = 0; i < themeFiles.length; i++) {
            const themeInfo = themeFiles[i]
            const themePath = themeInfo.path
            let themeFile = themeInfo.parsed
            if (!themeFile) {
                throw "Got an empty file for" + themeInfo.path
            }
            if (whitelist.size > 0 && !whitelist.has(themeFile.id)) {
                continue
            }

            if (themeFile.labels?.some((l) => labelBlacklist.has(l))) {
                console.log("Skipping theme due to label", themeFile.id)
                continue
            }

            const targetPath =
                LayerOverviewUtils.themePath + "/" + themePath.substring(themePath.lastIndexOf("/"))

            const usedLayers = Array.from(
                LayerOverviewUtils.extractLayerIdsFrom(themeFile, false)
            ).map((id) => LayerOverviewUtils.layerPath + id + ".json")

            if (
                !forceReload &&
                !LayerOverviewUtils.shouldBeUpdated([themePath, ...usedLayers], targetPath)
            ) {
                const parsed = <ThemeConfigJson>(
                    JSON.parse(
                        readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", "utf8")
                    )
                )

                skippedThemes.push(themeFile.id)

                ScriptUtils.erasableLog("Skipping", themeFile.id)
                fixed.set(themeFile.id, parsed)
                continue
            }

            recompiledThemes.push(themeFile.id)

            new PrevalidateTheme().convertStrict(
                themeFile,
                ConversionContext.construct([themePath], ["PrepareLayer"])
            )
            try {
                themeFile = new PrepareTheme(convertState, {
                    skipDefaultLayers: true,
                }).convertStrict(
                    themeFile,
                    ConversionContext.construct([themePath], ["PrepareLayer"])
                )
                new ValidateThemeAndLayers(
                    DoesImageExist.constructWithLicenses(existsSync, knownTagRenderings),
                    themePath,
                    true,
                    knownTagRenderings
                ).convertStrict(
                    themeFile,
                    ConversionContext.construct([themePath], ["PrepareLayer"])
                )
                if (themeFile.labels?.some((l) => labelBlacklist.has(l))) {
                    continue
                }

                themeFile = censorTheme.convertStrict(
                    <any>themeFile,
                    ConversionContext.construct([themePath], ["Censoring"])
                )

                if (themeFile.icon.endsWith(".svg")) {
                    try {
                        ScriptUtils.ReadSvgSync(themeFile.icon, (svg) => {
                            const width: string = svg["$"].width
                            if (width === undefined) {
                                throw (
                                    "The logo at " +
                                    themeFile.icon +
                                    " does not have a defined width"
                                )
                            }
                            const height: string = svg["$"].height
                            const err = themeFile.hideFromOverview ? console.warn : console.error
                            if (width !== height) {
                                const e =
                                    `the icon for theme ${themeFile.id} is not square. Please square the icon at ${themeFile.icon}` +
                                    ` Width = ${width} height = ${height}`
                                err(e)
                            }

                            if (width?.endsWith("%")) {
                                throw (
                                    "The logo at " +
                                    themeFile.icon +
                                    " has a relative width; this is not supported"
                                )
                            }

                            const w = Number(width)
                            const h = Number(height)
                            if (w < 370 || h < 370) {
                                const e: string = [
                                    `the icon for theme ${themeFile.id} is too small. Please rescale the icon at ${themeFile.icon}`,
                                    `Even though an SVG is 'infinitely scaleable', the icon should be dimensioned bigger. One of the build steps of the theme does convert the image to a PNG (to serve as PWA-icon) and having a small dimension will cause blurry images.`,
                                    ` Width = ${width} height = ${height}; we recommend a size of at least 500px * 500px and to use a square aspect ratio.`,
                                ].join("\n")
                                err(e)
                            }
                        })
                    } catch (e) {
                        console.error("Could not read " + themeFile.icon + " due to " + e)
                    }
                }

                const usedImages = Lists.dedup(
                    new ExtractImages(true, knownTagRenderings)
                        .convertStrict(themeFile)
                        .map((x) => x.path)
                )
                usedImages.sort()

                themeFile["_usedImages"] = usedImages

                this.writeTheme(themeFile)
                fixed.set(themeFile.id, themeFile)

                this.extractJavascriptCode(themeFile)
            } catch (e) {
                console.error("ERROR: could not prepare theme " + themePath + " due to " + e)
                throw e
            }
        }

        if (whitelist.size == 0) {
            this.writeSmallOverview(
                Array.from(fixed.values()).map((t) => {
                    return <any>{
                        ...t,
                        hideFromOverview: t.hideFromOverview ?? false,
                        shortDescription:
                            t.shortDescription ??
                            new Translation(t.description).FirstSentence(true),
                        mustHaveLanguage: t.mustHaveLanguage?.length > 0,
                    }
                }),
                sharedLayers
            )
        }

        console.log(
            "Recompiled themes " +
                recompiledThemes.join(", ") +
                " and skipped " +
                skippedThemes.length +
                " themes"
        )

        return fixed
    }
}

new GenerateFavouritesLayer().run()
new LayerOverviewUtils().run()
new ReorderFiles().run()