Merge develop

scripts/GenerateSeries.ts

@@ -1,4 +1,4 @@
-import fs, { existsSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
+import { existsSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
 import ScriptUtils from "./ScriptUtils"
 import { Utils } from "../Utils"
 import Script from "./Script"
@@ -56,9 +56,9 @@ class StatsDownloader {
                     }.day.json`
                     writtenFiles.push(path)
                     if (existsSync(path)) {
-                        let features = JSON.parse(readFileSync(path, { encoding: "utf-8" }))
-                        features = features?.features ?? features
-                        features.push(...features.features) // day-stats are generally a list already, but in some ad-hoc cases might be a geojson-collection too
+                        let loadedFeatures = JSON.parse(readFileSync(path, { encoding: "utf-8" }))
+                        loadedFeatures = loadedFeatures?.features ?? loadedFeatures
+                        features.push(...loadedFeatures) // day-stats are generally a list already, but in some ad-hoc cases might be a geojson-collection too
                         console.log(
                             "Loaded ",
                             path,
@@ -296,7 +296,7 @@ class GenerateSeries extends Script {
                 features.forEach((f) => {
                     delete f.bbox
                 })
-                fs.writeFileSync(
+                writeFileSync(
                     path,
                     JSON.stringify(
                         {

scripts/automoveTranslations.ts

@@ -1,4 +1,3 @@
-import * as languages from "../assets/generated/used_languages.json"
 import { readFileSync, writeFileSync } from "fs"
 
 /**

scripts/build.sh

@@ -9,6 +9,8 @@ rm -rf .cache
 mkdir dist 2> /dev/null
 mkdir dist/assets 2> /dev/null
 
+export NODE_OPTIONS="--max-old-space-size=8192"
+
 # This script ends every line with '&&' to chain everything. A failure will thus stop the build
 npm run generate:editor-layer-index &&
 npm run generate &&
@@ -33,9 +35,17 @@ then
     echo "Source maps are enabled"
 fi
 
-ASSET_URL="mc/$BRANCH"
-export ASSET_URL
-echo "$ASSET_URL"
+if [ $BRANCH = "master" ]
+then
+    ASSET_URL="./"
+    export ASSET_URL
+    echo "$ASSET_URL"
+else
+  ASSET_URL="mc/$BRANCH"
+  export ASSET_URL
+  echo "$ASSET_URL"
+fi
+
 export NODE_OPTIONS=--max-old-space-size=6500
 vite build $SRC_MAPS 
 
@@ -48,3 +58,5 @@ cp -r assets/templates/ dist/assets/templates/
 cp -r assets/tagRenderings/ dist/assets/tagRenderings/
 cp assets/*.png dist/assets/
 cp assets/*.svg dist/assets/
+
+export NODE_OPTIONS=""

scripts/downloadEli.ts

@@ -0,0 +1,87 @@
+import Script from "./Script"
+import { Utils } from "../Utils"
+import { FeatureCollection } from "geojson"
+import fs from "fs"
+
+class DownloadEli extends Script {
+    constructor() {
+        super("Downloads a fresh copy of the editor layer index, removes all unnecessary data.")
+    }
+    async main(args: string[]): Promise<void> {
+        const url = "https://osmlab.github.io/editor-layer-index/imagery.geojson"
+        // Target should use '.json' instead of '.geojson', as the latter cannot be imported by the build systems
+        const target = args[0] ?? "assets/editor-layer-index.json"
+
+        const eli = <FeatureCollection>await Utils.downloadJson(url)
+        const keptLayers = []
+        console.log("Got", eli.features.length, "ELI-entries")
+        for (let layer of eli.features) {
+            const props = layer.properties
+
+            if (props.type === "bing") {
+                // A lot of work to implement - see https://github.com/pietervdvn/MapComplete/issues/648
+                continue
+            }
+
+            if (props.id === "MAPNIK") {
+                // Already added by default
+                continue
+            }
+
+            if (props.overlay) {
+                continue
+            }
+
+            if (props.url.toLowerCase().indexOf("apikey") > 0) {
+                continue
+            }
+
+            if (props.permission_url === "no") {
+                continue
+            }
+
+            if (props.max_zoom < 19) {
+                // We want users to zoom to level 19 when adding a point
+                // If they are on a layer which hasn't enough precision, they can not zoom far enough. This is confusing, so we don't use this layer
+                continue
+            }
+
+            if (props.name === undefined) {
+                console.warn("Editor layer index: name not defined on ", props)
+                continue
+            }
+
+            const keptKeys = [
+                "name",
+                "id",
+                "url",
+                "attribution",
+                "type",
+                "category",
+                "min_zoom",
+                "max_zoom",
+                "best",
+                "default",
+                "tile-size",
+            ]
+            layer.properties = {}
+            for (const keptKey of keptKeys) {
+                if (props[keptKey]) {
+                    layer.properties[keptKey] = props[keptKey]
+                }
+            }
+
+            layer = { properties: layer.properties, type: layer.type, geometry: layer.geometry }
+            keptLayers.push(layer)
+        }
+
+        const contents =
+            '{"type":"FeatureCollection",\n  "features": [\n' +
+            keptLayers.map((l) => JSON.stringify(l)).join(",\n") +
+            "\n]}"
+        fs.writeFileSync(target, contents, { encoding: "utf8" })
+        console.log("Written", keptLayers.length + ", entries to the ELI")
+    }
+}
+
+new DownloadEli().run()

scripts/extractLayer.ts

@@ -15,7 +15,7 @@ function main(args: string[]) {
     const layerId = args[1]
 
     const themePath = "./assets/themes/" + themeId + "/" + themeId + ".json"
-    const contents = <LayoutConfigJson>JSON.parse(readFileSync(themePath, "UTF-8"))
+    const contents = <LayoutConfigJson>JSON.parse(readFileSync(themePath, { encoding: "utf8" }))
     const layers = <LayerConfigJson[]>contents.layers.filter((l) => {
         if (typeof l === "string") {
             return false

scripts/fetchLanguages.ts

@@ -7,10 +7,22 @@ import * as wds from "wikidata-sdk"
 import { Utils } from "../Utils"
 import ScriptUtils from "./ScriptUtils"
 import { existsSync, readFileSync, writeFileSync } from "fs"
-import { QuestionableTagRenderingConfigJson } from "../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
-import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
 import WikidataUtils from "../Utils/WikidataUtils"
 import LanguageUtils from "../Utils/LanguageUtils"
+import Wikidata from "../Logic/Web/Wikidata"
+
+interface value<T> {
+    value: T
+    type: "uri" | "literal" | string
+    "xml:lang"?: string
+}
+
+interface LanguageSpecResult {
+    directionalityLabel: value<string | "right-to-left" | "left-to-right">
+    lang: value<string>
+    code: value<string>
+    label: value<string>
+}
 
 async function fetch(target: string) {
     const regular = await fetchRegularLanguages()
@@ -36,33 +48,52 @@ async function fetchRegularLanguages() {
 
     // request the generated URL with your favorite HTTP request library
     const result = await Utils.downloadJson(url, { "User-Agent": "MapComplete script" })
-    const bindings = result.results.bindings
+    const bindings = <LanguageSpecResult[]>result.results.bindings
 
     const zh_hant = await fetchSpecial(18130932, "zh_Hant")
     const zh_hans = await fetchSpecial(13414913, "zh_Hant")
     const pt_br = await fetchSpecial(750553, "pt_BR")
+    const punjabi = await fetchSpecial(58635, "pa_PK")
+    const Shahmukhi = await Wikidata.LoadWikidataEntryAsync(133800)
+    punjabi.forEach((item) => {
+        const neededLanguage = item.label["xml:lang"]
+        const native = Shahmukhi.labels.get(neededLanguage) ?? Shahmukhi.labels.get("en")
+        item.label.value = item.label.value + " (" + native + ")"
+    })
+
     const fil = await fetchSpecial(33298, "fil")
 
     bindings.push(...zh_hant)
     bindings.push(...zh_hans)
     bindings.push(...pt_br)
     bindings.push(...fil)
+    bindings.push(...punjabi)
 
     return result.results.bindings
 }
 
-async function fetchSpecial(id: number, code: string) {
+/**
+ * Fetches the object as is. Sets a 'code' binding as predifined value
+ * @param id
+ * @param code
+ */
+async function fetchSpecial(id: number, code: string): Promise<LanguageSpecResult[]> {
     ScriptUtils.fixUtils()
     console.log("Fetching languages")
 
+    const lang = "  wd:Q" + id
     const sparql =
-        "SELECT ?lang ?label ?code \n" +
+        "SELECT ?label ?directionalityLabel \n" +
         "WHERE \n" +
         "{ \n" +
-        "  wd:Q" +
-        id +
-        " rdfs:label ?label. \n" +
+        lang +
+        " rdfs:label ?label." +
+        lang +
+        " wdt:P282 ?writing_system. \n" +
+        "  ?writing_system wdt:P1406 ?directionality. \n" +
+        '  SERVICE wikibase:label { bd:serviceParam wikibase:language "en". } \n' +
         "} "
+    console.log("Special sparql:", sparql)
     const url = wds.sparqlQuery(sparql)
 
     const result = await Utils.downloadJson(url, { "User-Agent": "MapComplete script" })

scripts/fixQuestionHint.ts

@@ -0,0 +1,117 @@
+import * as fs from "fs"
+import { DesugaringStep } from "../Models/ThemeConfig/Conversion/Conversion"
+import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
+import { QuestionableTagRenderingConfigJson } from "../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
+import * as fakedom from "fake-dom"
+import Script from "./Script"
+import { FixedUiElement } from "../UI/Base/FixedUiElement"
+
+class ExtractQuestionHint extends DesugaringStep<QuestionableTagRenderingConfigJson> {
+    constructor() {
+        super(
+            "Tries to extract a 'questionHint' from the question",
+            ["question", "questionhint"],
+            "ExtractQuestionHint"
+        )
+    }
+
+    convert(
+        json: QuestionableTagRenderingConfigJson,
+        context: string
+    ): {
+        result: QuestionableTagRenderingConfigJson
+        errors?: string[]
+        warnings?: string[]
+        information?: string[]
+    } {
+        json = { ...json }
+        if (json.question === undefined || json.questionHint !== undefined) {
+            return { result: json }
+        }
+
+        if (typeof json.question === "string") {
+            return { result: json }
+        }
+
+        const hint: Record<string, string> = {}
+
+        for (const language in json.question) {
+            const q = json.question[language]
+            const parts = q.split(/<br ?\/>/i)
+            if (parts.length == 2) {
+                json.question[language] = parts[0]
+                const txt = new FixedUiElement(parts[1]).ConstructElement().textContent
+                if (txt.length > 0) {
+                    hint[language] = txt
+                }
+                continue
+            }
+
+            const divStart = [q.indexOf("<div "), q.indexOf("<span "), q.indexOf("<p ")].find(
+                (i) => i > 0
+            ) // note: > 0, not >= : we are not interested in a span starting right away!
+            if (divStart > 0) {
+                json.question[language] = q.substring(0, divStart)
+                const txt = new FixedUiElement(q.substring(divStart)).ConstructElement().textContent
+                if (txt !== "") {
+                    hint[language] = txt
+                }
+            }
+        }
+        if (Object.keys(hint).length > 0) {
+            json.questionHint = hint
+        }
+
+        console.log("Inspecting ", json.question)
+
+        return { result: json }
+    }
+}
+
+class FixQuestionHint extends Script {
+    private fs: any
+    constructor() {
+        super("Extracts a 'questionHint' from a question for a given 'layer.json' or 'theme.json'")
+        if (fakedom === undefined) {
+            throw "Fakedom not active"
+        }
+    }
+
+    async main(args: string[]): Promise<void> {
+        const filepath = args[0]
+        const contents = JSON.parse(fs.readFileSync(filepath, { encoding: "utf8" }))
+        const convertor = new ExtractQuestionHint()
+        if (filepath.endsWith("/questions.json")) {
+            for (const key in contents) {
+                const tr = contents[key]
+                if (typeof tr !== "object") {
+                    continue
+                }
+                contents[key] = convertor.convertStrict(
+                    tr,
+                    "While automatically extracting questiondHints of " + filepath
+                )
+            }
+            fs.writeFileSync(filepath, JSON.stringify(contents, null, "  "), { encoding: "utf-8" })
+
+            return
+        }
+        const layers: LayerConfigJson[] = contents["layers"] ?? [contents]
+        for (const layer of layers) {
+            for (let i = 0; i < layer.tagRenderings?.length; i++) {
+                const tagRendering = layer.tagRenderings[i]
+                if (typeof tagRendering !== "object" || tagRendering["question"] === undefined) {
+                    continue
+                }
+                layer.tagRenderings[i] = convertor.convertStrict(
+                    <QuestionableTagRenderingConfigJson>tagRendering,
+                    "While automatically extracting questionHints of " + filepath
+                )
+            }
+        }
+        // The layer(s) are modified inPlace, so we can simply write to disk
+        fs.writeFileSync(filepath, JSON.stringify(contents, null, "  "), { encoding: "utf8" })
+    }
+}
+
+new FixQuestionHint().run()

scripts/fixSchemas.ts

@@ -42,7 +42,7 @@ function WalkScheme<T>(
         }
         const definitionName = ref.substr(prefix.length)
         if (isHandlingReference.indexOf(definitionName) >= 0) {
-            return
+            return []
         }
         const loadedScheme = fullScheme.definitions[definitionName]
         return WalkScheme(onEach, loadedScheme, fullScheme, path, [
@@ -137,7 +137,9 @@ function main() {
                 def["additionalProperties"] = false
             }
         }
-        writeFileSync(dir + "/" + name + ".schema.json", JSON.stringify(parsed, null, "  "), { encoding: "utf8" })
+        writeFileSync(dir + "/" + name + ".schema.json", JSON.stringify(parsed, null, "  "), {
+            encoding: "utf8",
+        })
     }
 
     extractMeta("LayoutConfigJson", "layoutconfigmeta")

scripts/generateCache.ts

@@ -21,9 +21,12 @@ import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSou
 import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource"
 import Constants from "../Models/Constants"
 import { GeoOperations } from "../Logic/GeoOperations"
-import SimpleMetaTaggers from "../Logic/SimpleMetaTagger"
+import SimpleMetaTaggers, { ReferencingWaysMetaTagger } from "../Logic/SimpleMetaTagger"
 import FilteringFeatureSource from "../Logic/FeatureSource/Sources/FilteringFeatureSource"
 import Loc from "../Models/Loc"
+import { Feature } from "geojson"
+import { BBox } from "../Logic/BBox"
+import { bboxClip } from "@turf/turf"
 
 ScriptUtils.fixUtils()
 
@@ -78,13 +81,13 @@ function geoJsonName(targetDir: string, x: number, y: number, z: number): string
     return targetDir + "_" + z + "_" + x + "_" + y + ".geojson"
 }
 
-/// Downloads the given feature and saves them to disk
+/// Downloads the given tilerange from overpass and saves them to disk
 async function downloadRaw(
     targetdir: string,
     r: TileRange,
     theme: LayoutConfig,
     relationTracker: RelationsTracker
-) /* : {failed: number, skipped :number} */ {
+): Promise<{ failed: number; skipped: number }> {
     let downloaded = 0
     let failed = 0
     let skipped = 0
@@ -232,7 +235,8 @@ function sliceToTiles(
     theme: LayoutConfig,
     relationsTracker: RelationsTracker,
     targetdir: string,
-    pointsOnlyLayers: string[]
+    pointsOnlyLayers: string[],
+    clip: boolean
 ) {
     const skippedLayers = new Set<string>()
 
@@ -310,6 +314,7 @@ function sliceToTiles(
             maxFeatureCount: undefined,
             registerTile: (tile) => {
                 const tileIndex = tile.tileIndex
+                const bbox = BBox.fromTileIndex(tileIndex).asGeoJson({})
                 console.log("Got tile:", tileIndex, tile.layer.layerDef.id)
                 if (tile.features.data.length === 0) {
                     return
@@ -343,9 +348,9 @@ function sliceToTiles(
                 }
                 let strictlyCalculated = 0
                 let featureCount = 0
-                for (const feature of filteredTile.features.data) {
+                let features: Feature[] = filteredTile.features.data.map((f) => f.feature)
+                for (const feature of features) {
                     // Some cleanup
-                    delete feature.feature["bbox"]
 
                     if (tile.layer.layerDef.calculatedTags !== undefined) {
                         // Evaluate all the calculated tags strictly
@@ -353,7 +358,7 @@ function sliceToTiles(
                             (ct) => ct[0]
                         )
                         featureCount++
-                        const props = feature.feature.properties
+                        const props = feature.properties
                         for (const calculatedTagKey of calculatedTagKeys) {
                             const strict = props[calculatedTagKey]
 
@@ -379,7 +384,16 @@ function sliceToTiles(
                             }
                         }
                     }
+                    delete feature["bbox"]
                 }
+
+                if (clip) {
+                    console.log("Clipping features")
+                    features = [].concat(
+                        ...features.map((f: Feature) => GeoOperations.clipWith(<any>f, bbox))
+                    )
+                }
+
                 // Lets save this tile!
                 const [z, x, y] = Tiles.tile_from_index(tileIndex)
                 // console.log("Writing tile ", z, x, y, layerId)
@@ -391,7 +405,7 @@ function sliceToTiles(
                     JSON.stringify(
                         {
                             type: "FeatureCollection",
-                            features: filteredTile.features.data.map((f) => f.feature),
+                            features,
                         },
                         null,
                         " "
@@ -474,10 +488,12 @@ function sliceToTiles(
 
 export async function main(args: string[]) {
     console.log("Cache builder started with args ", args.join(", "))
+    ReferencingWaysMetaTagger.enabled = false
     if (args.length < 6) {
         console.error(
-            "Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1 [--generate-point-overview layer-name,layer-name,...] [--force-zoom-level z] \n" +
-                "Note: a new directory named <theme> will be created in targetdirectory"
+            "Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1 [--generate-point-overview layer-name,layer-name,...] [--force-zoom-level z] [--clip]" +
+                "--force-zoom-level causes non-cached-layers to be donwnloaded\n" +
+                "--clip will erase parts of the feature falling outside of the bounding box"
         )
         return
     }
@@ -494,6 +510,7 @@ export async function main(args: string[]) {
     const lon0 = Number(args[4])
     const lat1 = Number(args[5])
     const lon1 = Number(args[6])
+    const clip = args.indexOf("--clip") >= 0
 
     if (isNaN(lat0)) {
         throw "The first number (a latitude) is not a valid number"
@@ -523,10 +540,7 @@ export async function main(args: string[]) {
 
     const theme = AllKnownLayouts.allKnownLayouts.get(themeName)
     if (theme === undefined) {
-        const keys = []
-        AllKnownLayouts.allKnownLayouts.forEach((_, key) => {
-            keys.push(key)
-        })
+        const keys = Array.from(AllKnownLayouts.allKnownLayouts.keys())
         console.error("The theme " + theme + " was not found; try one of ", keys)
         return
     }
@@ -570,7 +584,7 @@ export async function main(args: string[]) {
 
     const extraFeatures = await downloadExtraData(theme)
     const allFeaturesSource = loadAllTiles(targetdir, tileRange, theme, extraFeatures)
-    sliceToTiles(allFeaturesSource, theme, relationTracker, targetdir, generatePointLayersFor)
+    sliceToTiles(allFeaturesSource, theme, relationTracker, targetdir, generatePointLayersFor, clip)
 }
 
 let args = [...process.argv]

scripts/generateContributors.ts

@@ -1,10 +1,23 @@
 import { exec } from "child_process"
 import { writeFile, writeFileSync } from "fs"
 
-function asList(hist: Map<string, number>): {
-    contributors: { contributor: string; commits: number }[]
-} {
-    const ls = []
+interface Contributor {
+    /**
+     * The name of the contributor
+     */
+    contributor: string
+    /**
+     * The number of commits
+     */
+    commits: number
+}
+
+interface ContributorList {
+    contributors: Contributor[]
+}
+
+function asList(hist: Map<string, number>): ContributorList {
+    const ls: Contributor[] = []
     hist.forEach((commits, contributor) => {
         ls.push({ commits, contributor })
     })

scripts/generateDocs.ts

@@ -15,14 +15,20 @@ import List from "../UI/Base/List"
 import SharedTagRenderings from "../Customizations/SharedTagRenderings"
 import { writeFile } from "fs"
 import Translations from "../UI/i18n/Translations"
-import * as themeOverview from "../assets/generated/theme_overview.json"
+import themeOverview from "../assets/generated/theme_overview.json"
 import DefaultGUI from "../UI/DefaultGUI"
 import FeaturePipelineState from "../Logic/State/FeaturePipelineState"
 import LayoutConfig from "../Models/ThemeConfig/LayoutConfig"
-import * as bookcases from "../assets/generated/themes/bookcases.json"
+import bookcases from "../assets/generated/themes/bookcases.json"
 import { DefaultGuiState } from "../UI/DefaultGuiState"
-import * as fakedom from "fake-dom"
+import fakedom from "fake-dom"
 import Hotkeys from "../UI/Base/Hotkeys"
+import { QueryParameters } from "../Logic/Web/QueryParameters"
+import Link from "../UI/Base/Link"
+import Constants from "../Models/Constants"
+import LayerConfig from "../Models/ThemeConfig/LayerConfig"
+import DependencyCalculator from "../Models/ThemeConfig/DependencyCalculator"
+import { AllSharedLayers } from "../Customizations/AllSharedLayers"
 function WriteFile(
     filename,
     html: BaseUIElement,
@@ -63,7 +69,187 @@ function WriteFile(
 
     md.replace(/\n\n\n+/g, "\n\n")
 
-    writeFileSync(filename, md)
+    if (!md.endsWith("\n")) {
+        md += "\n"
+    }
+
+    const warnAutomated =
+        "[//]: # (WARNING: this file is automatically generated. Please find the sources at the bottom and edit those sources)"
+
+    writeFileSync(filename, warnAutomated + md)
+}
+
+function GenerateDocumentationForTheme(theme: LayoutConfig): BaseUIElement {
+    return new Combine([
+        new Title(
+            new Combine([
+                theme.title,
+                "(",
+                new Link(theme.id, "https://mapcomplete.osm.be/" + theme.id),
+                ")",
+            ]),
+            2
+        ),
+        theme.description,
+        "This theme contains the following layers:",
+        new List(
+            theme.layers
+                .filter((l) => !l.id.startsWith("note_import_"))
+                .map((l) => new Link(l.id, "../Layers/" + l.id + ".md"))
+        ),
+        "Available languages:",
+        new List(theme.language.filter((ln) => ln !== "_context")),
+    ]).SetClass("flex flex-col")
+}
+
+/**
+ * Generates the documentation for the layers overview page
+ * @constructor
+ */
+function GenLayerOverviewText(): BaseUIElement {
+    for (const id of Constants.priviliged_layers) {
+        if (!AllSharedLayers.sharedLayers.has(id)) {
+            throw "Priviliged layer definition not found: " + id
+        }
+    }
+
+    const allLayers: LayerConfig[] = Array.from(AllSharedLayers.sharedLayers.values()).filter(
+        (layer) => Constants.priviliged_layers.indexOf(layer.id) < 0
+    )
+
+    const builtinLayerIds: Set<string> = new Set<string>()
+    allLayers.forEach((l) => builtinLayerIds.add(l.id))
+
+    const themesPerLayer = new Map<string, string[]>()
+
+    for (const layout of Array.from(AllKnownLayouts.allKnownLayouts.values())) {
+        for (const layer of layout.layers) {
+            if (!builtinLayerIds.has(layer.id)) {
+                continue
+            }
+            if (!themesPerLayer.has(layer.id)) {
+                themesPerLayer.set(layer.id, [])
+            }
+            themesPerLayer.get(layer.id).push(layout.id)
+        }
+    }
+
+    // Determine the cross-dependencies
+    const layerIsNeededBy: Map<string, string[]> = new Map<string, string[]>()
+
+    for (const layer of allLayers) {
+        for (const dep of DependencyCalculator.getLayerDependencies(layer)) {
+            const dependency = dep.neededLayer
+            if (!layerIsNeededBy.has(dependency)) {
+                layerIsNeededBy.set(dependency, [])
+            }
+            layerIsNeededBy.get(dependency).push(layer.id)
+        }
+    }
+
+    return new Combine([
+        new Title("Special and other useful layers", 1),
+        "MapComplete has a few data layers available in the theme which have special properties through builtin-hooks. Furthermore, there are some normal layers (which are built from normal Theme-config files) but are so general that they get a mention here.",
+        new Title("Priviliged layers", 1),
+        new List(Constants.priviliged_layers.map((id) => "[" + id + "](#" + id + ")")),
+        ...Constants.priviliged_layers
+            .map((id) => AllSharedLayers.sharedLayers.get(id))
+            .map((l) =>
+                l.GenerateDocumentation(
+                    themesPerLayer.get(l.id),
+                    layerIsNeededBy,
+                    DependencyCalculator.getLayerDependencies(l),
+                    Constants.added_by_default.indexOf(l.id) >= 0,
+                    Constants.no_include.indexOf(l.id) < 0
+                )
+            ),
+        new Title("Normal layers", 1),
+        "The following layers are included in MapComplete:",
+        new List(
+            Array.from(AllSharedLayers.sharedLayers.keys()).map(
+                (id) => new Link(id, "./Layers/" + id + ".md")
+            )
+        ),
+    ])
+}
+
+/**
+ * Generates documentation for the layers.
+ * Inline layers are included (if the theme is public)
+ * @param callback
+ * @constructor
+ */
+function GenOverviewsForSingleLayer(
+    callback: (layer: LayerConfig, element: BaseUIElement, inlineSource: string) => void
+): void {
+    const allLayers: LayerConfig[] = Array.from(AllSharedLayers.sharedLayers.values()).filter(
+        (layer) => Constants.priviliged_layers.indexOf(layer.id) < 0
+    )
+    const builtinLayerIds: Set<string> = new Set<string>()
+    allLayers.forEach((l) => builtinLayerIds.add(l.id))
+    const inlineLayers = new Map<string, string>()
+
+    for (const layout of Array.from(AllKnownLayouts.allKnownLayouts.values())) {
+        if (layout.hideFromOverview) {
+            continue
+        }
+
+        for (const layer of layout.layers) {
+            if (Constants.priviliged_layers.indexOf(layer.id) >= 0) {
+                continue
+            }
+            if (builtinLayerIds.has(layer.id)) {
+                continue
+            }
+            if (layer.source.geojsonSource !== undefined) {
+                // Not an OSM-source
+                continue
+            }
+            allLayers.push(layer)
+            builtinLayerIds.add(layer.id)
+            inlineLayers.set(layer.id, layout.id)
+        }
+    }
+
+    const themesPerLayer = new Map<string, string[]>()
+
+    for (const layout of Array.from(AllKnownLayouts.allKnownLayouts.values())) {
+        if (layout.hideFromOverview) {
+            continue
+        }
+        for (const layer of layout.layers) {
+            if (!builtinLayerIds.has(layer.id)) {
+                // This is an inline layer
+                continue
+            }
+            if (!themesPerLayer.has(layer.id)) {
+                themesPerLayer.set(layer.id, [])
+            }
+            themesPerLayer.get(layer.id).push(layout.id)
+        }
+    }
+
+    // Determine the cross-dependencies
+    const layerIsNeededBy: Map<string, string[]> = new Map<string, string[]>()
+
+    for (const layer of allLayers) {
+        for (const dep of DependencyCalculator.getLayerDependencies(layer)) {
+            const dependency = dep.neededLayer
+            if (!layerIsNeededBy.has(dependency)) {
+                layerIsNeededBy.set(dependency, [])
+            }
+            layerIsNeededBy.get(dependency).push(layer.id)
+        }
+    }
+
+    allLayers.forEach((layer) => {
+        const element = layer.GenerateDocumentation(
+            themesPerLayer.get(layer.id),
+            layerIsNeededBy,
+            DependencyCalculator.getLayerDependencies(layer)
+        )
+        callback(layer, element, inlineLayers.get(layer.id))
+    })
 }
 
 /**
@@ -103,7 +289,7 @@ function generateWikipage() {
         "! Name, link !! Genre !! Covered region !! Language !! Description !! Free materials !! Image\n" +
         "|-"
 
-    for (const layout of themeOverview["default"] ?? themeOverview) {
+    for (const layout of themeOverview) {
         if (layout.hideFromOverview) {
             continue
         }
@@ -123,7 +309,7 @@ console.log("Starting documentation generation...")
 ScriptUtils.fixUtils()
 generateWikipage()
 
-AllKnownLayouts.GenOverviewsForSingleLayer((layer, element, inlineSource) => {
+GenOverviewsForSingleLayer((layer, element, inlineSource) => {
     console.log("Exporting ", layer.id)
     if (!existsSync("./Docs/Layers")) {
         mkdirSync("./Docs/Layers")
@@ -136,7 +322,7 @@ AllKnownLayouts.GenOverviewsForSingleLayer((layer, element, inlineSource) => {
 })
 
 Array.from(AllKnownLayouts.allKnownLayouts.values()).map((theme) => {
-    const docs = AllKnownLayouts.GenerateDocumentationForTheme(theme)
+    const docs = GenerateDocumentationForTheme(theme)
     WriteFile(
         "./Docs/Themes/" + theme.id + ".md",
         docs,
@@ -159,9 +345,7 @@ WriteFile(
 WriteFile("./Docs/SpecialInputElements.md", ValidatedTextField.HelpText(), [
     "UI/Input/ValidatedTextField.ts",
 ])
-WriteFile("./Docs/BuiltinLayers.md", AllKnownLayouts.GenLayerOverviewText(), [
-    "Customizations/AllKnownLayouts.ts",
-])
+WriteFile("./Docs/BuiltinLayers.md", GenLayerOverviewText(), ["Customizations/AllKnownLayouts.ts"])
 WriteFile("./Docs/BuiltinQuestions.md", SharedTagRenderings.HelpText(), [
     "Customizations/SharedTagRenderings.ts",
     "assets/tagRenderings/questions.json",
@@ -225,6 +409,12 @@ WriteFile("./Docs/URL_Parameters.md", QueryParameterDocumentation.GenerateQueryP
 if (fakedom === undefined || window === undefined) {
     throw "FakeDom not initialized"
 }
+QueryParameters.GetQueryParameter(
+    "mode",
+    "map",
+    "The mode the application starts in, e.g. 'map', 'dashboard' or 'statistics'"
+)
+
 new DefaultGUI(
     new FeaturePipelineState(new LayoutConfig(<any>bookcases)),
     new DefaultGuiState()

scripts/generateLayerOverview.ts

@@ -1,9 +1,10 @@
 import ScriptUtils from "./ScriptUtils"
 import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "fs"
-import * as licenses from "../assets/generated/license_info.json"
+import licenses from "../assets/generated/license_info.json"
 import { LayoutConfigJson } from "../Models/ThemeConfig/Json/LayoutConfigJson"
 import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
 import Constants from "../Models/Constants"
+import * as fakedom from "fake-dom"
 import {
     DetectDuplicateFilters,
     DoesImageExist,
@@ -14,16 +15,14 @@ import {
 } from "../Models/ThemeConfig/Conversion/Validation"
 import { Translation } from "../UI/i18n/Translation"
 import { TagRenderingConfigJson } from "../Models/ThemeConfig/Json/TagRenderingConfigJson"
-import * as questions from "../assets/tagRenderings/questions.json"
-import * as icons from "../assets/tagRenderings/icons.json"
+import questions from "../assets/tagRenderings/questions.json"
 import PointRenderingConfigJson from "../Models/ThemeConfig/Json/PointRenderingConfigJson"
 import { PrepareLayer } from "../Models/ThemeConfig/Conversion/PrepareLayer"
 import { PrepareTheme } from "../Models/ThemeConfig/Conversion/PrepareTheme"
 import { DesugaringContext } from "../Models/ThemeConfig/Conversion/Conversion"
 import { Utils } from "../Utils"
-import { AllKnownLayouts } from "../Customizations/AllKnownLayouts"
 import Script from "./Script"
-import { GenerateLicenseInfo } from "./generateLicenseInfo"
+import { AllSharedLayers } from "../Customizations/AllSharedLayers"
 
 // This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files.
 // It spits out an overview of those to be used to load them
@@ -159,7 +158,7 @@ class LayerOverviewUtils extends Script {
         const dict = new Map<string, TagRenderingConfigJson>()
 
         const validator = new ValidateTagRenderings(undefined, doesImageExist)
-        for (const key in questions["default"]) {
+        for (const key in questions) {
             if (key === "id") {
                 continue
             }
@@ -172,21 +171,6 @@ class LayerOverviewUtils extends Script {
             )
             dict.set(key, config)
         }
-        for (const key in icons["default"]) {
-            if (key === "id") {
-                continue
-            }
-            if (typeof icons[key] !== "object") {
-                continue
-            }
-            icons[key].id = key
-            const config = <TagRenderingConfigJson>icons[key]
-            validator.convertStrict(
-                config,
-                "generate-layer-overview:tagRenderings/icons.json:" + key
-            )
-            dict.set(key, config)
-        }
 
         dict.forEach((value, key) => {
             if (key === "id") {
@@ -246,6 +230,9 @@ class LayerOverviewUtils extends Script {
     }
 
     async main(args: string[]) {
+        if (fakedom === undefined) {
+            throw "Fakedom not initialized"
+        }
         const forceReload = args.some((a) => a == "--force")
 
         const licensePaths = new Set<string>()
@@ -256,16 +243,15 @@ class LayerOverviewUtils extends Script {
         const sharedLayers = this.buildLayerIndex(doesImageExist, forceReload)
         const recompiledThemes: string[] = []
         const sharedThemes = this.buildThemeIndex(
-            doesImageExist,
+            licensePaths,
             sharedLayers,
             recompiledThemes,
             forceReload
         )
 
         writeFileSync(
-            "./assets/generated/known_layers_and_themes.json",
+            "./assets/generated/known_themes.json",
             JSON.stringify({
-                layers: Array.from(sharedLayers.values()),
                 themes: Array.from(sharedThemes.values()),
             })
         )
@@ -310,7 +296,7 @@ class LayerOverviewUtils extends Script {
             "GenerateLayerOverview:"
         )
 
-        if (AllKnownLayouts.getSharedLayersConfigs().size == 0) {
+        if (AllSharedLayers.getSharedLayersConfigs().size == 0) {
             console.error("This was a bootstrapping-run. Run generate layeroverview again!")
         } else {
             const green = (s) => "\x1b[92m" + s + "\x1b[0m"
@@ -329,7 +315,7 @@ class LayerOverviewUtils extends Script {
         const sharedTagRenderings = this.getSharedTagRenderings(doesImageExist)
         const state: DesugaringContext = {
             tagRenderings: sharedTagRenderings,
-            sharedLayers: AllKnownLayouts.getSharedLayersConfigs(),
+            sharedLayers: AllSharedLayers.getSharedLayersConfigs(),
         }
         const sharedLayers = new Map<string, LayerConfigJson>()
         const prepLayer = new PrepareLayer(state)
@@ -386,7 +372,7 @@ class LayerOverviewUtils extends Script {
     }
 
     private buildThemeIndex(
-        doesImageExist: DoesImageExist,
+        licensePaths: Set<string>,
         sharedLayers: Map<string, LayerConfigJson>,
         recompiledThemes: string[],
         forceReload: boolean
@@ -401,9 +387,26 @@ class LayerOverviewUtils extends Script {
 
         const convertState: DesugaringContext = {
             sharedLayers,
-            tagRenderings: this.getSharedTagRenderings(doesImageExist),
+            tagRenderings: this.getSharedTagRenderings(
+                new DoesImageExist(licensePaths, existsSync)
+            ),
             publicLayers,
         }
+        const knownTagRenderings = new Set<string>()
+        convertState.tagRenderings.forEach((_, key) => knownTagRenderings.add(key))
+        sharedLayers.forEach((layer) => {
+            for (const tagRendering of layer.tagRenderings ?? []) {
+                if (tagRendering["id"]) {
+                    knownTagRenderings.add(layer.id + "." + tagRendering["id"])
+                }
+                if (tagRendering["labels"]) {
+                    for (const label of tagRendering["labels"]) {
+                        knownTagRenderings.add(layer.id + "." + label)
+                    }
+                }
+            }
+        })
+
         const skippedThemes: string[] = []
         for (const themeInfo of themeFiles) {
             const themePath = themeInfo.path
@@ -438,10 +441,10 @@ class LayerOverviewUtils extends Script {
                 themeFile = new PrepareTheme(convertState).convertStrict(themeFile, themePath)
 
                 new ValidateThemeAndLayers(
-                    doesImageExist,
+                    new DoesImageExist(licensePaths, existsSync, knownTagRenderings),
                     themePath,
                     true,
-                    convertState.tagRenderings
+                    knownTagRenderings
                 ).convertStrict(themeFile, themePath)
 
                 if (themeFile.icon.endsWith(".svg")) {

scripts/generateLayouts.ts

@@ -2,7 +2,7 @@ import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFile, writeFi
 import Locale from "../UI/i18n/Locale"
 import Translations from "../UI/i18n/Translations"
 import { Translation } from "../UI/i18n/Translation"
-import * as all_known_layouts from "../assets/generated/known_layers_and_themes.json"
+import all_known_layouts from "../assets/generated/known_themes.json"
 import { LayoutConfigJson } from "../Models/ThemeConfig/Json/LayoutConfigJson"
 import LayoutConfig from "../Models/ThemeConfig/LayoutConfig"
 import xml2js from "xml2js"
@@ -24,7 +24,7 @@ async function createIcon(iconPath: string, size: number, alreadyWritten: string
     }
 
     const newname = `assets/generated/images/${name.replace(/\//g, "_")}${size}.png`
-
+    const targetpath = `public/${newname}`
     if (alreadyWritten.indexOf(newname) >= 0) {
         return newname
     }
@@ -41,7 +41,7 @@ async function createIcon(iconPath: string, size: number, alreadyWritten: string
         // We already read to file, in order to crash here if the file is not found
         let img = await sharp(iconPath)
         let resized = await img.resize(size)
-        await resized.toFile(newname)
+        await resized.toFile(targetpath)
         console.log("Created png version at ", newname)
     } catch (e) {
         console.error("Could not read icon", iconPath, " to create a PNG due to", e)
@@ -60,7 +60,7 @@ async function createSocialImage(layout: LayoutConfig, template: "" | "Wide"): P
         )
         return undefined
     }
-    const path = `./assets/generated/images/social_image_${layout.id}_${template}.svg`
+    const path = `./public/assets/generated/images/social_image_${layout.id}_${template}.svg`
     if (existsSync(path)) {
         return path
     }
@@ -121,7 +121,7 @@ async function createManifest(
         // This is an svg. Lets create the needed pngs and do some checkes!
 
         const whiteBackgroundPath =
-            "./assets/generated/images/theme_" + layout.id + "_white_background.svg"
+            "./public/assets/generated/images/theme_" + layout.id + "_white_background.svg"
         {
             const svg = await ScriptUtils.ReadSvg(icon)
             const width: string = svg.$.width
@@ -136,7 +136,7 @@ async function createManifest(
         let path = layout.icon
         if (layout.icon.startsWith("<")) {
             // THis is already the svg
-            path = "./assets/generated/images/" + layout.id + "_logo.svg"
+            path = "./public/assets/generated/images/" + layout.id + "_logo.svg"
             writeFileSync(path, layout.icon)
         }
 
@@ -235,7 +235,7 @@ async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alr
     let icon = layout.icon
     if (icon.startsWith("<?xml") || icon.startsWith("<svg")) {
         // This already is an svg
-        icon = `./assets/generated/images/${layout.id}_icon.svg`
+        icon = `./public/assets/generated/images/${layout.id}_icon.svg`
         writeFileSync(icon, layout.icon)
     }
 
@@ -273,7 +273,7 @@ async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alr
             '<script type="module" src="./index.ts"></script>',
             `<script type="module"  src='./index_${layout.id}.ts'></script>`
         )
-    0
+
     try {
         output = output
             .replace(
@@ -295,7 +295,7 @@ async function createIndexFor(theme: LayoutConfig) {
     const filename = "index_" + theme.id + ".ts"
     writeFileSync(
         filename,
-        `import * as themeConfig from "./assets/generated/themes/${theme.id}.json"\n`
+        `import themeConfig from "./assets/generated/themes/${theme.id}.json"\n`
     )
     appendFileSync(filename, codeTemplate)
 }
@@ -311,7 +311,9 @@ async function main(): Promise<void> {
     createDir("./assets/generated")
     createDir("./assets/generated/layers")
     createDir("./assets/generated/themes")
-    createDir("./assets/generated/images")
+    createDir("./public/assets/")
+    createDir("./public/assets/generated")
+    createDir("./public/assets/generated/images")
 
     const blacklist = [
         "",
@@ -353,7 +355,7 @@ async function main(): Promise<void> {
         const { manifest, whiteIcons } = await createManifest(layout, alreadyWritten)
         const manif = JSON.stringify(manifest, undefined, 2)
         const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest"
-        writeFile(manifestLocation, manif, err)
+        writeFile("public/" + manifestLocation, manif, err)
 
         // Create a landing page for the given theme
         const landing = await createLandingPage(layout, manifest, whiteIcons, alreadyWritten)
@@ -377,7 +379,7 @@ async function main(): Promise<void> {
     )
 
     const manif = JSON.stringify(manifest, undefined, 2)
-    writeFileSync("index.manifest", manif)
+    writeFileSync("public/index.webmanifest", manif)
 }
 
 main().then(() => {

scripts/generateStats.ts

@@ -1,4 +1,4 @@
-import * as known_layers from "../assets/generated/known_layers.json"
+import known_layers from "../assets/generated/known_layers.json"
 import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
 import { TagUtils } from "../Logic/Tags/TagUtils"
 import { Utils } from "../Utils"
@@ -10,7 +10,7 @@ import Constants from "../Models/Constants"
 
 async function main(includeTags = true) {
     ScriptUtils.fixUtils()
-    const layers: LayerConfigJson[] = (known_layers["default"] ?? known_layers).layers
+    const layers = <LayerConfigJson[]>known_layers.layers
 
     const keysAndTags = new Map<string, Set<string>>()
 

scripts/generateTaginfoProjectFiles.ts

@@ -205,7 +205,7 @@ function main() {
 
     const files = []
 
-    for (const layout of AllKnownLayouts.layoutsList) {
+    for (const layout of AllKnownLayouts.allKnownLayouts.values()) {
         if (layout.hideFromOverview) {
             continue
         }

scripts/generateTranslations.ts

@@ -710,10 +710,7 @@ const l3 = generateTranslationsObjectFrom(
 
 const usedLanguages: string[] = Utils.Dedup(l1.concat(l2).concat(l3)).filter((v) => v !== "*")
 usedLanguages.sort()
-fs.writeFileSync(
-    "./assets/generated/used_languages.json",
-    JSON.stringify({ languages: usedLanguages })
-)
+fs.writeFileSync("./assets/used_languages.json", JSON.stringify({ languages: usedLanguages }))
 
 if (!themeOverwritesWeblate) {
     // Generates the core translations

scripts/removeTranslationString.ts

@@ -13,7 +13,7 @@ async function main(args: string[]) {
     console.log("Removing translation string ", path, "from the general translations")
     const files = ScriptUtils.readDirRecSync("./langs", 1).filter((f) => f.endsWith(".json"))
     for (const file of files) {
-        const json = JSON.parse(fs.readFileSync(file, "UTF-8"))
+        const json = JSON.parse(fs.readFileSync(file, { encoding: "utf-8" }))
         Utils.WalkPath(path, json, (_) => undefined)
         fs.writeFileSync(file, JSON.stringify(json, null, "    ") + "\n")
     }

scripts/schools/README.md

@@ -1,3 +0,0 @@
-# Little scripts to parse Belgian school data 
-
-

scripts/schools/amendSchoolData.ts

@@ -1,364 +0,0 @@
-import { parse } from "csv-parse/sync"
-import { readFileSync, writeFileSync } from "fs"
-import { Utils } from "../../Utils"
-import { GeoJSONObject, geometry } from "@turf/turf"
-
-function parseAndClean(filename: string): Record<any, string>[] {
-    const csvOptions = {
-        columns: true,
-        skip_empty_lines: true,
-        trim: true,
-    }
-    const records: Record<any, string>[] = parse(readFileSync(filename), csvOptions)
-    return records.map((r) => {
-        for (const key of Object.keys(r)) {
-            if (r[key].endsWith("niet van toepassing")) {
-                delete r[key]
-            }
-        }
-
-        return r
-    })
-}
-
-const structuren = {
-    "Voltijds Gewoon Secundair Onderwijs": "secondary",
-    "Gewoon Lager Onderwijs": "primary",
-    "Gewoon Kleuteronderwijs": "kindergarten",
-    Kleuteronderwijs: "kindergarten",
-    "Buitengewoon Lager Onderwijs": "primary",
-    "Buitengewoon Secundair Onderwijs": "secondary",
-    "Buitengewoon Kleuteronderwijs": "kindergarten",
-    "Deeltijds Beroepssecundair Onderwijs": "secondary",
-}
-
-const degreesMapping = {
-    "Derde graad": "upper_secondary",
-    "Tweede graad": "middle_secondary",
-    "Eerste graad": "lower_secondary",
-}
-const classificationOrder = [
-    "kindergarten",
-    "primary",
-    "secondary",
-    "lower_secondary",
-    "middle_secondary",
-    "upper_secondary",
-]
-
-const stelselsMapping = {
-    "Beide stelsels": "linear_courses;modular_courses",
-    "Lineair stelsel": "linear_courses",
-    "Modulair stelsel": "modular_courses",
-}
-
-const rmKeys = [
-    "schoolnummer",
-    "instellingstype",
-    "adres",
-    "begindatum",
-    "hoofdzetel",
-    "huisnummer",
-    "kbo-nummer",
-    "beheerder(s)",
-    "bestuur",
-    "clb",
-    "ingerichte hoofdstructuren",
-    "busnummer",
-    "crab-code",
-    "crab-huisnr",
-    "einddatum",
-    "fax",
-    "gemeente",
-    "intern_vplnummer",
-    "kbo_nummer",
-    "lx",
-    "ly",
-    "niscode",
-    "onderwijsniveau",
-    "onderwijsvorm",
-    "scholengemeenschap",
-    "postcode",
-    "provincie",
-    "provinciecode",
-    "soort instelling",
-    "status erkenning",
-    "straat",
-    "VWO-vestigingsplaatscode",
-    "taalstelsel",
-    "net",
-]
-
-const rename = {
-    "e-mail": "email",
-    naam: "name",
-    telefoon: "phone",
-}
-
-function fuzzIdenticals(features: { geometry: { coordinates: [number, number] } }[]) {
-    var seen = new Set<string>()
-    for (const feature of features) {
-        var coors = feature.geometry.coordinates
-        let k = coors[0] + "," + coors[1]
-        while (seen.has(k)) {
-            coors[0] += 0.00025
-            k = coors[0] + "," + coors[1]
-        }
-        seen.add(k)
-    }
-}
-
-/**
- * Sorts classifications in order
- *
- * sortClassifications(["primary","secondary","kindergarten"] // => ["kindergarten", "primary", "secondary"]
- */
-function sortClassifications(classification: string[]) {
-    return classification.sort(
-        (a, b) => classificationOrder.indexOf(a) - classificationOrder.indexOf(b)
-    )
-}
-
-function main() {
-    console.log("Parsing schools...")
-    const aantallen = "/home/pietervdvn/Downloads/Scholen/aantallen.csv"
-    const perSchool = "/home/pietervdvn/Downloads/Scholen/perschool.csv"
-
-    const schoolfields = [
-        "schoolnummer",
-        "intern_vplnummer",
-        "net",
-        "naam",
-        "hoofdzetel",
-        "adres",
-        "straat",
-        "huisnummer",
-        "busnummer",
-        "postcode",
-        "gemeente",
-        "niscode",
-        "provinciecode",
-        "provincie",
-        "VWO-vestigingsplaatscode",
-        "crab-code",
-        "crab-huisnr",
-        "lx",
-        "ly",
-        "kbo-nummer",
-        "telefoon",
-        "fax",
-        "e-mail",
-        "website",
-        "beheerder(s)",
-        "soort instelling",
-        "onderwijsniveau",
-        "instellingstype",
-        "begindatum",
-        "einddatum",
-        "status erkenning",
-        "clb",
-        "bestuur",
-        "scholengemeenschap",
-        "taalstelsel",
-        "ingerichte hoofdstructuren",
-    ] as const
-
-    const schoolGeojson: {
-        features: {
-            properties: Record<typeof schoolfields[number], string>
-            geometry: {
-                type: "Point"
-                coordinates: [number, number]
-            }
-        }[]
-    } = JSON.parse(readFileSync("scripts/schools/scholen.geojson", "utf8"))
-
-    fuzzIdenticals(schoolGeojson.features)
-
-    const aantallenFields = [
-        "schooljaar",
-        "nr koepel",
-        "koepel",
-        "instellingscode",
-        "intern volgnr vpl",
-        "volgnr vpl",
-        "naam instelling",
-        "GON-school",
-        "GOK-school",
-        "instellingsnummer scholengemeenschap",
-        "scholengemeenschap",
-        "code schoolbestuur",
-        "schoolbestuur",
-        "type vestigingsplaats",
-        "fusiegemeente hoofdvestigingsplaats",
-        "straatnaam vestigingsplaats",
-        "huisnr vestigingsplaats",
-        "bus vestigingsplaats",
-        "postcode vestigingsplaats",
-        "deelgemeente vestigingsplaats",
-        "fusiegemeente vestigingsplaats",
-        "hoofdstructuur (code)",
-        "hoofdstructuur",
-        "administratieve groep (code)",
-        "administratieve groep",
-        "graad lager onderwijs",
-        "pedagogische methode",
-        "graad secundair onderwijs",
-        "leerjaar",
-        "A of B-stroom",
-        "basisopties",
-        "beroepenveld",
-        "onderwijsvorm",
-        "studiegebied",
-        "studierichting",
-        "stelsel",
-        "okan cluster",
-        "type buitengewoon onderwijs",
-        "opleidingsvorm (code)",
-        "opleidingsvorm",
-        "fase",
-        "opleidingen",
-        "geslacht",
-        "aantal inschrijvingen",
-    ] as const
-    const aantallenParsed: Record<typeof aantallenFields[number], string>[] =
-        parseAndClean(aantallen)
-    const perschoolFields = [
-        "schooljaar",
-        "nr koepel",
-        "koepel",
-        "instellingscode",
-        "naam instelling",
-        "straatnaam",
-        "huisnr",
-        "bus",
-        "postcode",
-        "deelgemeente",
-        "fusiegemeente",
-        "aantal inschrijvingen",
-    ] as const
-    const perschoolParsed: Record<typeof perschoolFields[number], string>[] =
-        parseAndClean(perSchool)
-
-    schoolGeojson.features = schoolGeojson.features
-        .filter((sch) => sch.properties.lx != "0" && sch.properties.ly != "0")
-        .filter((sch) => sch.properties.instellingstype !== "Universiteit")
-
-    const c = schoolGeojson.features.length
-    console.log("Got ", schoolGeojson.features.length, "items after filtering")
-    let i = 0
-    let lastWrite = 0
-    for (const feature of schoolGeojson.features) {
-        i++
-        const now = Date.now()
-        if (now - lastWrite > 1000) {
-            lastWrite = now
-            console.log("Processing " + i + "/" + c)
-        }
-        const props = feature.properties
-
-        const aantallen = aantallenParsed.filter((i) => i.instellingscode == props.schoolnummer)
-
-        if (aantallen.length > 0) {
-            const fetch = (key: typeof aantallenFields[number]) =>
-                Utils.NoNull(Utils.Dedup(aantallen.map((x) => x[key])))
-
-            props["onderwijsvorm"] = fetch("onderwijsvorm").join(";")
-
-            /*
-            const gonSchool = aantallen.some(x => x["GON-school"] === "GON-school")
-            const gokSchool = aantallen.some(x => x["GOK-school"] === "GON-school")
-            const onderwijsvorm = fetch("onderwijsvorm")
-            const koepel = fetch("koepel")
-            const stelsel = fetch("stelsel").join(";")
-            const scholengemeenschap = fetch("scholengemeenschap")
-            
-            */
-            const hoofdstructuur = fetch("hoofdstructuur")
-
-            let specialEducation = false
-            let classification = hoofdstructuur.map((s) => {
-                const v = structuren[s]
-                if (s.startsWith("Buitengewoon")) {
-                    specialEducation = true
-                }
-                if (v === undefined) {
-                    console.error("Type not found: " + s)
-                    return ""
-                }
-                return v
-            })
-            const graden = fetch("graad secundair onderwijs")
-            if (classification[0] === "secondary") {
-                if (graden.length !== 3) {
-                    classification = graden.map((degree) => degreesMapping[degree])
-                }
-            }
-            sortClassifications(classification)
-            props["school"] = Utils.Dedup(classification).join("; ")
-
-            // props["koepel"] = koepel.join(";")
-            // props["scholengemeenschap"] = scholengemeenschap.join(";")
-            // props["stelsel"] = stelselsMapping[stelsel]
-
-            if (specialEducation) {
-                props["school:for"] = "special_education"
-            }
-            if (props.taalstelsel === "Nederlandstalig") {
-                props["language:nl"] = "yes"
-            }
-
-            if (props.instellingstype === "Instelling voor deeltijds kunstonderwijs") {
-                props["amenity"] = "college"
-                props["school:subject"] = "art"
-            }
-        }
-
-        const schoolinfo = perschoolParsed.filter((i) => i.instellingscode == props.schoolnummer)
-        if (schoolinfo.length == 0) {
-            // pass
-        } else if (schoolinfo.length == 1) {
-            props["capacity"] = schoolinfo[0]["aantal inschrijvingen"]
-                .split(";")
-                .map((i) => Number(i))
-                .reduce((sum, i) => sum + i, 0)
-        } else {
-            throw "Multiple schoolinfo's found for " + props.schoolnummer
-        }
-
-        //props["source:ref"] = props.schoolnummer
-        props["amenity"] = "school"
-        if (props["school"] === "kindergarten") {
-            props["amenity"] = "kindergarten"
-            props["isced:2011:level"] = "early_education"
-            delete props["school"]
-        }
-
-        for (const renameKey in rename) {
-            const into = rename[renameKey]
-            if (props[renameKey] !== undefined) {
-                props[into] = props[renameKey]
-                delete props[renameKey]
-            }
-        }
-
-        for (const rmKey of rmKeys) {
-            delete props[rmKey]
-        }
-    }
-
-    //schoolGeojson.features = schoolGeojson.features.filter(f => f.properties["capacity"] !== undefined)
-    /*schoolGeojson.features.forEach((f, i) => {
-        f.properties["id"] = "school/"+i
-    })*/
-    schoolGeojson.features = schoolGeojson.features.filter(
-        (f) => f.properties["amenity"] === "kindergarten"
-    )
-
-    writeFileSync("scripts/schools/amended_schools.geojson", JSON.stringify(schoolGeojson), "utf8")
-    console.log("Done")
-}
-
-if (!process.argv[1].endsWith("mocha")) {
-    main()
-}

scripts/slice.ts

@@ -134,20 +134,40 @@ class Slice extends Script {
             }
             delete f.bbox
         }
+        const maxNumberOfTiles = Math.pow(2, zoomlevel) * Math.pow(2, zoomlevel)
+        let handled = 0
         TiledFeatureSource.createHierarchy(StaticFeatureSource.fromGeojson(allFeatures), {
             minZoomLevel: zoomlevel,
             maxZoomLevel: zoomlevel,
             maxFeatureCount: Number.MAX_VALUE,
             registerTile: (tile) => {
+                handled = handled + 1
                 const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson`
                 const box = BBox.fromTile(tile.z, tile.x, tile.y)
                 let features = tile.features.data.map((ff) => ff.feature)
                 if (doSlice) {
                     features = Utils.NoNull(
                         features.map((f) => {
+                            const bbox = box.asGeoJson({})
+                            const properties = {
+                                ...f.properties,
+                                id:
+                                    (f.properties?.id ?? "") +
+                                    "_" +
+                                    tile.z +
+                                    "_" +
+                                    tile.x +
+                                    "_" +
+                                    tile.y,
+                            }
+
+                            if (GeoOperations.completelyWithin(bbox, f)) {
+                                bbox.properties = properties
+                                return bbox
+                            }
                             const intersection = GeoOperations.intersect(f, box.asGeoJson({}))
                             if (intersection) {
-                                intersection.properties = f.properties
+                                intersection.properties = properties
                             }
                             return intersection
                         })
@@ -156,6 +176,15 @@ class Slice extends Script {
                 features.forEach((f) => {
                     delete f.bbox
                 })
+                if (features.length === 0) {
+                    ScriptUtils.erasableLog(
+                        handled + "/" + maxNumberOfTiles,
+                        "Not writing ",
+                        path,
+                        ": no features"
+                    )
+                    return
+                }
                 fs.writeFileSync(
                     path,
                     JSON.stringify(
@@ -168,6 +197,7 @@ class Slice extends Script {
                     )
                 )
                 ScriptUtils.erasableLog(
+                    handled + "/" + maxNumberOfTiles,
                     "Written ",
                     path,
                     "which has ",

scripts/thieves/readIdPresets.ts

@@ -3,7 +3,7 @@
  */
 import ScriptUtils from "../ScriptUtils"
 import { existsSync, readFileSync, writeFileSync } from "fs"
-import * as known_languages from "../../assets/language_native.json"
+import known_languages from "../../assets/language_native.json"
 import { LayerConfigJson } from "../../Models/ThemeConfig/Json/LayerConfigJson"
 import { MappingConfigJson } from "../../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
 import SmallLicense from "../../Models/smallLicense"