Generate layer overview now only recompiles files that need to be recompiled

package.json

@@ -28,7 +28,7 @@
     "generate:cache:speelplekken": "npm run generate:layeroverview && ts-node scripts/generateCache.ts speelplekken 14 ../MapComplete-data/speelplekken_cache/ 51.20 4.35 51.09 4.56",
     "generate:cache:natuurpunt": "npm run generate:layeroverview && ts-node scripts/generateCache.ts natuurpunt 12 ../MapComplete-data/natuurpunt_cache/ 50.40 2.1 51.54 6.4 --generate-point-overview nature_reserve,visitor_information_centre",
     "generate:cache:natuurpunt:mini": "ts-node scripts/generateCache.ts natuurpunt 12 ../../git/MapComplete-data/natuurpunt_cache_mini/ 51.00792239979105 4.497699737548828 51.0353492224462554 4.539070129394531 --generate-point-overview nature_reserve,visitor_information_centre",
-    "generate:layeroverview": "ts-node scripts/generateLayerOverview.ts --no-fail",
+    "generate:layeroverview": "ts-node scripts/generateLayerOverview.ts",
     "generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail",
     "query:licenses": "ts-node scripts/generateLicenseInfo.ts --query",
     "generate:report": "cd Docs/Tools && ./compileStats.sh && git commit . -m 'New statistics ands graphs' && git push",

scripts/ScriptUtils.ts

@@ -46,12 +46,102 @@ export default class ScriptUtils {
         })
     }
 
-    private static async DownloadJSON(url: string, headers?: any): Promise<any>{
+    public static erasableLog(...text) {
+        process.stdout.write("\r " + text.join(" ") + "                \r")
+    }
+
+    public static sleep(ms) {
+        if (ms <= 0) {
+            process.stdout.write("\r                                       \r")
+            return;
+        }
+        return new Promise((resolve) => {
+            process.stdout.write("\r Sleeping for " + (ms / 1000) + "s \r")
+            setTimeout(resolve, 1000);
+        }).then(() => ScriptUtils.sleep(ms - 1000));
+    }
+
+    public static getLayerPaths(): string[] {
+        return ScriptUtils.readDirRecSync("./assets/layers")
+            .filter(path => path.indexOf(".json") > 0)
+            .filter(path => path.indexOf(".proto.json") < 0)
+            .filter(path => path.indexOf("license_info.json") < 0)
+    }
+
+    public static getLayerFiles(): { parsed: LayerConfigJson, path: string }[] {
+        return ScriptUtils.readDirRecSync("./assets/layers")
+            .filter(path => path.indexOf(".json") > 0)
+            .filter(path => path.indexOf(".proto.json") < 0)
+            .filter(path => path.indexOf("license_info.json") < 0)
+            .map(path => {
+                try {
+                    const contents = readFileSync(path, "UTF8")
+                    if (contents === "") {
+                        throw "The file " + path + " is empty, did you properly save?"
+                    }
+
+                    const parsed = JSON.parse(contents);
+                    return {parsed, path}
+                } catch (e) {
+                    console.error("Could not parse file ", "./assets/layers/" + path, "due to ", e)
+                    throw e
+                }
+            })
+    }
+
+    public static getThemePaths(): string[] {
+        return ScriptUtils.readDirRecSync("./assets/themes")
+            .filter(path => path.endsWith(".json") && !path.endsWith(".proto.json"))
+            .filter(path => path.indexOf("license_info.json") < 0)
+    }
+
+    public static getThemeFiles(): { parsed: LayoutConfigJson, path: string }[] {
+        return this.getThemePaths()
+            .map(path => {
+                try {
+                    const contents = readFileSync(path, "UTF8");
+                    if (contents === "") {
+                        throw "The file " + path + " is empty, did you properly save?"
+                    }
+                    const parsed = JSON.parse(contents);
+                    return {parsed: parsed, path: path}
+                } catch (e) {
+                    console.error("Could not read file ", path, "due to ", e)
+                    throw e
+                }
+            });
+    }
+
+    public static TagInfoHistogram(key: string): Promise<{
+        data: { count: number, value: string, fraction: number }[]
+    }> {
+        const url = `https://taginfo.openstreetmap.org/api/4/key/values?key=${key}&filter=all&lang=en&sortname=count&sortorder=desc&page=1&rp=17&qtype=value`
+        return ScriptUtils.DownloadJSON(url)
+    }
+
+    public static async ReadSvg(path: string): Promise<any> {
+        if (!existsSync(path)) {
+            throw "File not found: " + path
+        }
+        const root = await xml2js.parseStringPromise(readFileSync(path, "UTF8"))
+        return root.svg
+    }
+
+    public static async ReadSvgSync(path: string, callback: ((svg: any) => void)): Promise<any> {
+        xml2js.parseString(readFileSync(path, "UTF8"), {async: false}, (err, root) => {
+            if (err) {
+                throw err
+            }
+            callback(root["svg"]);
+        })
+    }
+
+    private static async DownloadJSON(url: string, headers?: any): Promise<any> {
         const data = await ScriptUtils.Download(url, headers);
         return JSON.parse(data.content)
     }
 
-    private static Download(url, headers?: any): Promise<{content: string}> {
+    private static Download(url, headers?: any): Promise<{ content: string }> {
         return new Promise((resolve, reject) => {
             try {
                 headers = headers ?? {}
@@ -83,84 +173,4 @@ export default class ScriptUtils {
 
     }
 
-    public static erasableLog(...text) {
-        process.stdout.write("\r " + text.join(" ") + "                \r")
-    }
-
-    public static sleep(ms) {
-        if (ms <= 0) {
-            process.stdout.write("\r                                       \r")
-            return;
-        }
-        return new Promise((resolve) => {
-            process.stdout.write("\r Sleeping for " + (ms / 1000) + "s \r")
-            setTimeout(resolve, 1000);
-        }).then(() => ScriptUtils.sleep(ms - 1000));
-    }
-
-    public static getLayerFiles(): { parsed: LayerConfigJson, path: string }[] {
-        return ScriptUtils.readDirRecSync("./assets/layers")
-            .filter(path => path.indexOf(".json") > 0)
-            .filter(path => path.indexOf(".proto.json") < 0)
-            .filter(path => path.indexOf("license_info.json") < 0)
-            .map(path => {
-                try {
-                    const contents = readFileSync(path, "UTF8")
-                    if (contents === "") {
-                        throw "The file " + path + " is empty, did you properly save?"
-                    }
-
-                    const parsed = JSON.parse(contents);
-                    return {parsed, path}
-                } catch (e) {
-                    console.error("Could not parse file ", "./assets/layers/" + path, "due to ", e)
-                    throw e
-                }
-            })
-    }
-
-    public static getThemeFiles(): { parsed: LayoutConfigJson, path: string }[] {
-        return ScriptUtils.readDirRecSync("./assets/themes")
-            .filter(path => path.endsWith(".json") && !path.endsWith(".proto.json"))
-            .filter(path => path.indexOf("license_info.json") < 0)
-            .map(path => {
-                try {
-                    const contents = readFileSync(path, "UTF8");
-                    if (contents === "") {
-                        throw "The file " + path + " is empty, did you properly save?"
-                    }
-                    const parsed = JSON.parse(contents);
-                    return {parsed: parsed, path: path}
-                } catch (e) {
-                    console.error("Could not read file ", path, "due to ", e)
-                    throw e
-                }
-            });
-    }
-
-
-    public static TagInfoHistogram(key: string): Promise<{
-        data: { count: number, value: string, fraction: number }[]
-    }> {
-        const url = `https://taginfo.openstreetmap.org/api/4/key/values?key=${key}&filter=all&lang=en&sortname=count&sortorder=desc&page=1&rp=17&qtype=value`
-        return ScriptUtils.DownloadJSON(url)
-    }
-
-    public static async ReadSvg(path: string): Promise<any>{
-        if(!existsSync(path)){
-            throw "File not found: "+path
-        }
-        const root =  await xml2js.parseStringPromise(readFileSync(path, "UTF8"))
-        return root.svg
-    }
-
-    public static async ReadSvgSync(path: string, callback: ((svg: any) => void)): Promise<any>{
-         xml2js.parseString(readFileSync(path, "UTF8"),{async: false} , (err, root) => {
-             if(err){
-                 throw err
-             }
-             callback(root["svg"]);
-         })
-    }
-    
 }

scripts/build.sh

@@ -12,7 +12,7 @@ mkdir dist/assets 2> /dev/null
 # This script ends every line with '&&' to chain everything. A failure will thus stop the build
 npm run generate:editor-layer-index 
 npm run generate &&
-npm run generate:layeroverview && # generate:layeroverview has to be run twice: the personal theme won't pick up all the layers otherwise
+ts-node ./scripts/generateLayeroverview --force && # generate:layeroverview has to be run twice: the personal theme won't pick up all the layers otherwise
 npm run test &&
 npm run generate:layouts 
 

scripts/generateLayerOverview.ts

@@ -1,5 +1,5 @@
 import ScriptUtils from "./ScriptUtils";
-import {existsSync, mkdirSync, readFileSync, writeFileSync} from "fs";
+import {existsSync, mkdirSync, readFileSync, statSync, writeFileSync} from "fs";
 import * as licenses from "../assets/generated/license_info.json"
 import {LayoutConfigJson} from "../Models/ThemeConfig/Json/LayoutConfigJson";
 import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
@@ -26,6 +26,51 @@ import {Utils} from "../Utils";
 
 class LayerOverviewUtils {
 
+    public static readonly layerPath = "./assets/generated/layers/"
+    public static readonly themePath = "./assets/generated/themes/"
+
+    private static publicLayerIdsFrom(themefiles: LayoutConfigJson[]): Set<string> {
+        const publicThemes = [].concat(...themefiles
+            .filter(th => !th.hideFromOverview))
+
+        return new Set([].concat(...publicThemes.map(th => this.extractLayerIdsFrom(th))))
+    }
+
+    private static extractLayerIdsFrom(themeFile: LayoutConfigJson, includeInlineLayers = true): string[] {
+        const publicLayerIds = []
+        for (const publicLayer of themeFile.layers) {
+            if (typeof publicLayer === "string") {
+                publicLayerIds.push(publicLayer)
+                continue
+            }
+            if (publicLayer["builtin"] !== undefined) {
+                const bi = publicLayer["builtin"]
+                if (typeof bi === "string") {
+                    publicLayerIds.push(bi)
+                    continue
+                }
+                bi.forEach(id => publicLayerIds.push(id))
+                continue
+            }
+            if (includeInlineLayers) {
+                publicLayerIds.push(publicLayer["id"])
+            }
+        }
+        return publicLayerIds
+    }
+
+    shouldBeUpdated(sourcefile: string | string[], targetfile: string): boolean {
+        if (!existsSync(targetfile)) {
+            return true;
+        }
+        const targetModified = statSync(targetfile).mtime
+        if (typeof sourcefile === "string") {
+            sourcefile = [sourcefile]
+        }
+
+        return sourcefile.some(sourcefile => statSync(sourcefile).mtime > targetModified)
+    }
+
     writeSmallOverview(themes: { id: string, title: any, shortDescription: any, icon: string, hideFromOverview: boolean, mustHaveLanguage: boolean, layers: (LayerConfigJson | string | { builtin })[] }[]) {
         const perId = new Map<string, any>();
         for (const theme of themes) {
@@ -70,17 +115,17 @@ class LayerOverviewUtils {
     }
 
     writeTheme(theme: LayoutConfigJson) {
-        if (!existsSync("./assets/generated/themes")) {
+        if (!existsSync(LayerOverviewUtils.themePath)) {
-            mkdirSync("./assets/generated/themes");
+            mkdirSync(LayerOverviewUtils.themePath);
         }
-        writeFileSync(`./assets/generated/themes/${theme.id}.json`, JSON.stringify(theme, null, "  "), "UTF8");
+        writeFileSync(`${LayerOverviewUtils.themePath}${theme.id}.json`, JSON.stringify(theme, null, "  "), "UTF8");
     }
 
     writeLayer(layer: LayerConfigJson) {
-        if (!existsSync("./assets/generated/layers")) {
+        if (!existsSync(LayerOverviewUtils.layerPath)) {
-            mkdirSync("./assets/generated/layers");
+            mkdirSync(LayerOverviewUtils.layerPath);
         }
-        writeFileSync(`./assets/generated/layers/${layer.id}.json`, JSON.stringify(layer, null, "  "), "UTF8");
+        writeFileSync(`${LayerOverviewUtils.layerPath}${layer.id}.json`, JSON.stringify(layer, null, "  "), "UTF8");
     }
 
     getSharedTagRenderings(doesImageExist: DoesImageExist): Map<string, TagRenderingConfigJson> {
@@ -94,7 +139,7 @@ class LayerOverviewUtils {
             questions[key].id = key;
             questions[key]["source"] = "shared-questions"
             const config = <TagRenderingConfigJson>questions[key]
-            validator.convertStrict(config, "generate-layer-overview:tagRenderings/questions.json:"+key)
+            validator.convertStrict(config, "generate-layer-overview:tagRenderings/questions.json:" + key)
             dict.set(key, config)
         }
         for (const key in icons["default"]) {
@@ -105,9 +150,9 @@ class LayerOverviewUtils {
                 continue
             }
             icons[key].id = key;
-            const config =  <TagRenderingConfigJson>icons[key]
+            const config = <TagRenderingConfigJson>icons[key]
-            validator.convertStrict(config, "generate-layer-overview:tagRenderings/icons.json:"+key)
+            validator.convertStrict(config, "generate-layer-overview:tagRenderings/icons.json:" + key)
-            dict.set(key,config)
+            dict.set(key, config)
         }
 
         dict.forEach((value, key) => {
@@ -150,16 +195,18 @@ class LayerOverviewUtils {
         }
     }
 
+    main(args: string[]) {
         
-    main(_: string[]) {
+        const forceReload = args.some(a => a == "--force")
 
         const licensePaths = new Set<string>()
         for (const i in licenses) {
             licensePaths.add(licenses[i].path)
         }
         const doesImageExist = new DoesImageExist(licensePaths, existsSync)
-        const sharedLayers = this.buildLayerIndex(doesImageExist);
+        const sharedLayers = this.buildLayerIndex(doesImageExist, forceReload);
-        const sharedThemes = this.buildThemeIndex(doesImageExist, sharedLayers)
+        const recompiledThemes : string[] = []
+        const sharedThemes = this.buildThemeIndex(doesImageExist, sharedLayers, recompiledThemes, forceReload)
 
         writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
             "layers": Array.from(sharedLayers.values()),
@@ -169,7 +216,7 @@ class LayerOverviewUtils {
         writeFileSync("./assets/generated/known_layers.json", JSON.stringify({layers: Array.from(sharedLayers.values())}))
 
 
-        {
+       if(recompiledThemes.length > 0) {
             // mapcomplete-changes shows an icon for each corresponding mapcomplete-theme
             const iconsPerTheme =
                 Array.from(sharedThemes.values()).map(th => ({
@@ -189,28 +236,42 @@ class LayerOverviewUtils {
         console.log(green("All done!"))
     }
 
-    private buildLayerIndex(doesImageExist: DoesImageExist): Map<string, LayerConfigJson> {
+    private buildLayerIndex(doesImageExist: DoesImageExist, forceReload: boolean): Map<string, LayerConfigJson> {
         // First, we expand and validate all builtin layers. These are written to assets/generated/layers
         // At the same time, an index of available layers is built.
         console.log("   ---------- VALIDATING BUILTIN LAYERS ---------")
 
         const sharedTagRenderings = this.getSharedTagRenderings(doesImageExist);
-        const layerFiles = ScriptUtils.getLayerFiles();
         const sharedLayers = new Map<string, LayerConfigJson>()
         const state: DesugaringContext = {
             tagRenderings: sharedTagRenderings,
             sharedLayers
         }
         const prepLayer = new PrepareLayer(state);
-        for (const sharedLayerJson of layerFiles) {
+        const skippedLayers: string[] = []
-            const context = "While building builtin layer " + sharedLayerJson.path
+        const recompiledLayers: string[] = []
-            const fixed = prepLayer.convertStrict(sharedLayerJson.parsed, context)
+        for (const sharedLayerPath of ScriptUtils.getLayerPaths()) {
 
-            if(fixed.source.osmTags["and"] === undefined){
+            {
+                const targetPath = LayerOverviewUtils.layerPath + sharedLayerPath.substring(sharedLayerPath.lastIndexOf("/"))
+                if (!forceReload && !this.shouldBeUpdated(sharedLayerPath, targetPath)) {
+                    const sharedLayer = JSON.parse(readFileSync(targetPath, "utf8"))
+                    sharedLayers.set(sharedLayer.id, sharedLayer)
+                    skippedLayers.push(sharedLayer.id)
+                    continue;
+                }
+            
+            }
+
+            const parsed = JSON.parse(readFileSync(sharedLayerPath, "utf8"))
+            const context = "While building builtin layer " + sharedLayerPath
+            const fixed = prepLayer.convertStrict(parsed, context)
+
+            if (fixed.source.osmTags["and"] === undefined) {
                 fixed.source.osmTags = {"and": [fixed.source.osmTags]}
             }
 
-            const validator = new ValidateLayer(sharedLayerJson.path, true, doesImageExist);
+            const validator = new ValidateLayer(sharedLayerPath, true, doesImageExist);
             validator.convertStrict(fixed, context)
 
             if (sharedLayers.has(fixed.id)) {
@@ -218,39 +279,18 @@ class LayerOverviewUtils {
             }
 
             sharedLayers.set(fixed.id, fixed)
+            recompiledLayers.push(fixed.id)
 
             this.writeLayer(fixed)
 
         }
+
+        console.log("Recompiled layers " + recompiledLayers.join(", ") + " and skipped " + skippedLayers.length + " layers")
+
         return sharedLayers;
     }
 
-    private static publicLayerIdsFrom(themefiles: LayoutConfigJson[]): Set<string> {
+    private buildThemeIndex(doesImageExist: DoesImageExist, sharedLayers: Map<string, LayerConfigJson>, recompiledThemes: string[], forceReload: boolean): Map<string, LayoutConfigJson> {
-        const publicLayers = [].concat(...themefiles
-            .filter(th => !th.hideFromOverview)
-            .map(th => th.layers))
-
-        const publicLayerIds = new Set<string>()
-        for (const publicLayer of publicLayers) {
-            if (typeof publicLayer === "string") {
-                publicLayerIds.add(publicLayer)
-                continue
-            }
-            if (publicLayer["builtin"] !== undefined) {
-                const bi = publicLayer["builtin"]
-                if (typeof bi === "string") {
-                    publicLayerIds.add(bi)
-                    continue
-                }
-                bi.forEach(id => publicLayerIds.add(id))
-                continue
-            }
-            publicLayerIds.add(publicLayer.id)
-        }
-        return publicLayerIds
-    }
-
-    private buildThemeIndex(doesImageExist: DoesImageExist, sharedLayers: Map<string, LayerConfigJson>): Map<string, LayoutConfigJson> {
         console.log("   ---------- VALIDATING BUILTIN THEMES ---------")
         const themeFiles = ScriptUtils.getThemeFiles();
         const fixed = new Map<string, LayoutConfigJson>();
@@ -262,9 +302,23 @@ class LayerOverviewUtils {
             tagRenderings: this.getSharedTagRenderings(doesImageExist),
             publicLayers
         }
+        const skippedThemes: string[] = []
         for (const themeInfo of themeFiles) {
+
+            const themePath = themeInfo.path;
             let themeFile = themeInfo.parsed
-            const themePath = themeInfo.path
+
+            {
+                const targetPath = LayerOverviewUtils.themePath + "/" + themePath.substring(themePath.lastIndexOf("/"))
+                const usedLayers = Array.from(LayerOverviewUtils.extractLayerIdsFrom(themeFile, false))
+                    .map(id => LayerOverviewUtils.layerPath + id + ".json")
+                if (!forceReload && !this.shouldBeUpdated([themePath, ...usedLayers], targetPath)) {
+                    fixed.set(themeFile.id, themeFile)
+                    skippedThemes.push(themeFile.id)
+                    continue;
+                }
+                recompiledThemes.push(themeFile.id)
+            }
 
             new PrevalidateTheme().convertStrict(themeFile, themePath)
             try {
@@ -290,6 +344,9 @@ class LayerOverviewUtils {
                 mustHaveLanguage: t.mustHaveLanguage?.length > 0,
             }
         }));
+
+        console.log("Recompiled themes " + recompiledThemes.join(", ") + " and skipped " + skippedThemes.length + " themes")
+
         return fixed;
 
     }