From 8c036e159f0fbf5247f925ecd884f7c43aa40a13 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Wed, 6 Jul 2022 13:58:56 +0200 Subject: [PATCH] Generate layer overview now only recompiles files that need to be recompiled --- package.json | 2 +- scripts/ScriptUtils.ts | 176 ++++++++++++++++--------------- scripts/build.sh | 2 +- scripts/generateLayerOverview.ts | 161 +++++++++++++++++++--------- 4 files changed, 204 insertions(+), 137 deletions(-) diff --git a/package.json b/package.json index 58c8dabfa..e1a66f1c3 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "generate:cache:speelplekken": "npm run generate:layeroverview && ts-node scripts/generateCache.ts speelplekken 14 ../MapComplete-data/speelplekken_cache/ 51.20 4.35 51.09 4.56", "generate:cache:natuurpunt": "npm run generate:layeroverview && ts-node scripts/generateCache.ts natuurpunt 12 ../MapComplete-data/natuurpunt_cache/ 50.40 2.1 51.54 6.4 --generate-point-overview nature_reserve,visitor_information_centre", "generate:cache:natuurpunt:mini": "ts-node scripts/generateCache.ts natuurpunt 12 ../../git/MapComplete-data/natuurpunt_cache_mini/ 51.00792239979105 4.497699737548828 51.0353492224462554 4.539070129394531 --generate-point-overview nature_reserve,visitor_information_centre", - "generate:layeroverview": "ts-node scripts/generateLayerOverview.ts --no-fail", + "generate:layeroverview": "ts-node scripts/generateLayerOverview.ts", "generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail", "query:licenses": "ts-node scripts/generateLicenseInfo.ts --query", "generate:report": "cd Docs/Tools && ./compileStats.sh && git commit . -m 'New statistics ands graphs' && git push", diff --git a/scripts/ScriptUtils.ts b/scripts/ScriptUtils.ts index 651ad4351..a86f3b2d4 100644 --- a/scripts/ScriptUtils.ts +++ b/scripts/ScriptUtils.ts @@ -45,13 +45,103 @@ export default class ScriptUtils { }) } - - private static async DownloadJSON(url: string, headers?: any): Promise{ + + public static erasableLog(...text) { + process.stdout.write("\r " + text.join(" ") + " \r") + } + + public static sleep(ms) { + if (ms <= 0) { + process.stdout.write("\r \r") + return; + } + return new Promise((resolve) => { + process.stdout.write("\r Sleeping for " + (ms / 1000) + "s \r") + setTimeout(resolve, 1000); + }).then(() => ScriptUtils.sleep(ms - 1000)); + } + + public static getLayerPaths(): string[] { + return ScriptUtils.readDirRecSync("./assets/layers") + .filter(path => path.indexOf(".json") > 0) + .filter(path => path.indexOf(".proto.json") < 0) + .filter(path => path.indexOf("license_info.json") < 0) + } + + public static getLayerFiles(): { parsed: LayerConfigJson, path: string }[] { + return ScriptUtils.readDirRecSync("./assets/layers") + .filter(path => path.indexOf(".json") > 0) + .filter(path => path.indexOf(".proto.json") < 0) + .filter(path => path.indexOf("license_info.json") < 0) + .map(path => { + try { + const contents = readFileSync(path, "UTF8") + if (contents === "") { + throw "The file " + path + " is empty, did you properly save?" + } + + const parsed = JSON.parse(contents); + return {parsed, path} + } catch (e) { + console.error("Could not parse file ", "./assets/layers/" + path, "due to ", e) + throw e + } + }) + } + + public static getThemePaths(): string[] { + return ScriptUtils.readDirRecSync("./assets/themes") + .filter(path => path.endsWith(".json") && !path.endsWith(".proto.json")) + .filter(path => path.indexOf("license_info.json") < 0) + } + + public static getThemeFiles(): { parsed: LayoutConfigJson, path: string }[] { + return this.getThemePaths() + .map(path => { + try { + const contents = readFileSync(path, "UTF8"); + if (contents === "") { + throw "The file " + path + " is empty, did you properly save?" + } + const parsed = JSON.parse(contents); + return {parsed: parsed, path: path} + } catch (e) { + console.error("Could not read file ", path, "due to ", e) + throw e + } + }); + } + + public static TagInfoHistogram(key: string): Promise<{ + data: { count: number, value: string, fraction: number }[] + }> { + const url = `https://taginfo.openstreetmap.org/api/4/key/values?key=${key}&filter=all&lang=en&sortname=count&sortorder=desc&page=1&rp=17&qtype=value` + return ScriptUtils.DownloadJSON(url) + } + + public static async ReadSvg(path: string): Promise { + if (!existsSync(path)) { + throw "File not found: " + path + } + const root = await xml2js.parseStringPromise(readFileSync(path, "UTF8")) + return root.svg + } + + public static async ReadSvgSync(path: string, callback: ((svg: any) => void)): Promise { + xml2js.parseString(readFileSync(path, "UTF8"), {async: false}, (err, root) => { + if (err) { + throw err + } + callback(root["svg"]); + }) + } + + private static async DownloadJSON(url: string, headers?: any): Promise { const data = await ScriptUtils.Download(url, headers); return JSON.parse(data.content) } - private static Download(url, headers?: any): Promise<{content: string}> { + private static Download(url, headers?: any): Promise<{ content: string }> { return new Promise((resolve, reject) => { try { headers = headers ?? {} @@ -83,84 +173,4 @@ export default class ScriptUtils { } - public static erasableLog(...text) { - process.stdout.write("\r " + text.join(" ") + " \r") - } - - public static sleep(ms) { - if (ms <= 0) { - process.stdout.write("\r \r") - return; - } - return new Promise((resolve) => { - process.stdout.write("\r Sleeping for " + (ms / 1000) + "s \r") - setTimeout(resolve, 1000); - }).then(() => ScriptUtils.sleep(ms - 1000)); - } - - public static getLayerFiles(): { parsed: LayerConfigJson, path: string }[] { - return ScriptUtils.readDirRecSync("./assets/layers") - .filter(path => path.indexOf(".json") > 0) - .filter(path => path.indexOf(".proto.json") < 0) - .filter(path => path.indexOf("license_info.json") < 0) - .map(path => { - try { - const contents = readFileSync(path, "UTF8") - if (contents === "") { - throw "The file " + path + " is empty, did you properly save?" - } - - const parsed = JSON.parse(contents); - return {parsed, path} - } catch (e) { - console.error("Could not parse file ", "./assets/layers/" + path, "due to ", e) - throw e - } - }) - } - - public static getThemeFiles(): { parsed: LayoutConfigJson, path: string }[] { - return ScriptUtils.readDirRecSync("./assets/themes") - .filter(path => path.endsWith(".json") && !path.endsWith(".proto.json")) - .filter(path => path.indexOf("license_info.json") < 0) - .map(path => { - try { - const contents = readFileSync(path, "UTF8"); - if (contents === "") { - throw "The file " + path + " is empty, did you properly save?" - } - const parsed = JSON.parse(contents); - return {parsed: parsed, path: path} - } catch (e) { - console.error("Could not read file ", path, "due to ", e) - throw e - } - }); - } - - - public static TagInfoHistogram(key: string): Promise<{ - data: { count: number, value: string, fraction: number }[] - }> { - const url = `https://taginfo.openstreetmap.org/api/4/key/values?key=${key}&filter=all&lang=en&sortname=count&sortorder=desc&page=1&rp=17&qtype=value` - return ScriptUtils.DownloadJSON(url) - } - - public static async ReadSvg(path: string): Promise{ - if(!existsSync(path)){ - throw "File not found: "+path - } - const root = await xml2js.parseStringPromise(readFileSync(path, "UTF8")) - return root.svg - } - - public static async ReadSvgSync(path: string, callback: ((svg: any) => void)): Promise{ - xml2js.parseString(readFileSync(path, "UTF8"),{async: false} , (err, root) => { - if(err){ - throw err - } - callback(root["svg"]); - }) - } - } diff --git a/scripts/build.sh b/scripts/build.sh index 804ce9b52..28bd83dee 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -12,7 +12,7 @@ mkdir dist/assets 2> /dev/null # This script ends every line with '&&' to chain everything. A failure will thus stop the build npm run generate:editor-layer-index npm run generate && -npm run generate:layeroverview && # generate:layeroverview has to be run twice: the personal theme won't pick up all the layers otherwise +ts-node ./scripts/generateLayeroverview --force && # generate:layeroverview has to be run twice: the personal theme won't pick up all the layers otherwise npm run test && npm run generate:layouts diff --git a/scripts/generateLayerOverview.ts b/scripts/generateLayerOverview.ts index 866155db8..99e733a20 100644 --- a/scripts/generateLayerOverview.ts +++ b/scripts/generateLayerOverview.ts @@ -1,5 +1,5 @@ import ScriptUtils from "./ScriptUtils"; -import {existsSync, mkdirSync, readFileSync, writeFileSync} from "fs"; +import {existsSync, mkdirSync, readFileSync, statSync, writeFileSync} from "fs"; import * as licenses from "../assets/generated/license_info.json" import {LayoutConfigJson} from "../Models/ThemeConfig/Json/LayoutConfigJson"; import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson"; @@ -26,6 +26,51 @@ import {Utils} from "../Utils"; class LayerOverviewUtils { + public static readonly layerPath = "./assets/generated/layers/" + public static readonly themePath = "./assets/generated/themes/" + + private static publicLayerIdsFrom(themefiles: LayoutConfigJson[]): Set { + const publicThemes = [].concat(...themefiles + .filter(th => !th.hideFromOverview)) + + return new Set([].concat(...publicThemes.map(th => this.extractLayerIdsFrom(th)))) + } + + private static extractLayerIdsFrom(themeFile: LayoutConfigJson, includeInlineLayers = true): string[] { + const publicLayerIds = [] + for (const publicLayer of themeFile.layers) { + if (typeof publicLayer === "string") { + publicLayerIds.push(publicLayer) + continue + } + if (publicLayer["builtin"] !== undefined) { + const bi = publicLayer["builtin"] + if (typeof bi === "string") { + publicLayerIds.push(bi) + continue + } + bi.forEach(id => publicLayerIds.push(id)) + continue + } + if (includeInlineLayers) { + publicLayerIds.push(publicLayer["id"]) + } + } + return publicLayerIds + } + + shouldBeUpdated(sourcefile: string | string[], targetfile: string): boolean { + if (!existsSync(targetfile)) { + return true; + } + const targetModified = statSync(targetfile).mtime + if (typeof sourcefile === "string") { + sourcefile = [sourcefile] + } + + return sourcefile.some(sourcefile => statSync(sourcefile).mtime > targetModified) + } + writeSmallOverview(themes: { id: string, title: any, shortDescription: any, icon: string, hideFromOverview: boolean, mustHaveLanguage: boolean, layers: (LayerConfigJson | string | { builtin })[] }[]) { const perId = new Map(); for (const theme of themes) { @@ -70,22 +115,22 @@ class LayerOverviewUtils { } writeTheme(theme: LayoutConfigJson) { - if (!existsSync("./assets/generated/themes")) { - mkdirSync("./assets/generated/themes"); + if (!existsSync(LayerOverviewUtils.themePath)) { + mkdirSync(LayerOverviewUtils.themePath); } - writeFileSync(`./assets/generated/themes/${theme.id}.json`, JSON.stringify(theme, null, " "), "UTF8"); + writeFileSync(`${LayerOverviewUtils.themePath}${theme.id}.json`, JSON.stringify(theme, null, " "), "UTF8"); } writeLayer(layer: LayerConfigJson) { - if (!existsSync("./assets/generated/layers")) { - mkdirSync("./assets/generated/layers"); + if (!existsSync(LayerOverviewUtils.layerPath)) { + mkdirSync(LayerOverviewUtils.layerPath); } - writeFileSync(`./assets/generated/layers/${layer.id}.json`, JSON.stringify(layer, null, " "), "UTF8"); + writeFileSync(`${LayerOverviewUtils.layerPath}${layer.id}.json`, JSON.stringify(layer, null, " "), "UTF8"); } getSharedTagRenderings(doesImageExist: DoesImageExist): Map { const dict = new Map(); - + const validator = new ValidateTagRenderings(undefined, doesImageExist); for (const key in questions["default"]) { if (key === "id") { @@ -94,7 +139,7 @@ class LayerOverviewUtils { questions[key].id = key; questions[key]["source"] = "shared-questions" const config = questions[key] - validator.convertStrict(config, "generate-layer-overview:tagRenderings/questions.json:"+key) + validator.convertStrict(config, "generate-layer-overview:tagRenderings/questions.json:" + key) dict.set(key, config) } for (const key in icons["default"]) { @@ -105,9 +150,9 @@ class LayerOverviewUtils { continue } icons[key].id = key; - const config = icons[key] - validator.convertStrict(config, "generate-layer-overview:tagRenderings/icons.json:"+key) - dict.set(key,config) + const config = icons[key] + validator.convertStrict(config, "generate-layer-overview:tagRenderings/icons.json:" + key) + dict.set(key, config) } dict.forEach((value, key) => { @@ -150,16 +195,18 @@ class LayerOverviewUtils { } } - - main(_: string[]) { + main(args: string[]) { + + const forceReload = args.some(a => a == "--force") const licensePaths = new Set() for (const i in licenses) { licensePaths.add(licenses[i].path) } const doesImageExist = new DoesImageExist(licensePaths, existsSync) - const sharedLayers = this.buildLayerIndex(doesImageExist); - const sharedThemes = this.buildThemeIndex(doesImageExist, sharedLayers) + const sharedLayers = this.buildLayerIndex(doesImageExist, forceReload); + const recompiledThemes : string[] = [] + const sharedThemes = this.buildThemeIndex(doesImageExist, sharedLayers, recompiledThemes, forceReload) writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({ "layers": Array.from(sharedLayers.values()), @@ -169,7 +216,7 @@ class LayerOverviewUtils { writeFileSync("./assets/generated/known_layers.json", JSON.stringify({layers: Array.from(sharedLayers.values())})) - { + if(recompiledThemes.length > 0) { // mapcomplete-changes shows an icon for each corresponding mapcomplete-theme const iconsPerTheme = Array.from(sharedThemes.values()).map(th => ({ @@ -189,28 +236,42 @@ class LayerOverviewUtils { console.log(green("All done!")) } - private buildLayerIndex(doesImageExist: DoesImageExist): Map { + private buildLayerIndex(doesImageExist: DoesImageExist, forceReload: boolean): Map { // First, we expand and validate all builtin layers. These are written to assets/generated/layers // At the same time, an index of available layers is built. console.log(" ---------- VALIDATING BUILTIN LAYERS ---------") const sharedTagRenderings = this.getSharedTagRenderings(doesImageExist); - const layerFiles = ScriptUtils.getLayerFiles(); const sharedLayers = new Map() const state: DesugaringContext = { tagRenderings: sharedTagRenderings, sharedLayers } const prepLayer = new PrepareLayer(state); - for (const sharedLayerJson of layerFiles) { - const context = "While building builtin layer " + sharedLayerJson.path - const fixed = prepLayer.convertStrict(sharedLayerJson.parsed, context) + const skippedLayers: string[] = [] + const recompiledLayers: string[] = [] + for (const sharedLayerPath of ScriptUtils.getLayerPaths()) { + + { + const targetPath = LayerOverviewUtils.layerPath + sharedLayerPath.substring(sharedLayerPath.lastIndexOf("/")) + if (!forceReload && !this.shouldBeUpdated(sharedLayerPath, targetPath)) { + const sharedLayer = JSON.parse(readFileSync(targetPath, "utf8")) + sharedLayers.set(sharedLayer.id, sharedLayer) + skippedLayers.push(sharedLayer.id) + continue; + } - if(fixed.source.osmTags["and"] === undefined){ + } + + const parsed = JSON.parse(readFileSync(sharedLayerPath, "utf8")) + const context = "While building builtin layer " + sharedLayerPath + const fixed = prepLayer.convertStrict(parsed, context) + + if (fixed.source.osmTags["and"] === undefined) { fixed.source.osmTags = {"and": [fixed.source.osmTags]} } - - const validator = new ValidateLayer(sharedLayerJson.path, true, doesImageExist); + + const validator = new ValidateLayer(sharedLayerPath, true, doesImageExist); validator.convertStrict(fixed, context) if (sharedLayers.has(fixed.id)) { @@ -218,39 +279,18 @@ class LayerOverviewUtils { } sharedLayers.set(fixed.id, fixed) + recompiledLayers.push(fixed.id) this.writeLayer(fixed) } + + console.log("Recompiled layers " + recompiledLayers.join(", ") + " and skipped " + skippedLayers.length + " layers") + return sharedLayers; } - private static publicLayerIdsFrom(themefiles: LayoutConfigJson[]): Set { - const publicLayers = [].concat(...themefiles - .filter(th => !th.hideFromOverview) - .map(th => th.layers)) - - const publicLayerIds = new Set() - for (const publicLayer of publicLayers) { - if (typeof publicLayer === "string") { - publicLayerIds.add(publicLayer) - continue - } - if (publicLayer["builtin"] !== undefined) { - const bi = publicLayer["builtin"] - if (typeof bi === "string") { - publicLayerIds.add(bi) - continue - } - bi.forEach(id => publicLayerIds.add(id)) - continue - } - publicLayerIds.add(publicLayer.id) - } - return publicLayerIds - } - - private buildThemeIndex(doesImageExist: DoesImageExist, sharedLayers: Map): Map { + private buildThemeIndex(doesImageExist: DoesImageExist, sharedLayers: Map, recompiledThemes: string[], forceReload: boolean): Map { console.log(" ---------- VALIDATING BUILTIN THEMES ---------") const themeFiles = ScriptUtils.getThemeFiles(); const fixed = new Map(); @@ -262,9 +302,23 @@ class LayerOverviewUtils { tagRenderings: this.getSharedTagRenderings(doesImageExist), publicLayers } + const skippedThemes: string[] = [] for (const themeInfo of themeFiles) { + + const themePath = themeInfo.path; let themeFile = themeInfo.parsed - const themePath = themeInfo.path + + { + const targetPath = LayerOverviewUtils.themePath + "/" + themePath.substring(themePath.lastIndexOf("/")) + const usedLayers = Array.from(LayerOverviewUtils.extractLayerIdsFrom(themeFile, false)) + .map(id => LayerOverviewUtils.layerPath + id + ".json") + if (!forceReload && !this.shouldBeUpdated([themePath, ...usedLayers], targetPath)) { + fixed.set(themeFile.id, themeFile) + skippedThemes.push(themeFile.id) + continue; + } + recompiledThemes.push(themeFile.id) + } new PrevalidateTheme().convertStrict(themeFile, themePath) try { @@ -290,6 +344,9 @@ class LayerOverviewUtils { mustHaveLanguage: t.mustHaveLanguage?.length > 0, } })); + + console.log("Recompiled themes " + recompiledThemes.join(", ") + " and skipped " + skippedThemes.length + " themes") + return fixed; }