From dbff4c9b834ef02f35951b5ecdfa2d72b4985680 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Tue, 16 Jan 2024 04:01:10 +0100 Subject: [PATCH] Tooling: allow to generate only a few themes --- package.json | 1 + scripts/generateLayerOverview.ts | 172 +++++++++++++++++-------------- 2 files changed, 97 insertions(+), 76 deletions(-) diff --git a/package.json b/package.json index 284e24768..c50c85c3a 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "generate:cache:speelplekken": "npm run generate:layeroverview && vite-node scripts/generateCache.ts -- speelplekken 14 ../MapComplete-data/speelplekken_cache/ 51.20 4.35 51.09 4.56", "generate:cache:natuurpunt": "npm run generate:layeroverview && vite-node scripts/generateCache.ts -- natuurpunt 12 ../MapComplete-data/natuurpunt_cache/ 50.40 2.1 51.54 6.4 --generate-point-overview nature_reserve,visitor_information_centre", "generate:layeroverview": "export NODE_OPTIONS=\"--max-old-space-size=8192\" && vite-node scripts/generateLayerOverview.ts", + "generate:layeroverview:velopark": "export NODE_OPTIONS=\"--max-old-space-size=8192\" && vite-node scripts/generateLayerOverview.ts -- --force --themes=velopark --layers=bike_parking,maproulette_challenge", "generate:mapcomplete-changes-theme": "export NODE_OPTIONS=\"--max-old-space-size=8192\" && vite-node scripts/generateLayerOverview.ts -- --generate-change-map", "refresh:layeroverview": "export NODE_OPTIONS=\"--max-old-space-size=8192\" && vite-node scripts/generateLayerOverview.ts -- --force", "generate:licenses": "vite-node scripts/generateLicenseInfo.ts -- --no-fail", diff --git a/scripts/generateLayerOverview.ts b/scripts/generateLayerOverview.ts index ef32924e4..39e8c6424 100644 --- a/scripts/generateLayerOverview.ts +++ b/scripts/generateLayerOverview.ts @@ -14,11 +14,7 @@ import { import { Translation } from "../src/UI/i18n/Translation" import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer" import { PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme" -import { - Conversion, - DesugaringContext, - DesugaringStep, -} from "../src/Models/ThemeConfig/Conversion/Conversion" +import { Conversion, DesugaringContext, DesugaringStep } from "../src/Models/ThemeConfig/Conversion/Conversion" import { Utils } from "../src/Utils" import Script from "./Script" import { AllSharedLayers } from "../src/Customizations/AllSharedLayers" @@ -51,7 +47,7 @@ class ParseLayer extends Conversion< convert( path: string, - context: ConversionContext + context: ConversionContext, ): { parsed: LayerConfig raw: LayerConfigJson @@ -106,7 +102,7 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye const fixed = json.raw const layerConfig = json.parsed const pointRendering: PointRenderingConfig = layerConfig.mapRendering.find((pr) => - pr.location.has("point") + pr.location.has("point"), ) const defaultTags = layerConfig.GetBaseTags() fixed["_layerIcon"] = Utils.NoNull( @@ -121,7 +117,7 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye result["color"] = c } return result - }) + }), ) return { raw: fixed, parsed: layerConfig } } @@ -143,7 +139,7 @@ class LayerOverviewUtils extends Script { private static extractLayerIdsFrom( themeFile: LayoutConfigJson, - includeInlineLayers = true + includeInlineLayers = true, ): string[] { const publicLayerIds = [] if (!Array.isArray(themeFile.layers)) { @@ -206,10 +202,10 @@ class LayerOverviewUtils extends Script { | LayerConfigJson | string | { - builtin - } - )[] - }[] + builtin + } + )[] + }[], ) { const perId = new Map() for (const theme of themes) { @@ -250,7 +246,7 @@ class LayerOverviewUtils extends Script { writeFileSync( "./src/assets/generated/theme_overview.json", JSON.stringify(sorted, null, " "), - { encoding: "utf8" } + { encoding: "utf8" }, ) } @@ -262,7 +258,7 @@ class LayerOverviewUtils extends Script { writeFileSync( `${LayerOverviewUtils.themePath}${theme.id}.json`, JSON.stringify(theme, null, " "), - { encoding: "utf8" } + { encoding: "utf8" }, ) } @@ -273,13 +269,13 @@ class LayerOverviewUtils extends Script { writeFileSync( `${LayerOverviewUtils.layerPath}${layer.id}.json`, JSON.stringify(layer, null, " "), - { encoding: "utf8" } + { encoding: "utf8" }, ) } getSharedTagRenderings( doesImageExist: DoesImageExist, - bootstrapTagRenderings: Map = null + bootstrapTagRenderings: Map = null, ): Map { const prepareLayer = new PrepareLayer({ tagRenderings: bootstrapTagRenderings, @@ -340,8 +336,8 @@ class LayerOverviewUtils extends Script { if (contents.indexOf(" 0) { console.warn( "The SVG at " + - path + - " contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path" + path + + " contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path", ) errCount++ } @@ -353,6 +349,12 @@ class LayerOverviewUtils extends Script { async main(args: string[]) { console.log("Generating layer overview...") + const themeWhitelist = new Set(args.find(a => a.startsWith("--themes=")) + ?.substring("--themes=".length)?.split(",") ?? []) + + const layerWhitelist = new Set(args.find(a => a.startsWith("--layers=")) + ?.substring("--layers=".length)?.split(",") ?? []) + const start = new Date() const forceReload = args.some((a) => a == "--force") @@ -361,7 +363,7 @@ class LayerOverviewUtils extends Script { licensePaths.add(licenses[i].path) } const doesImageExist = new DoesImageExist(licensePaths, existsSync) - const sharedLayers = this.buildLayerIndex(doesImageExist, forceReload) + const sharedLayers = this.buildLayerIndex(doesImageExist, forceReload, layerWhitelist) const priviliged = new Set(Constants.priviliged_layers) sharedLayers.forEach((_, key) => { @@ -379,15 +381,18 @@ class LayerOverviewUtils extends Script { licensePaths, sharedLayers, recompiledThemes, - forceReload + forceReload, + themeWhitelist ) - writeFileSync( - "./src/assets/generated/known_layers.json", - JSON.stringify({ - layers: Array.from(sharedLayers.values()).filter((l) => l.id !== "favourite"), - }) - ) + if (recompiledThemes.length > 0){ + writeFileSync( + "./src/assets/generated/known_layers.json", + JSON.stringify({ + layers: Array.from(sharedLayers.values()).filter((l) => l.id !== "favourite"), + }), + ) + } const mcChangesPath = "./assets/themes/mapcomplete-changes/mapcomplete-changes.json" if ( @@ -406,7 +411,7 @@ class LayerOverviewUtils extends Script { const proto: LayoutConfigJson = JSON.parse( readFileSync("./assets/themes/mapcomplete-changes/mapcomplete-changes.proto.json", { encoding: "utf8", - }) + }), ) const protolayer = ( proto.layers.filter((l) => l["id"] === "mapcomplete-changes")[0] @@ -423,29 +428,31 @@ class LayerOverviewUtils extends Script { layers: ScriptUtils.getLayerFiles().map((f) => f.parsed), themes: ScriptUtils.getThemeFiles().map((f) => f.parsed), }, - ConversionContext.construct([], []) + ConversionContext.construct([], []), ) for (const [_, theme] of sharedThemes) { theme.layers = theme.layers.filter( - (l) => Constants.added_by_default.indexOf(l["id"]) < 0 + (l) => Constants.added_by_default.indexOf(l["id"]) < 0, ) } - writeFileSync( - "./src/assets/generated/known_themes.json", - JSON.stringify({ - themes: Array.from(sharedThemes.values()), - }) - ) + if(recompiledThemes.length > 0) { + writeFileSync( + "./src/assets/generated/known_themes.json", + JSON.stringify({ + themes: Array.from(sharedThemes.values()), + }), + ) + } const end = new Date() const millisNeeded = end.getTime() - start.getTime() if (AllSharedLayers.getSharedLayersConfigs().size == 0) { console.error( "This was a bootstrapping-run. Run generate layeroverview again!(" + - millisNeeded + - " ms)" + millisNeeded + + " ms)", ) } else { const green = (s) => "\x1b[92m" + s + "\x1b[0m" @@ -456,7 +463,7 @@ class LayerOverviewUtils extends Script { private parseLayer( doesImageExist: DoesImageExist, prepLayer: PrepareLayer, - sharedLayerPath: string + sharedLayerPath: string, ): { raw: LayerConfigJson parsed: LayerConfig @@ -467,14 +474,15 @@ class LayerOverviewUtils extends Script { const parsed = parser.convertStrict(sharedLayerPath, context) const result = AddIconSummary.singleton.convertStrict( parsed, - context.inOperation("AddIconSummary") + context.inOperation("AddIconSummary"), ) return { ...result, context } } private buildLayerIndex( doesImageExist: DoesImageExist, - forceReload: boolean + forceReload: boolean, + whitelist: Set ): Map { // First, we expand and validate all builtin layers. These are written to src/assets/generated/layers // At the same time, an index of available layers is built. @@ -492,6 +500,12 @@ class LayerOverviewUtils extends Script { const recompiledLayers: string[] = [] let warningCount = 0 for (const sharedLayerPath of ScriptUtils.getLayerPaths()) { + if(whitelist.size > 0){ + const idByPath = sharedLayerPath.split("/").at(-1).split(".")[0] + if(Constants.priviliged_layers.indexOf( idByPath) < 0 && !whitelist.has(idByPath)){ + continue + } + } { const targetPath = LayerOverviewUtils.layerPath + @@ -520,17 +534,17 @@ class LayerOverviewUtils extends Script { console.log( "Recompiled layers " + - recompiledLayers.join(", ") + - " and skipped " + - skippedLayers.length + - " layers. Detected " + - warningCount + - " warnings" + recompiledLayers.join(", ") + + " and skipped " + + skippedLayers.length + + " layers. Detected " + + warningCount + + " warnings", ) // We always need the calculated tags of 'usersettings', so we export them separately this.extractJavascriptCodeForLayer( state.sharedLayers.get("usersettings"), - "./src/Logic/State/UserSettingsMetaTagging.ts" + "./src/Logic/State/UserSettingsMetaTagging.ts", ) return sharedLayers @@ -547,8 +561,8 @@ class LayerOverviewUtils extends Script { private extractJavascriptCode(themeFile: LayoutConfigJson) { const allCode = [ "import {Feature} from 'geojson'", - 'import { ExtraFuncType } from "../../../Logic/ExtraFunctions";', - 'import { Utils } from "../../../Utils"', + "import { ExtraFuncType } from \"../../../Logic/ExtraFunctions\";", + "import { Utils } from \"../../../Utils\"", "export class ThemeMetaTagging {", " public static readonly themeName = " + JSON.stringify(themeFile.id), "", @@ -560,8 +574,8 @@ class LayerOverviewUtils extends Script { allCode.push( " public metaTaggging_for_" + - id + - "(feat: Feature, helperFunctions: Record Function>) {" + id + + "(feat: Feature, helperFunctions: Record Function>) {", ) allCode.push(" const {" + ExtraFunctions.types.join(", ") + "} = helperFunctions") for (const line of code) { @@ -572,10 +586,10 @@ class LayerOverviewUtils extends Script { if (!isStrict) { allCode.push( " Utils.AddLazyProperty(feat.properties, '" + - attributeName + - "', () => " + - expression + - " ) " + attributeName + + "', () => " + + expression + + " ) ", ) } else { attributeName = attributeName.substring(0, attributeName.length - 1).trim() @@ -620,7 +634,7 @@ class LayerOverviewUtils extends Script { const code = l.calculatedTags ?? [] allCode.push( - " public metaTaggging_for_" + l.id + "(feat: {properties: Record}) {" + " public metaTaggging_for_" + l.id + "(feat: {properties: Record}) {", ) for (const line of code) { const firstEq = line.indexOf("=") @@ -630,10 +644,10 @@ class LayerOverviewUtils extends Script { if (!isStrict) { allCode.push( " Utils.AddLazyProperty(feat.properties, '" + - attributeName + - "', () => " + - expression + - " ) " + attributeName + + "', () => " + + expression + + " ) ", ) } else { attributeName = attributeName.substring(0, attributeName.length - 2).trim() @@ -657,20 +671,21 @@ class LayerOverviewUtils extends Script { licensePaths: Set, sharedLayers: Map, recompiledThemes: string[], - forceReload: boolean + forceReload: boolean, + whitelist: Set ): Map { console.log(" ---------- VALIDATING BUILTIN THEMES ---------") const themeFiles = ScriptUtils.getThemeFiles() const fixed = new Map() const publicLayers = LayerOverviewUtils.publicLayerIdsFrom( - themeFiles.map((th) => th.parsed) + themeFiles.map((th) => th.parsed), ) const convertState: DesugaringContext = { sharedLayers, tagRenderings: this.getSharedTagRenderings( - new DoesImageExist(licensePaths, existsSync) + new DoesImageExist(licensePaths, existsSync), ), publicLayers, } @@ -695,20 +710,23 @@ class LayerOverviewUtils extends Script { const themeInfo = themeFiles[i] const themePath = themeInfo.path let themeFile = themeInfo.parsed + if(whitelist.size > 0 && !whitelist.has(themeFile.id)){ + continue + } const targetPath = LayerOverviewUtils.themePath + "/" + themePath.substring(themePath.lastIndexOf("/")) const usedLayers = Array.from( - LayerOverviewUtils.extractLayerIdsFrom(themeFile, false) + LayerOverviewUtils.extractLayerIdsFrom(themeFile, false), ).map((id) => LayerOverviewUtils.layerPath + id + ".json") if (!forceReload && !this.shouldBeUpdated([themePath, ...usedLayers], targetPath)) { fixed.set( themeFile.id, JSON.parse( - readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", "utf8") - ) + readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", "utf8"), + ), ) ScriptUtils.erasableLog("Skipping", themeFile.id) skippedThemes.push(themeFile.id) @@ -719,23 +737,23 @@ class LayerOverviewUtils extends Script { new PrevalidateTheme().convertStrict( themeFile, - ConversionContext.construct([themePath], ["PrepareLayer"]) + ConversionContext.construct([themePath], ["PrepareLayer"]), ) try { themeFile = new PrepareTheme(convertState, { skipDefaultLayers: true, }).convertStrict( themeFile, - ConversionContext.construct([themePath], ["PrepareLayer"]) + ConversionContext.construct([themePath], ["PrepareLayer"]), ) new ValidateThemeAndLayers( new DoesImageExist(licensePaths, existsSync, knownTagRenderings), themePath, true, - knownTagRenderings + knownTagRenderings, ).convertStrict( themeFile, - ConversionContext.construct([themePath], ["PrepareLayer"]) + ConversionContext.construct([themePath], ["PrepareLayer"]), ) if (themeFile.icon.endsWith(".svg")) { @@ -777,6 +795,7 @@ class LayerOverviewUtils extends Script { } } + if(whitelist.size == 0){ this.writeSmallOverview( Array.from(fixed.values()).map((t) => { return { @@ -789,15 +808,16 @@ class LayerOverviewUtils extends Script { .OnEveryLanguage((s) => parse_html(s).textContent).translations, mustHaveLanguage: t.mustHaveLanguage?.length > 0, } - }) + }), ) + } console.log( "Recompiled themes " + - recompiledThemes.join(", ") + - " and skipped " + - skippedThemes.length + - " themes" + recompiledThemes.join(", ") + + " and skipped " + + skippedThemes.length + + " themes", ) return fixed