forked from MapComplete/MapComplete
Tooling: allow to generate only a few themes
This commit is contained in:
Pieter Vander Vennet
parent
7445ad5fc2
commit
dbff4c9b83
2 changed files with 97 additions and 76 deletions
|
|
@ -54,6 +54,7 @@
|
|||
"generate:cache:speelplekken": "npm run generate:layeroverview && vite-node scripts/generateCache.ts -- speelplekken 14 ../MapComplete-data/speelplekken_cache/ 51.20 4.35 51.09 4.56",
|
||||
"generate:cache:natuurpunt": "npm run generate:layeroverview && vite-node scripts/generateCache.ts -- natuurpunt 12 ../MapComplete-data/natuurpunt_cache/ 50.40 2.1 51.54 6.4 --generate-point-overview nature_reserve,visitor_information_centre",
|
||||
"generate:layeroverview": "export NODE_OPTIONS=\"--max-old-space-size=8192\" && vite-node scripts/generateLayerOverview.ts",
|
||||
"generate:layeroverview:velopark": "export NODE_OPTIONS=\"--max-old-space-size=8192\" && vite-node scripts/generateLayerOverview.ts -- --force --themes=velopark --layers=bike_parking,maproulette_challenge",
|
||||
"generate:mapcomplete-changes-theme": "export NODE_OPTIONS=\"--max-old-space-size=8192\" && vite-node scripts/generateLayerOverview.ts -- --generate-change-map",
|
||||
"refresh:layeroverview": "export NODE_OPTIONS=\"--max-old-space-size=8192\" && vite-node scripts/generateLayerOverview.ts -- --force",
|
||||
"generate:licenses": "vite-node scripts/generateLicenseInfo.ts -- --no-fail",
|
||||
|
|
|
|||
|
|
@ -14,11 +14,7 @@ import {
|
|||
import { Translation } from "../src/UI/i18n/Translation"
|
||||
import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
|
||||
import { PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme"
|
||||
import {
|
||||
Conversion,
|
||||
DesugaringContext,
|
||||
DesugaringStep,
|
||||
} from "../src/Models/ThemeConfig/Conversion/Conversion"
|
||||
import { Conversion, DesugaringContext, DesugaringStep } from "../src/Models/ThemeConfig/Conversion/Conversion"
|
||||
import { Utils } from "../src/Utils"
|
||||
import Script from "./Script"
|
||||
import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
|
||||
|
|
@ -51,7 +47,7 @@ class ParseLayer extends Conversion<
|
|||
|
||||
convert(
|
||||
path: string,
|
||||
context: ConversionContext
|
||||
context: ConversionContext,
|
||||
): {
|
||||
parsed: LayerConfig
|
||||
raw: LayerConfigJson
|
||||
|
|
@ -106,7 +102,7 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye
|
|||
const fixed = json.raw
|
||||
const layerConfig = json.parsed
|
||||
const pointRendering: PointRenderingConfig = layerConfig.mapRendering.find((pr) =>
|
||||
pr.location.has("point")
|
||||
pr.location.has("point"),
|
||||
)
|
||||
const defaultTags = layerConfig.GetBaseTags()
|
||||
fixed["_layerIcon"] = Utils.NoNull(
|
||||
|
|
@ -121,7 +117,7 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye
|
|||
result["color"] = c
|
||||
}
|
||||
return result
|
||||
})
|
||||
}),
|
||||
)
|
||||
return { raw: fixed, parsed: layerConfig }
|
||||
}
|
||||
|
|
@ -143,7 +139,7 @@ class LayerOverviewUtils extends Script {
|
|||
|
||||
private static extractLayerIdsFrom(
|
||||
themeFile: LayoutConfigJson,
|
||||
includeInlineLayers = true
|
||||
includeInlineLayers = true,
|
||||
): string[] {
|
||||
const publicLayerIds = []
|
||||
if (!Array.isArray(themeFile.layers)) {
|
||||
|
|
@ -209,7 +205,7 @@ class LayerOverviewUtils extends Script {
|
|||
builtin
|
||||
}
|
||||
)[]
|
||||
}[]
|
||||
}[],
|
||||
) {
|
||||
const perId = new Map<string, any>()
|
||||
for (const theme of themes) {
|
||||
|
|
@ -250,7 +246,7 @@ class LayerOverviewUtils extends Script {
|
|||
writeFileSync(
|
||||
"./src/assets/generated/theme_overview.json",
|
||||
JSON.stringify(sorted, null, " "),
|
||||
{ encoding: "utf8" }
|
||||
{ encoding: "utf8" },
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -262,7 +258,7 @@ class LayerOverviewUtils extends Script {
|
|||
writeFileSync(
|
||||
`${LayerOverviewUtils.themePath}${theme.id}.json`,
|
||||
JSON.stringify(theme, null, " "),
|
||||
{ encoding: "utf8" }
|
||||
{ encoding: "utf8" },
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -273,13 +269,13 @@ class LayerOverviewUtils extends Script {
|
|||
writeFileSync(
|
||||
`${LayerOverviewUtils.layerPath}${layer.id}.json`,
|
||||
JSON.stringify(layer, null, " "),
|
||||
{ encoding: "utf8" }
|
||||
{ encoding: "utf8" },
|
||||
)
|
||||
}
|
||||
|
||||
getSharedTagRenderings(
|
||||
doesImageExist: DoesImageExist,
|
||||
bootstrapTagRenderings: Map<string, QuestionableTagRenderingConfigJson> = null
|
||||
bootstrapTagRenderings: Map<string, QuestionableTagRenderingConfigJson> = null,
|
||||
): Map<string, QuestionableTagRenderingConfigJson> {
|
||||
const prepareLayer = new PrepareLayer({
|
||||
tagRenderings: bootstrapTagRenderings,
|
||||
|
|
@ -341,7 +337,7 @@ class LayerOverviewUtils extends Script {
|
|||
console.warn(
|
||||
"The SVG at " +
|
||||
path +
|
||||
" contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path"
|
||||
" contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path",
|
||||
)
|
||||
errCount++
|
||||
}
|
||||
|
|
@ -353,6 +349,12 @@ class LayerOverviewUtils extends Script {
|
|||
|
||||
async main(args: string[]) {
|
||||
console.log("Generating layer overview...")
|
||||
const themeWhitelist = new Set(args.find(a => a.startsWith("--themes="))
|
||||
?.substring("--themes=".length)?.split(",") ?? [])
|
||||
|
||||
const layerWhitelist = new Set(args.find(a => a.startsWith("--layers="))
|
||||
?.substring("--layers=".length)?.split(",") ?? [])
|
||||
|
||||
const start = new Date()
|
||||
const forceReload = args.some((a) => a == "--force")
|
||||
|
||||
|
|
@ -361,7 +363,7 @@ class LayerOverviewUtils extends Script {
|
|||
licensePaths.add(licenses[i].path)
|
||||
}
|
||||
const doesImageExist = new DoesImageExist(licensePaths, existsSync)
|
||||
const sharedLayers = this.buildLayerIndex(doesImageExist, forceReload)
|
||||
const sharedLayers = this.buildLayerIndex(doesImageExist, forceReload, layerWhitelist)
|
||||
|
||||
const priviliged = new Set<string>(Constants.priviliged_layers)
|
||||
sharedLayers.forEach((_, key) => {
|
||||
|
|
@ -379,15 +381,18 @@ class LayerOverviewUtils extends Script {
|
|||
licensePaths,
|
||||
sharedLayers,
|
||||
recompiledThemes,
|
||||
forceReload
|
||||
forceReload,
|
||||
themeWhitelist
|
||||
)
|
||||
|
||||
if (recompiledThemes.length > 0){
|
||||
writeFileSync(
|
||||
"./src/assets/generated/known_layers.json",
|
||||
JSON.stringify({
|
||||
layers: Array.from(sharedLayers.values()).filter((l) => l.id !== "favourite"),
|
||||
})
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
const mcChangesPath = "./assets/themes/mapcomplete-changes/mapcomplete-changes.json"
|
||||
if (
|
||||
|
|
@ -406,7 +411,7 @@ class LayerOverviewUtils extends Script {
|
|||
const proto: LayoutConfigJson = JSON.parse(
|
||||
readFileSync("./assets/themes/mapcomplete-changes/mapcomplete-changes.proto.json", {
|
||||
encoding: "utf8",
|
||||
})
|
||||
}),
|
||||
)
|
||||
const protolayer = <LayerConfigJson>(
|
||||
proto.layers.filter((l) => l["id"] === "mapcomplete-changes")[0]
|
||||
|
|
@ -423,21 +428,23 @@ class LayerOverviewUtils extends Script {
|
|||
layers: ScriptUtils.getLayerFiles().map((f) => f.parsed),
|
||||
themes: ScriptUtils.getThemeFiles().map((f) => f.parsed),
|
||||
},
|
||||
ConversionContext.construct([], [])
|
||||
ConversionContext.construct([], []),
|
||||
)
|
||||
|
||||
for (const [_, theme] of sharedThemes) {
|
||||
theme.layers = theme.layers.filter(
|
||||
(l) => Constants.added_by_default.indexOf(l["id"]) < 0
|
||||
(l) => Constants.added_by_default.indexOf(l["id"]) < 0,
|
||||
)
|
||||
}
|
||||
|
||||
if(recompiledThemes.length > 0) {
|
||||
writeFileSync(
|
||||
"./src/assets/generated/known_themes.json",
|
||||
JSON.stringify({
|
||||
themes: Array.from(sharedThemes.values()),
|
||||
})
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
const end = new Date()
|
||||
const millisNeeded = end.getTime() - start.getTime()
|
||||
|
|
@ -445,7 +452,7 @@ class LayerOverviewUtils extends Script {
|
|||
console.error(
|
||||
"This was a bootstrapping-run. Run generate layeroverview again!(" +
|
||||
millisNeeded +
|
||||
" ms)"
|
||||
" ms)",
|
||||
)
|
||||
} else {
|
||||
const green = (s) => "\x1b[92m" + s + "\x1b[0m"
|
||||
|
|
@ -456,7 +463,7 @@ class LayerOverviewUtils extends Script {
|
|||
private parseLayer(
|
||||
doesImageExist: DoesImageExist,
|
||||
prepLayer: PrepareLayer,
|
||||
sharedLayerPath: string
|
||||
sharedLayerPath: string,
|
||||
): {
|
||||
raw: LayerConfigJson
|
||||
parsed: LayerConfig
|
||||
|
|
@ -467,14 +474,15 @@ class LayerOverviewUtils extends Script {
|
|||
const parsed = parser.convertStrict(sharedLayerPath, context)
|
||||
const result = AddIconSummary.singleton.convertStrict(
|
||||
parsed,
|
||||
context.inOperation("AddIconSummary")
|
||||
context.inOperation("AddIconSummary"),
|
||||
)
|
||||
return { ...result, context }
|
||||
}
|
||||
|
||||
private buildLayerIndex(
|
||||
doesImageExist: DoesImageExist,
|
||||
forceReload: boolean
|
||||
forceReload: boolean,
|
||||
whitelist: Set<string>
|
||||
): Map<string, LayerConfigJson> {
|
||||
// First, we expand and validate all builtin layers. These are written to src/assets/generated/layers
|
||||
// At the same time, an index of available layers is built.
|
||||
|
|
@ -492,6 +500,12 @@ class LayerOverviewUtils extends Script {
|
|||
const recompiledLayers: string[] = []
|
||||
let warningCount = 0
|
||||
for (const sharedLayerPath of ScriptUtils.getLayerPaths()) {
|
||||
if(whitelist.size > 0){
|
||||
const idByPath = sharedLayerPath.split("/").at(-1).split(".")[0]
|
||||
if(Constants.priviliged_layers.indexOf(<any> idByPath) < 0 && !whitelist.has(idByPath)){
|
||||
continue
|
||||
}
|
||||
}
|
||||
{
|
||||
const targetPath =
|
||||
LayerOverviewUtils.layerPath +
|
||||
|
|
@ -525,12 +539,12 @@ class LayerOverviewUtils extends Script {
|
|||
skippedLayers.length +
|
||||
" layers. Detected " +
|
||||
warningCount +
|
||||
" warnings"
|
||||
" warnings",
|
||||
)
|
||||
// We always need the calculated tags of 'usersettings', so we export them separately
|
||||
this.extractJavascriptCodeForLayer(
|
||||
state.sharedLayers.get("usersettings"),
|
||||
"./src/Logic/State/UserSettingsMetaTagging.ts"
|
||||
"./src/Logic/State/UserSettingsMetaTagging.ts",
|
||||
)
|
||||
|
||||
return sharedLayers
|
||||
|
|
@ -547,8 +561,8 @@ class LayerOverviewUtils extends Script {
|
|||
private extractJavascriptCode(themeFile: LayoutConfigJson) {
|
||||
const allCode = [
|
||||
"import {Feature} from 'geojson'",
|
||||
'import { ExtraFuncType } from "../../../Logic/ExtraFunctions";',
|
||||
'import { Utils } from "../../../Utils"',
|
||||
"import { ExtraFuncType } from \"../../../Logic/ExtraFunctions\";",
|
||||
"import { Utils } from \"../../../Utils\"",
|
||||
"export class ThemeMetaTagging {",
|
||||
" public static readonly themeName = " + JSON.stringify(themeFile.id),
|
||||
"",
|
||||
|
|
@ -561,7 +575,7 @@ class LayerOverviewUtils extends Script {
|
|||
allCode.push(
|
||||
" public metaTaggging_for_" +
|
||||
id +
|
||||
"(feat: Feature, helperFunctions: Record<ExtraFuncType, (feature: Feature) => Function>) {"
|
||||
"(feat: Feature, helperFunctions: Record<ExtraFuncType, (feature: Feature) => Function>) {",
|
||||
)
|
||||
allCode.push(" const {" + ExtraFunctions.types.join(", ") + "} = helperFunctions")
|
||||
for (const line of code) {
|
||||
|
|
@ -575,7 +589,7 @@ class LayerOverviewUtils extends Script {
|
|||
attributeName +
|
||||
"', () => " +
|
||||
expression +
|
||||
" ) "
|
||||
" ) ",
|
||||
)
|
||||
} else {
|
||||
attributeName = attributeName.substring(0, attributeName.length - 1).trim()
|
||||
|
|
@ -620,7 +634,7 @@ class LayerOverviewUtils extends Script {
|
|||
const code = l.calculatedTags ?? []
|
||||
|
||||
allCode.push(
|
||||
" public metaTaggging_for_" + l.id + "(feat: {properties: Record<string, string>}) {"
|
||||
" public metaTaggging_for_" + l.id + "(feat: {properties: Record<string, string>}) {",
|
||||
)
|
||||
for (const line of code) {
|
||||
const firstEq = line.indexOf("=")
|
||||
|
|
@ -633,7 +647,7 @@ class LayerOverviewUtils extends Script {
|
|||
attributeName +
|
||||
"', () => " +
|
||||
expression +
|
||||
" ) "
|
||||
" ) ",
|
||||
)
|
||||
} else {
|
||||
attributeName = attributeName.substring(0, attributeName.length - 2).trim()
|
||||
|
|
@ -657,20 +671,21 @@ class LayerOverviewUtils extends Script {
|
|||
licensePaths: Set<string>,
|
||||
sharedLayers: Map<string, LayerConfigJson>,
|
||||
recompiledThemes: string[],
|
||||
forceReload: boolean
|
||||
forceReload: boolean,
|
||||
whitelist: Set<string>
|
||||
): Map<string, LayoutConfigJson> {
|
||||
console.log(" ---------- VALIDATING BUILTIN THEMES ---------")
|
||||
const themeFiles = ScriptUtils.getThemeFiles()
|
||||
const fixed = new Map<string, LayoutConfigJson>()
|
||||
|
||||
const publicLayers = LayerOverviewUtils.publicLayerIdsFrom(
|
||||
themeFiles.map((th) => th.parsed)
|
||||
themeFiles.map((th) => th.parsed),
|
||||
)
|
||||
|
||||
const convertState: DesugaringContext = {
|
||||
sharedLayers,
|
||||
tagRenderings: this.getSharedTagRenderings(
|
||||
new DoesImageExist(licensePaths, existsSync)
|
||||
new DoesImageExist(licensePaths, existsSync),
|
||||
),
|
||||
publicLayers,
|
||||
}
|
||||
|
|
@ -695,20 +710,23 @@ class LayerOverviewUtils extends Script {
|
|||
const themeInfo = themeFiles[i]
|
||||
const themePath = themeInfo.path
|
||||
let themeFile = themeInfo.parsed
|
||||
if(whitelist.size > 0 && !whitelist.has(themeFile.id)){
|
||||
continue
|
||||
}
|
||||
|
||||
const targetPath =
|
||||
LayerOverviewUtils.themePath + "/" + themePath.substring(themePath.lastIndexOf("/"))
|
||||
|
||||
const usedLayers = Array.from(
|
||||
LayerOverviewUtils.extractLayerIdsFrom(themeFile, false)
|
||||
LayerOverviewUtils.extractLayerIdsFrom(themeFile, false),
|
||||
).map((id) => LayerOverviewUtils.layerPath + id + ".json")
|
||||
|
||||
if (!forceReload && !this.shouldBeUpdated([themePath, ...usedLayers], targetPath)) {
|
||||
fixed.set(
|
||||
themeFile.id,
|
||||
JSON.parse(
|
||||
readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", "utf8")
|
||||
)
|
||||
readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", "utf8"),
|
||||
),
|
||||
)
|
||||
ScriptUtils.erasableLog("Skipping", themeFile.id)
|
||||
skippedThemes.push(themeFile.id)
|
||||
|
|
@ -719,23 +737,23 @@ class LayerOverviewUtils extends Script {
|
|||
|
||||
new PrevalidateTheme().convertStrict(
|
||||
themeFile,
|
||||
ConversionContext.construct([themePath], ["PrepareLayer"])
|
||||
ConversionContext.construct([themePath], ["PrepareLayer"]),
|
||||
)
|
||||
try {
|
||||
themeFile = new PrepareTheme(convertState, {
|
||||
skipDefaultLayers: true,
|
||||
}).convertStrict(
|
||||
themeFile,
|
||||
ConversionContext.construct([themePath], ["PrepareLayer"])
|
||||
ConversionContext.construct([themePath], ["PrepareLayer"]),
|
||||
)
|
||||
new ValidateThemeAndLayers(
|
||||
new DoesImageExist(licensePaths, existsSync, knownTagRenderings),
|
||||
themePath,
|
||||
true,
|
||||
knownTagRenderings
|
||||
knownTagRenderings,
|
||||
).convertStrict(
|
||||
themeFile,
|
||||
ConversionContext.construct([themePath], ["PrepareLayer"])
|
||||
ConversionContext.construct([themePath], ["PrepareLayer"]),
|
||||
)
|
||||
|
||||
if (themeFile.icon.endsWith(".svg")) {
|
||||
|
|
@ -777,6 +795,7 @@ class LayerOverviewUtils extends Script {
|
|||
}
|
||||
}
|
||||
|
||||
if(whitelist.size == 0){
|
||||
this.writeSmallOverview(
|
||||
Array.from(fixed.values()).map((t) => {
|
||||
return {
|
||||
|
|
@ -789,15 +808,16 @@ class LayerOverviewUtils extends Script {
|
|||
.OnEveryLanguage((s) => parse_html(s).textContent).translations,
|
||||
mustHaveLanguage: t.mustHaveLanguage?.length > 0,
|
||||
}
|
||||
})
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
console.log(
|
||||
"Recompiled themes " +
|
||||
recompiledThemes.join(", ") +
|
||||
" and skipped " +
|
||||
skippedThemes.length +
|
||||
" themes"
|
||||
" themes",
|
||||
)
|
||||
|
||||
return fixed
|
||||
|
|
|
|||
Loading…
Reference in a new issue