Feature: layer validation system now builds a dependency graph and only updates what is needed, makes "refresh:layeroverview" redundant

This commit is contained in:
Pieter Vander Vennet 2025-04-21 23:53:10 +02:00
parent 1bd060df82
commit fda0bc6b2e
19 changed files with 301 additions and 186 deletions

View file

@ -6,13 +6,14 @@ import { AllKnownLayoutsLazy } from "../src/Customizations/AllKnownLayouts"
import { Utils } from "../src/Utils"
import {
MappingConfigJson,
QuestionableTagRenderingConfigJson,
QuestionableTagRenderingConfigJson
} from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import { TagConfigJson } from "../src/Models/ThemeConfig/Json/TagConfigJson"
import { TagUtils } from "../src/Logic/Tags/TagUtils"
import { TagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/TagRenderingConfigJson"
import { Translatable } from "../src/Models/ThemeConfig/Json/Translatable"
import * as questions from "../assets/layers/questions/questions.json"
export class GenerateFavouritesLayer extends Script {
private readonly layers: LayerConfigJson[] = []
@ -202,7 +203,7 @@ export class GenerateFavouritesLayer extends Script {
string,
TagRenderingConfigJson[]
>()
const path = "./src/assets/generated/layers/icons.json"
const path = "./public/assets/generated/layers/icons.json"
if (existsSync(path)) {
const config = <LayerConfigJson>JSON.parse(readFileSync(path, "utf8"))
for (const tagRendering of config.tagRenderings) {

View file

@ -9,16 +9,12 @@ import {
DoesImageExist,
PrevalidateTheme,
ValidateLayer,
ValidateThemeEnsemble,
ValidateThemeEnsemble
} from "../src/Models/ThemeConfig/Conversion/Validation"
import { Translation } from "../src/UI/i18n/Translation"
import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
import { PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme"
import {
Conversion,
DesugaringContext,
DesugaringStep,
} from "../src/Models/ThemeConfig/Conversion/Conversion"
import { Conversion, DesugaringContext, DesugaringStep } from "../src/Models/ThemeConfig/Conversion/Conversion"
import { Utils } from "../src/Utils"
import Script from "./Script"
import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
@ -35,6 +31,7 @@ import { Translatable } from "../src/Models/ThemeConfig/Json/Translatable"
import { ValidateThemeAndLayers } from "../src/Models/ThemeConfig/Conversion/ValidateThemeAndLayers"
import { ExtractImages } from "../src/Models/ThemeConfig/Conversion/FixImages"
import { TagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/TagRenderingConfigJson"
import { LayerConfigDependencyGraph, LevelInfo } from "../src/Models/ThemeConfig/LayerConfigDependencyGraph"
// This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files.
// It spits out an overview of those to be used to load them
@ -138,8 +135,171 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye
}
}
class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
private readonly _dependencies: ReadonlyMap<string, string[]>
private readonly _states: Map<string, "clean" | "dirty" | "changed">
private readonly prepareLayer: PrepareLayer
private readonly _levels: LevelInfo[]
private readonly _loadedIds: Set<string> = new Set<string>()
private readonly _layerConfigJsons = new Map<string, LayerConfigJson>
private readonly _desugaringState: DesugaringContext
constructor(
layerConfigJsons: LayerConfigJson[],
dependencies: Map<string, string[]>,
levels: LevelInfo[],
states: Map<string, "clean" | "dirty" | "changed">,
sharedTagRenderings: QuestionableTagRenderingConfigJson[]) {
super("Builds all the layers, writes them to file", [], "LayerBuilder")
this._levels = levels
this._dependencies = dependencies
this._states = states
this._desugaringState = {
tagRenderings: LayerOverviewUtils.asDict(sharedTagRenderings),
tagRenderingOrder: sharedTagRenderings.map((tr) => tr.id),
sharedLayers: AllSharedLayers.getSharedLayersConfigs()
}
this.prepareLayer = new PrepareLayer(this._desugaringState)
for (const layerConfigJson of layerConfigJsons) {
this._layerConfigJsons.set(layerConfigJson.id, layerConfigJson)
}
}
public static targetPath(id: string): string {
return `${LayerOverviewUtils.layerPath}${id}.json`
}
public static sourcePath(id: string): string {
return `./assets/layers/${id}/${id}.json`
}
writeLayer(layer: LayerConfigJson) {
if (!existsSync(LayerOverviewUtils.layerPath)) {
mkdirSync(LayerOverviewUtils.layerPath)
}
writeFileSync(
LayerBuilder.targetPath(layer.id),
JSON.stringify(layer, null, " "),
{ encoding: "utf8" }
)
}
public buildLayer(id: string, context: ConversionContext, isLooping: boolean = false): LayerConfigJson {
if (id === "questions") {
return undefined
}
const deps = this._dependencies.get(id)
if (!isLooping) {
// Beware of the looping traps. Bring the leaf to the statue to teleport to "The Lab" (<ref>submachine 4</ref>)
const unbuilt = deps.filter(depId => !this._loadedIds.has(depId))
for (const unbuiltId of unbuilt) {
this.buildLayer(unbuiltId, context)
}
}
context = context.inOperation("building Layer " + id).enters("layer", id)
const config = this._layerConfigJsons.get(id)
const prepped = this.prepareLayer.convert(config, context)
this._loadedIds.add(id)
this._desugaringState.sharedLayers.set(id, prepped)
return prepped
}
private buildLooping(ids: string[], context: ConversionContext) {
const origIds: ReadonlyArray<string> = [...ids]
const deps = this._dependencies
const allDeps = Utils.Dedup([].concat(...ids.map(id => deps.get(id))))
const depsRecord = Utils.asRecord(Array.from(deps.keys()), k =>
deps.get(k).filter(dep => ids.indexOf(dep) >= 0))
const revDeps = Utils.TransposeMap(depsRecord)
for (const someDep of allDeps) {
if (ids.indexOf(someDep) >= 0) {
// BY definition, we _will_ need this dependency
// We add a small stub
this._desugaringState.sharedLayers.set(someDep, {
id: someDep,
pointRendering: [],
tagRenderings: [],
filter: [],
source: "special:stub",
allowMove: true
})
continue
}
// Make sure all are direct dependencies are loaded
if (!this._loadedIds.has(someDep)) {
this.buildLayer(someDep, context)
}
}
while (ids.length > 0) {
const first = ids.pop()
if (first === "questions") {
continue
}
const oldConfig = this._desugaringState.sharedLayers.get(first) ?? this._layerConfigJsons.get(first)
const newConfig = this.buildLayer(first, context.inOperation("resolving a looped dependency"), true)
const isDifferent = JSON.stringify(oldConfig) !== JSON.stringify(newConfig)
if (isDifferent) {
const toRunAgain = revDeps[first] ?? []
for (const id of toRunAgain) {
if (ids.indexOf(id) < 0) {
ids.push(id)
}
}
}
}
for (const id of origIds) {
this.writeLayer(this._desugaringState.sharedLayers.get(id))
}
console.log("Done with the looping layers!")
}
public convert(o, context: ConversionContext):
Map<string, LayerConfigJson> {
for (const level of this._levels
) {
if (level.loop) {
this.buildLooping(level.ids, context)
continue
}
for (let i = 0; i < level.ids.length; i++) {
const id = level.ids[i]
ScriptUtils.erasableLog(`Building level ${level.level}: validating layer ${i + 1}/${level.ids.length}: ${id}`)
if (id === "questions") {
continue
}
if (this._states.get(id) === "clean") {
const file = readFileSync(LayerBuilder.targetPath(id), "utf-8")
if (file.length > 3) {
try {
const loaded = JSON.parse(file)
this._desugaringState.sharedLayers.set(id, loaded)
continue
} catch (e) {
console.error("Could not load generated layer file for ", id, " building it instead")
}
}
}
const prepped = this.buildLayer(id, context)
this.writeLayer(prepped)
LayerOverviewUtils.extractJavascriptCodeForLayer(prepped)
}
}
context.info("Recompiled " + this._loadedIds.size + " layers")
return this._desugaringState.sharedLayers
}
}
class LayerOverviewUtils extends Script {
public static readonly layerPath = "./src/assets/generated/layers/"
public static readonly layerPath = "./public/assets/generated/layers/"
public static readonly themePath = "./public/assets/generated/themes/"
constructor() {
@ -190,7 +350,7 @@ class LayerOverviewUtils extends Script {
return Translations.T(t).OnEveryLanguage((s) => parse_html(s).textContent).translations
}
shouldBeUpdated(sourcefile: string | string[], targetfile: string): boolean {
public static shouldBeUpdated(sourcefile: string | string[], targetfile: string): boolean {
if (!existsSync(targetfile)) {
return true
}
@ -359,17 +519,6 @@ class LayerOverviewUtils extends Script {
)
}
writeLayer(layer: LayerConfigJson) {
if (!existsSync(LayerOverviewUtils.layerPath)) {
mkdirSync(LayerOverviewUtils.layerPath)
}
writeFileSync(
`${LayerOverviewUtils.layerPath}${layer.id}.json`,
JSON.stringify(layer, null, " "),
{ encoding: "utf8" }
)
}
static asDict(
trs: QuestionableTagRenderingConfigJson[]
): Map<string, QuestionableTagRenderingConfigJson> {
@ -481,13 +630,6 @@ class LayerOverviewUtils extends Script {
?.split(",") ?? []
)
const layerWhitelist = new Set(
args
.find((a) => a.startsWith("--layers="))
?.substring("--layers=".length)
?.split(",") ?? []
)
const forceReload = args.some((a) => a == "--force")
const licensePaths = new Set<string>()
@ -495,7 +637,7 @@ class LayerOverviewUtils extends Script {
licensePaths.add(licenses[i].path)
}
const doesImageExist = new DoesImageExist(licensePaths, existsSync)
const sharedLayers = this.buildLayerIndex(doesImageExist, forceReload, layerWhitelist)
const sharedLayers = this.buildLayerIndex(doesImageExist)
const priviliged = new Set<string>(Constants.priviliged_layers)
sharedLayers.forEach((_, key) => {
@ -582,9 +724,6 @@ class LayerOverviewUtils extends Script {
)
}
if (AllSharedLayers.getSharedLayersConfigs().size == 0) {
console.error("This was a bootstrapping-run. Run generate layeroverview again!")
}
}
private parseLayer(
@ -606,81 +745,89 @@ class LayerOverviewUtils extends Script {
return { ...result, context }
}
private getAllLayerConfigs(): LayerConfigJson[] {
const allPaths = ScriptUtils.getLayerPaths()
const results: LayerConfigJson[] = []
for (let i = 0; i < allPaths.length; i++) {
const path = allPaths[i]
ScriptUtils.erasableLog(`Parsing layerConfig ${i + 1}/${allPaths.length}: ${path} `)
const data = JSON.parse(readFileSync(path, "utf8"))
results.push(data)
}
return results
}
private buildLayerIndex(
doesImageExist: DoesImageExist,
forceReload: boolean,
whitelist: Set<string>
doesImageExist: DoesImageExist
): Map<string, LayerConfigJson> {
// First, we expand and validate all builtin layers. These are written to src/assets/generated/layers
// At the same time, an index of available layers is built.
console.log("------------- VALIDATING THE BUILTIN QUESTIONS ---------------")
const sharedTagRenderings = this.getSharedTagRenderings(doesImageExist)
console.log(" ---------- VALIDATING BUILTIN LAYERS ---------")
const state: DesugaringContext = {
tagRenderings: LayerOverviewUtils.asDict(sharedTagRenderings),
tagRenderingOrder: sharedTagRenderings.map((tr) => tr.id),
sharedLayers: AllSharedLayers.getSharedLayersConfigs(),
}
const sharedLayers = new Map<string, LayerConfigJson>()
const prepLayer = new PrepareLayer(state)
const skippedLayers: string[] = []
const recompiledLayers: string[] = []
let warningCount = 0
for (const sharedLayerPath of ScriptUtils.getLayerPaths()) {
if (whitelist.size > 0) {
const idByPath = sharedLayerPath.split("/").at(-1).split(".")[0]
if (!Constants.isPriviliged(idByPath) && !whitelist.has(idByPath)) {
continue
}
}
{
const targetPath =
LayerOverviewUtils.layerPath +
sharedLayerPath.substring(sharedLayerPath.lastIndexOf("/"))
if (!forceReload && !this.shouldBeUpdated(sharedLayerPath, targetPath)) {
try {
const sharedLayer = JSON.parse(readFileSync(targetPath, "utf8"))
sharedLayers.set(sharedLayer.id, sharedLayer)
skippedLayers.push(sharedLayer.id)
continue
} catch (e) {
throw "Could not parse " + targetPath + " : " + e
const sharedQuestions = this.getSharedTagRenderings(doesImageExist)
const allLayerConfigs = this.getAllLayerConfigs()
const sharedQuestionsDef = allLayerConfigs.find(l => l.id === "questions")
sharedQuestionsDef.tagRenderings = sharedQuestions
const dependencyGraph = LayerConfigDependencyGraph.buildDirectDependencies(allLayerConfigs)
const levels = LayerConfigDependencyGraph.buildLevels(dependencyGraph)
const layerState = new Map<string, "clean" | "dirty" | "changed">()
console.log("# BUILD PLAN\n\n")
for (const levelInfo of levels) {
console.log(`## LEVEL ${levelInfo.level}${levelInfo.loop ? " (LOOP)" : ""}`)
for (const id of levelInfo.ids) {
const deps = dependencyGraph.get(id) ?? []
const dirtyDeps = deps.filter(dep => {
const depState = layerState.get(dep)
if (levelInfo.loop && depState === undefined) {
const depIsClean =
LayerOverviewUtils.shouldBeUpdated(
LayerBuilder.sourcePath(dep),
LayerBuilder.targetPath(dep))
if (depIsClean) {
return false
}
}
return depState !== "clean"
})
if (dirtyDeps.length > 0) {
layerState.set(id, "dirty")
} else {
const sourcePath = `./assets/layers/${id}/${id}.json`
const targetPath = `./public/assets/generated/layers/${id}.json`
if (id === "questions") {
layerState.set(id, "clean")
} else if (LayerOverviewUtils.shouldBeUpdated(sourcePath, targetPath)) {
layerState.set(id, "changed")
} else {
layerState.set(id, "clean")
}
}
const state = layerState.get(id)
console.log(`- ${id} (${state}; ${dirtyDeps.map(dd => dd + "*").join(", ")})`)
}
const parsed = this.parseLayer(doesImageExist, prepLayer, sharedLayerPath)
warningCount += parsed.context.getAll("warning").length
const fixed = parsed.raw
if (sharedLayers.has(fixed.id)) {
throw "There are multiple layers with the id " + fixed.id + ", " + sharedLayerPath
}
if (parsed.context.hasErrors()) {
throw "Some layers contain errors"
}
sharedLayers.set(fixed.id, fixed)
recompiledLayers.push(fixed.id)
this.writeLayer(fixed)
}
console.log(
"Recompiled layers " +
recompiledLayers.join(", ") +
" and skipped " +
skippedLayers.length +
" layers. Detected " +
warningCount +
" warnings"
)
// We always need the calculated tags of 'usersettings', so we export them separately
this.extractJavascriptCodeForLayer(
state.sharedLayers.get("usersettings"),
const builder = new LayerBuilder(allLayerConfigs, dependencyGraph, levels, layerState, sharedQuestions)
builder.writeLayer(sharedQuestionsDef)
const allLayers = builder.convertStrict({}, ConversionContext.construct([], []))
if (layerState.get("usersettings") !== "clean") {
// We always need the calculated tags of 'usersettings', so we export them separately if dirty
LayerOverviewUtils.extractJavascriptCodeForLayer(
allLayers.get("usersettings"),
"./src/Logic/State/UserSettingsMetaTagging.ts"
)
}
return allLayers
return sharedLayers
}
/**
@ -741,7 +888,7 @@ class LayerOverviewUtils extends Script {
writeFileSync(targetDir + themeFile.id + ".ts", allCode.join("\n"))
}
private extractJavascriptCodeForLayer(l: LayerConfigJson, targetPath?: string) {
public static extractJavascriptCodeForLayer(l: LayerConfigJson, targetPath?: string) {
if (!l) {
return // Probably a bootstrapping run
}
@ -858,7 +1005,7 @@ class LayerOverviewUtils extends Script {
LayerOverviewUtils.extractLayerIdsFrom(themeFile, false)
).map((id) => LayerOverviewUtils.layerPath + id + ".json")
if (!forceReload && !this.shouldBeUpdated([themePath, ...usedLayers], targetPath)) {
if (!forceReload && !LayerOverviewUtils.shouldBeUpdated([themePath, ...usedLayers], targetPath)) {
fixed.set(
themeFile.id,
JSON.parse(

View file

@ -5,12 +5,12 @@
mkdir -p ./src/assets/generated/layers
mkdir -p ./public/assets/generated/themes
echo '{"layers": []}' > ./src/assets/generated/known_layers.json
rm -f ./src/assets/generated/layers/*.json
rm -f ./public/assets/generated/layers/*.json
rm -f ./public/assets/generated/themes/*.json
cp ./assets/layers/usersettings/usersettings.json ./src/assets/generated/layers/usersettings.json
echo '{}' > ./src/assets/generated/layers/favourite.json
echo '{}' > ./src/assets/generated/layers/summary.json
echo '{}' > ./src/assets/generated/layers/last_click.json
echo '{}' > ./src/assets/generated/layers/search.json
echo '[]' > ./src/assets/generated/theme_overview.json
echo '{}' > ./src/assets/generated/layers/geocoded_image.json
echo '{}' > ./public/assets/generated/layers/favourite.json
echo '{}' > ./public/assets/generated/layers/summary.json
echo '{}' > ./public/assets/generated/layers/last_click.json
echo '{}' > ./public/assets/generated/layers/search.json
echo '[]' > ./public/assets/generated/theme_overview.json
echo '{}' > ./public/assets/generated/layers/geocoded_image.json
echo '{}' > ./public/assets/generated/layers/usersettings.json

View file

@ -20,8 +20,6 @@ npm run download:editor-layer-index &&
npm run prep:layeroverview &&
npm run generate && # includes a single "refresh:layeroverview". Resetting the files is unnecessary as they are not in there in the first place
npm run generate:mapcomplete-changes-theme &&
npm run refresh:layeroverview && # a second time to propagate all calls
npm run refresh:layeroverview && # a third time to fix some issues with the favourite layer all calls
npm run generate:layouts
if [ $? -ne 0 ]; then