Osmwithspace/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Scripts: generateLayerOverview can now drop tagRenderings/layers/themes based on labels (preparation for play store censoring)

Browse source
This commit is contained in:
Pieter Vander Vennet 2025-07-08 03:38:39 +02:00
parent cb0cb710a9
commit e3bd18ba52
1 changed files with 108 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

125
scripts/generateLayerOverview.ts
View file

@ -13,7 +13,14 @@ import {
import { Translation } from "../src/UI/i18n/Translation"
import { OrderLayer, PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
import { OrderTheme, PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme"
import { Conversion, DesugaringContext, DesugaringStep } from "../src/Models/ThemeConfig/Conversion/Conversion"
import {
Conversion,
DesugaringContext,
DesugaringStep,
Each,
Fuse,
On,
} from "../src/Models/ThemeConfig/Conversion/Conversion"
import { Utils } from "../src/Utils"
import Script from "./Script"
import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
@ -142,18 +149,21 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
private readonly _loadedIds: Set<string> = new Set<string>()
private readonly _layerConfigJsons = new Map<string, LayerConfigJson>()
private readonly _desugaringState: DesugaringContext
private readonly _labelBlacklist: ReadonlySet<string>
constructor(
layerConfigJsons: LayerConfigJson[],
dependencies: Map<string, string[]>,
levels: LevelInfo[],
states: Map<string, "clean" | "dirty" | "changed">,
sharedTagRenderings: QuestionableTagRenderingConfigJson[]
sharedTagRenderings: QuestionableTagRenderingConfigJson[],
labelBlacklist: ReadonlySet<string>,
) {
super("LayerBuilder", "Builds all the layers, writes them to file")
this._levels = levels
this._dependencies = dependencies
this._states = states
this._labelBlacklist = labelBlacklist
this._desugaringState = {
tagRenderings: LayerOverviewUtils.asDict(sharedTagRenderings),
tagRenderingOrder: sharedTagRenderings.map((tr) => tr.id),
@ -174,6 +184,11 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
}
writeLayer(layer: LayerConfigJson) {
if (layer.labels?.some(l => this._labelBlacklist.has(l))) {
console.log("Not writing layer " + layer.id + ", censored")
return
}
layer = new CensorLayer(this._labelBlacklist).convertStrict(layer)
if (!existsSync(LayerOverviewUtils.layerPath)) {
mkdirSync(LayerOverviewUtils.layerPath)
}
@ -340,12 +355,63 @@ class ReorderFiles extends Script {
}
}
class CensorLayer extends DesugaringStep<LayerConfigJson> {
private readonly _excludedLabels: ReadonlySet<string>
constructor(excludedLabels: ReadonlySet<string>) {
super("CensorLayer", "Removes unwanted layers for specific builds (mostly play store)")
this._excludedLabels = excludedLabels
}
convert(json: LayerConfigJson, context: ConversionContext): LayerConfigJson {
json = { ...json }
json.tagRenderings = json.tagRenderings?.filter(trs => {
const tr = <QuestionableTagRenderingConfigJson>trs
const keep = !(tr.labels ?? [])?.some(l => this._excludedLabels.has(l))
if (!keep) {
const forbidden = (tr.labels ?? [])?.filter(l => this._excludedLabels.has(l))
context.info("Dropping tagRendering " + tr.id + " from layer " + json.id + " due to forbidden label: " + forbidden.join(", "))
}
return keep
})
return json
}
}
class CensorTheme extends Fuse<ThemeConfigJson & {layers: LayerConfigJson[]}> {
private readonly _excludedLabels: ReadonlySet<string>
constructor(excludedLabels: ReadonlySet<string>) {
super("Removes unwanted layers for specific builds (mostly play store)",
new On("layers", new Each(
new CensorLayer(excludedLabels),
)),
)
this._excludedLabels = excludedLabels
}
convert(json: ThemeConfigJson & {layers: LayerConfigJson[]}, context: ConversionContext): ThemeConfigJson & {layers: LayerConfigJson[]} {
json = { ...json }
const newLayers: LayerConfigJson[] = []
for (const layer of <LayerConfigJson[]>json.layers) {
if (layer.labels?.some(label => this._excludedLabels.has(label))) {
context.info("Dropping layer " + layer.id + " from theme " + json.id + " due to forbidden label: " + layer.labels?.filter(l => this._excludedLabels.has(l)).join(", "))
continue
}
newLayers.push(layer)
}
json.layers = newLayers
super.convert(json, context)
return json
}
}
class LayerOverviewUtils extends Script {
public static readonly layerPath = "./public/assets/generated/layers/"
public static readonly themePath = "./public/assets/generated/themes/"
constructor() {
super("Reviews and generates the compiled themes")
super("Reviews and generates the compiled themes. Arguments: '[--exclude-labels=label0,label1] --themes=theme0,theme1'")
}
private static publicLayerIdsFrom(themefiles: ThemeConfigJson[]): Set<string> {
@ -402,6 +468,9 @@ class LayerOverviewUtils extends Script {
}
for (const path of sourcefile) {
if (!existsSync(path)) {
return true
}
const hasChange = statSync(path).mtime > targetModified
if (hasChange) {
return true
@ -670,11 +739,16 @@ class LayerOverviewUtils extends Script {
?.substring("--themes=".length)
?.split(",") ?? []
)
const labelBlacklist = new Set(args.find(a => a.startsWith("--exclude-labels="))
?.substring("--exclude-labels=".length)
?.split(",") ?? [])
const forceReload = args.some((a) => a == "--force")
const forceReload = args.some((a) => a == "--force") || labelBlacklist.size > 0
console.log("Arguments are:",{ labelBlacklist, themeWhitelist, forceReload })
const doesImageExist = DoesImageExist.constructWithLicenses(existsSync)
const sharedLayers = this.buildLayerIndex(doesImageExist)
const sharedLayers = this.buildLayerIndex(doesImageExist, labelBlacklist)
const priviliged = new Set<string>(Constants.priviliged_layers)
sharedLayers.forEach((_, key) => {
@ -699,7 +773,8 @@ class LayerOverviewUtils extends Script {
sharedLayers,
recompiledThemes,
forceReload,
themeWhitelist
themeWhitelist,
labelBlacklist,
)
new ValidateThemeEnsemble().convertStrict(
@ -810,7 +885,7 @@ class LayerOverviewUtils extends Script {
return results
}
private buildLayerIndex(doesImageExist: DoesImageExist): Map<string, LayerConfigJson> {
private buildLayerIndex(doesImageExist: DoesImageExist, labelBlacklist: Set<string>): Map<string, LayerConfigJson> {
// First, we expand and validate all builtin layers. These are written to src/assets/generated/layers
// At the same time, an index of available layers is built.
const sharedQuestions = this.getSharedTagRenderings(doesImageExist)
@ -870,7 +945,8 @@ class LayerOverviewUtils extends Script {
dependencyGraph,
levels,
layerState,
sharedQuestions
sharedQuestions,
labelBlacklist,
)
builder.writeLayer(sharedQuestionsDef)
const allLayers = builder.convertStrict({}, ConversionContext.construct([], ["Building the layer index"]))
@ -1011,7 +1087,8 @@ class LayerOverviewUtils extends Script {
sharedLayers: Map<string, LayerConfigJson>,
recompiledThemes: string[],
forceReload: boolean,
whitelist: Set<string>
whitelist: ReadonlySet<string>,
labelBlacklist: ReadonlySet<string>,
): Map<string, ThemeConfigJson> {
console.log(" ---------- VALIDATING BUILTIN THEMES ---------")
const themeFiles = ScriptUtils.getThemeFiles()
@ -1045,7 +1122,7 @@ class LayerOverviewUtils extends Script {
})
const skippedThemes: string[] = []
const censorTheme = new CensorTheme(labelBlacklist)
for (let i = 0; i < themeFiles.length; i++) {
const themeInfo = themeFiles[i]
const themePath = themeInfo.path
@ -1057,6 +1134,11 @@ class LayerOverviewUtils extends Script {
continue
}
if (themeFile.labels?.some(l => labelBlacklist.has(l))) {
console.log("Skipping theme due to label", themeFile.id)
continue
}
const targetPath =
LayerOverviewUtils.themePath + "/" + themePath.substring(themePath.lastIndexOf("/"))
@ -1068,14 +1150,17 @@ class LayerOverviewUtils extends Script {
!forceReload &&
!LayerOverviewUtils.shouldBeUpdated([themePath, ...usedLayers], targetPath)
) {
const parsed = <ThemeConfigJson>JSON.parse(
readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", "utf8"),
)
skippedThemes.push(themeFile.id)
ScriptUtils.erasableLog("Skipping", themeFile.id)
fixed.set(
themeFile.id,
JSON.parse(
readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", "utf8")
)
parsed,
)
ScriptUtils.erasableLog("Skipping", themeFile.id)
skippedThemes.push(themeFile.id)
continue
}
@ -1101,6 +1186,12 @@ class LayerOverviewUtils extends Script {
themeFile,
ConversionContext.construct([themePath], ["PrepareLayer"])
)
if(themeFile.labels?.some(l => labelBlacklist.has(l))){
continue
}
themeFile = censorTheme.convertStrict(<any> themeFile,
ConversionContext.construct([themePath], ["Censoring"]))
if (themeFile.icon.endsWith(".svg")) {
try {
@ -1130,8 +1221,8 @@ class LayerOverviewUtils extends Script {
)
}
const w = parseInt(width)
const h = parseInt(height)
const w = Number(width)
const h = Number(height)
if (w < 370 || h < 370) {
const e: string = [
`the icon for theme ${themeFile.id} is too small. Please rescale the icon at ${themeFile.icon}`,