Scripts: generateLayerOverview can now drop tagRenderings/layers/themes based on labels (preparation for play store censoring)

scripts/generateLayerOverview.ts

@@ -13,7 +13,14 @@ import {
 import { Translation } from "../src/UI/i18n/Translation"
 import { OrderLayer, PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
 import { OrderTheme, PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme"
-import { Conversion, DesugaringContext, DesugaringStep } from "../src/Models/ThemeConfig/Conversion/Conversion"
+import {
+    Conversion,
+    DesugaringContext,
+    DesugaringStep,
+    Each,
+    Fuse,
+    On,
+} from "../src/Models/ThemeConfig/Conversion/Conversion"
 import { Utils } from "../src/Utils"
 import Script from "./Script"
 import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
@@ -142,18 +149,21 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
     private readonly _loadedIds: Set<string> = new Set<string>()
     private readonly _layerConfigJsons = new Map<string, LayerConfigJson>()
     private readonly _desugaringState: DesugaringContext
+    private readonly _labelBlacklist: ReadonlySet<string>
 
     constructor(
         layerConfigJsons: LayerConfigJson[],
         dependencies: Map<string, string[]>,
         levels: LevelInfo[],
         states: Map<string, "clean" | "dirty" | "changed">,
-        sharedTagRenderings: QuestionableTagRenderingConfigJson[]
+        sharedTagRenderings: QuestionableTagRenderingConfigJson[],
+        labelBlacklist: ReadonlySet<string>,
     ) {
         super("LayerBuilder", "Builds all the layers, writes them to file")
         this._levels = levels
         this._dependencies = dependencies
         this._states = states
+        this._labelBlacklist = labelBlacklist
         this._desugaringState = {
             tagRenderings: LayerOverviewUtils.asDict(sharedTagRenderings),
             tagRenderingOrder: sharedTagRenderings.map((tr) => tr.id),
@@ -174,6 +184,11 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
     }
 
     writeLayer(layer: LayerConfigJson) {
+        if (layer.labels?.some(l => this._labelBlacklist.has(l))) {
+            console.log("Not writing layer " + layer.id + ", censored")
+            return
+        }
+        layer = new CensorLayer(this._labelBlacklist).convertStrict(layer)
         if (!existsSync(LayerOverviewUtils.layerPath)) {
             mkdirSync(LayerOverviewUtils.layerPath)
         }
@@ -340,12 +355,63 @@ class ReorderFiles extends Script {
     }
 }
 
+class CensorLayer extends DesugaringStep<LayerConfigJson> {
+    private readonly _excludedLabels: ReadonlySet<string>
+
+    constructor(excludedLabels: ReadonlySet<string>) {
+        super("CensorLayer", "Removes unwanted layers for specific builds (mostly play store)")
+        this._excludedLabels = excludedLabels
+    }
+
+    convert(json: LayerConfigJson, context: ConversionContext): LayerConfigJson {
+        json = { ...json }
+        json.tagRenderings = json.tagRenderings?.filter(trs => {
+            const tr = <QuestionableTagRenderingConfigJson>trs
+            const keep = !(tr.labels ?? [])?.some(l => this._excludedLabels.has(l))
+            if (!keep) {
+                const forbidden = (tr.labels ?? [])?.filter(l => this._excludedLabels.has(l))
+                context.info("Dropping tagRendering " + tr.id + " from layer " + json.id + " due to forbidden label: " + forbidden.join(", "))
+            }
+            return keep
+        })
+        return json
+    }
+}
+
+class CensorTheme extends Fuse<ThemeConfigJson & {layers: LayerConfigJson[]}> {
+    private readonly _excludedLabels: ReadonlySet<string>
+
+    constructor(excludedLabels: ReadonlySet<string>) {
+        super("Removes unwanted layers for specific builds (mostly play store)",
+            new On("layers", new Each(
+                new CensorLayer(excludedLabels),
+            )),
+        )
+        this._excludedLabels = excludedLabels
+    }
+
+    convert(json: ThemeConfigJson & {layers: LayerConfigJson[]}, context: ConversionContext): ThemeConfigJson & {layers: LayerConfigJson[]} {
+        json = { ...json }
+        const newLayers: LayerConfigJson[] = []
+        for (const layer of <LayerConfigJson[]>json.layers) {
+            if (layer.labels?.some(label => this._excludedLabels.has(label))) {
+                context.info("Dropping layer " + layer.id + " from theme " + json.id + " due to forbidden label: " + layer.labels?.filter(l => this._excludedLabels.has(l)).join(", "))
+                continue
+            }
+            newLayers.push(layer)
+        }
+        json.layers = newLayers
+        super.convert(json, context)
+        return json
+    }
+}
+
 class LayerOverviewUtils extends Script {
     public static readonly layerPath = "./public/assets/generated/layers/"
     public static readonly themePath = "./public/assets/generated/themes/"
 
     constructor() {
-        super("Reviews and generates the compiled themes")
+        super("Reviews and generates the compiled themes. Arguments: '[--exclude-labels=label0,label1] --themes=theme0,theme1'")
     }
 
     private static publicLayerIdsFrom(themefiles: ThemeConfigJson[]): Set<string> {
@@ -402,6 +468,9 @@ class LayerOverviewUtils extends Script {
         }
 
         for (const path of sourcefile) {
+            if (!existsSync(path)) {
+                return true
+            }
             const hasChange = statSync(path).mtime > targetModified
             if (hasChange) {
                 return true
@@ -670,11 +739,16 @@ class LayerOverviewUtils extends Script {
                 ?.substring("--themes=".length)
                 ?.split(",") ?? []
         )
+        const labelBlacklist = new Set(args.find(a => a.startsWith("--exclude-labels="))
+            ?.substring("--exclude-labels=".length)
+            ?.split(",") ?? [])
 
-        const forceReload = args.some((a) => a == "--force")
 
+        const forceReload = args.some((a) => a == "--force") || labelBlacklist.size > 0
+
+        console.log("Arguments are:",{ labelBlacklist, themeWhitelist, forceReload })
         const doesImageExist = DoesImageExist.constructWithLicenses(existsSync)
-        const sharedLayers = this.buildLayerIndex(doesImageExist)
+        const sharedLayers = this.buildLayerIndex(doesImageExist, labelBlacklist)
 
         const priviliged = new Set<string>(Constants.priviliged_layers)
         sharedLayers.forEach((_, key) => {
@@ -699,7 +773,8 @@ class LayerOverviewUtils extends Script {
             sharedLayers,
             recompiledThemes,
             forceReload,
-            themeWhitelist
+            themeWhitelist,
+            labelBlacklist,
         )
 
         new ValidateThemeEnsemble().convertStrict(
@@ -810,7 +885,7 @@ class LayerOverviewUtils extends Script {
         return results
     }
 
-    private buildLayerIndex(doesImageExist: DoesImageExist): Map<string, LayerConfigJson> {
+    private buildLayerIndex(doesImageExist: DoesImageExist, labelBlacklist: Set<string>): Map<string, LayerConfigJson> {
         // First, we expand and validate all builtin layers. These are written to src/assets/generated/layers
         // At the same time, an index of available layers is built.
         const sharedQuestions = this.getSharedTagRenderings(doesImageExist)
@@ -870,7 +945,8 @@ class LayerOverviewUtils extends Script {
             dependencyGraph,
             levels,
             layerState,
-            sharedQuestions
+            sharedQuestions,
+            labelBlacklist,
         )
         builder.writeLayer(sharedQuestionsDef)
         const allLayers = builder.convertStrict({}, ConversionContext.construct([], ["Building the layer index"]))
@@ -1011,7 +1087,8 @@ class LayerOverviewUtils extends Script {
         sharedLayers: Map<string, LayerConfigJson>,
         recompiledThemes: string[],
         forceReload: boolean,
-        whitelist: Set<string>
+        whitelist: ReadonlySet<string>,
+        labelBlacklist: ReadonlySet<string>,
     ): Map<string, ThemeConfigJson> {
         console.log("   ---------- VALIDATING BUILTIN THEMES ---------")
         const themeFiles = ScriptUtils.getThemeFiles()
@@ -1045,7 +1122,7 @@ class LayerOverviewUtils extends Script {
         })
 
         const skippedThemes: string[] = []
-
+        const censorTheme = new CensorTheme(labelBlacklist)
         for (let i = 0; i < themeFiles.length; i++) {
             const themeInfo = themeFiles[i]
             const themePath = themeInfo.path
@@ -1057,6 +1134,11 @@ class LayerOverviewUtils extends Script {
                 continue
             }
 
+             if (themeFile.labels?.some(l => labelBlacklist.has(l))) {
+                console.log("Skipping theme due to label", themeFile.id)
+                continue
+            }
+
             const targetPath =
                 LayerOverviewUtils.themePath + "/" + themePath.substring(themePath.lastIndexOf("/"))
 
@@ -1068,14 +1150,17 @@ class LayerOverviewUtils extends Script {
                 !forceReload &&
                 !LayerOverviewUtils.shouldBeUpdated([themePath, ...usedLayers], targetPath)
             ) {
+                const parsed = <ThemeConfigJson>JSON.parse(
+                    readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", "utf8"),
+                )
+
+                skippedThemes.push(themeFile.id)
+
+                ScriptUtils.erasableLog("Skipping", themeFile.id)
                 fixed.set(
                     themeFile.id,
-                    JSON.parse(
+                    parsed,
-                        readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", "utf8")
                 )
-                )
-                ScriptUtils.erasableLog("Skipping", themeFile.id)
-                skippedThemes.push(themeFile.id)
                 continue
             }
 
@@ -1101,6 +1186,12 @@ class LayerOverviewUtils extends Script {
                     themeFile,
                     ConversionContext.construct([themePath], ["PrepareLayer"])
                 )
+                if(themeFile.labels?.some(l => labelBlacklist.has(l))){
+                    continue
+                }
+
+                themeFile = censorTheme.convertStrict(<any> themeFile,
+                    ConversionContext.construct([themePath], ["Censoring"]))
 
                 if (themeFile.icon.endsWith(".svg")) {
                     try {
@@ -1130,8 +1221,8 @@ class LayerOverviewUtils extends Script {
                                 )
                             }
 
-                            const w = parseInt(width)
+                            const w = Number(width)
-                            const h = parseInt(height)
+                            const h = Number(height)
                             if (w < 370 || h < 370) {
                                 const e: string = [
                                     `the icon for theme ${themeFile.id} is too small. Please rescale the icon at ${themeFile.icon}`,