Add validation scripts

scripts/ScriptUtils.ts

@@ -0,0 +1,20 @@
+import {lstatSync, readdirSync} from "fs";
+
+export default class ScriptUtils {
+    public static readDirRecSync(path): string[] {
+        const result = []
+        for (const entry of readdirSync(path)) {
+            const fullEntry = path + "/" + entry
+            const stats = lstatSync(fullEntry)
+            if (stats.isDirectory()) {
+                // Subdirectory
+                // @ts-ignore
+                result.push(...ScriptUtils.readDirRecSync(fullEntry))
+            } else {
+                result.push(fullEntry)
+            }
+        }
+        return result;
+    }
+
+}

scripts/generateLayerOverview.ts

@@ -0,0 +1,110 @@
+import ScriptUtils from "./ScriptUtils";
+import {Utils} from "../Utils";
+import {lstatSync, readdirSync, readFileSync, writeFileSync} from "fs";
+
+Utils.runningFromConsole = true
+import LayerConfig from "../Customizations/JSON/LayerConfig";
+import {error} from "util";
+import * as licenses from "../assets/generated/license_info.json"
+import SmallLicense from "../Models/smallLicense";
+import LayoutConfig from "../Customizations/JSON/LayoutConfig";
+// This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files.
+// It spits out an overview of those to be used to load them
+
+
+// First, remove the old file. It might be buggy!
+writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
+    "layers": [],
+    "themes": []
+}))
+const layerFiles = ScriptUtils.readDirRecSync("./assets/layers")
+    .filter(path => path.indexOf(".json") > 0)
+    .filter(path => path.indexOf("license_info.json") < 0)
+    .map(path => {
+        try {
+            const parsed = JSON.parse(readFileSync(path, "UTF8"));
+            return parsed
+        } catch (e) {
+            console.error("Could not parse file ", path, "due to ", e)
+        }
+    })
+const themeFiles : any[] = ScriptUtils.readDirRecSync("./assets/themes")
+    .filter(path => path.indexOf(".json") > 0)
+    .filter(path => path.indexOf("license_info.json") < 0)
+    .map(path => {
+        return JSON.parse(readFileSync(path, "UTF8"));
+    })
+writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
+    "layers": layerFiles,
+    "themes": themeFiles
+}))
+
+
+console.log("Discovered ", layerFiles.length, "layers and ", themeFiles.length, "themes\n")
+console.log("   ---------- VALIDATING ---------")
+// ------------- VALIDATION --------------
+const licensePaths = []
+for (const i in licenses) {
+    licensePaths.push(licenses[i].path)
+}
+const knownPaths = new Set<string>(licensePaths)
+
+function validateLayer(layer: LayerConfig, context?:string) : number{
+    let errorCount = 0;
+    const images = Array.from(layer.ExtractImages())
+    const remoteImages = images.filter(img => img.indexOf("http") == 0)
+    for (const remoteImage of remoteImages) {
+        console.error("Found a remote image:", remoteImage, "in layer", layer.id)
+        errorCount++
+    }
+    for (const image of images) {
+        if (!knownPaths.has(image)) {
+            console.error("Image with path", image, "not found or not attributed; it is used in", layer.id, context === undefined ? "" : ` in a layer defined in the theme ${context}`)
+            errorCount++
+        }
+    }
+    return errorCount
+}
+let layerErrorCount = 0
+const knownLayerIds = new Set<string>();
+for (const layerFile of layerFiles) {
+    knownLayerIds.add(layerFile.id)
+    try {
+        const layer = new LayerConfig(layerFile, "test", true)
+       layerErrorCount += validateLayer(layer)
+    } catch (e) {
+        console.error("Layer ", layerFile.id, " is invalid: ", e)
+        layerErrorCount++
+    }
+    if (layerFile.overpassTags !== undefined) {
+        layerErrorCount++
+        console.warn("Layer ", layerFile.id, "still uses the old 'overpassTags'-format. Please use 'source: {osmTags: <tags>}' instead")
+    }
+}
+
+let themeErrorCount = 0
+for (const themeFile of themeFiles) {
+
+    for (const layer of themeFile.layers) {
+        if(typeof layer === "string"){
+            if(!knownLayerIds.has(layer)){
+                console.error("Unknown layer id: ", layer)
+                themeErrorCount++
+            }
+        }
+    }
+    
+    themeFile.layers = themeFile.layers.filter(l => typeof l != "string")
+    
+    try {
+        const layout = new LayoutConfig(themeFile, true, "test")
+        for (const layer of layout.layers) {
+            layerErrorCount += validateLayer(layer, layout.id)
+        }
+    } catch (e) {
+        console.error("Could not parse theme", themeFile["id"], "due to", e)
+        themeErrorCount++;
+    }
+}
+
+console.log("Found ", layerErrorCount, "errors in the layers; "+themeErrorCount+" errors in the themes")

scripts/generateLayouts.ts

@@ -234,7 +234,7 @@ if (!existsSync(generatedDir)) {
 }
 
 const blacklist = ["", "test", ".", "..", "manifest", "index", "land", "preferences", "account", "openstreetmap", "custom"]
-const all = AllKnownLayouts.allSets;
+const all = AllKnownLayouts.allKnownLayouts;
 
 for (const layoutName in all) {
     if (blacklist.indexOf(layoutName.toLowerCase()) >= 0) {

scripts/generateLicenseInfo.ts

@@ -1,6 +1,7 @@
 import {Utils} from "../Utils";
 import {lstatSync, readdirSync, readFileSync, writeFileSync} from "fs";
 import SmallLicense from "../Models/smallLicense";
+import ScriptUtils from "./ScriptUtils";
 
 Utils.runningFromConsole = true;
 
@@ -9,22 +10,6 @@ Utils.runningFromConsole = true;
  * Checks that the license info is included for each of them and generates a compiles license_info.json for those
  */
 
-function readDirRecSync(path): string[] {
-    const result = []
-    for (const entry of readdirSync(path)) {
-        const fullEntry = path + "/" + entry
-        const stats = lstatSync(fullEntry)
-        if (stats.isDirectory()) {
-            // Subdirectory
-            // @ts-ignore
-            result.push(...readDirRecSync(fullEntry))
-        } else {
-            result.push(fullEntry)
-        }
-    }
-    return result;
-}
-
 function generateLicenseInfos(paths: string[]): SmallLicense[] {
     const licenses = []
     for (const path of paths) {
@@ -156,7 +141,7 @@ function shuffle(array) {
 }
 
 console.log("Checking and compiling license info")
-const contents = readDirRecSync("./assets")
+const contents = ScriptUtils.readDirRecSync("./assets")
     .filter(entry => entry.indexOf("./assets/generated") != 0)
 const licensePaths = contents.filter(entry => entry.indexOf("license_info.json") >= 0)
 const licenseInfos = generateLicenseInfos(licensePaths);