Merge develop

scripts/ScriptUtils.ts

@@ -0,0 +1,20 @@
+import {lstatSync, readdirSync} from "fs";
+
+export default class ScriptUtils {
+    public static readDirRecSync(path): string[] {
+        const result = []
+        for (const entry of readdirSync(path)) {
+            const fullEntry = path + "/" + entry
+            const stats = lstatSync(fullEntry)
+            if (stats.isDirectory()) {
+                // Subdirectory
+                // @ts-ignore
+                result.push(...ScriptUtils.readDirRecSync(fullEntry))
+            } else {
+                result.push(fullEntry)
+            }
+        }
+        return result;
+    }
+
+}

scripts/generateIncludedImages.ts

@@ -9,6 +9,10 @@ function genImages() {
     const allNames: string[] = [];
     for (const path of dir) {
 
+        if(path.endsWith("license_info.json")){
+            continue;
+        }
+        
         if (!path.endsWith(".svg")) {
             throw "Non-svg file detected in the svg files: " + path;
         }

scripts/generateLayerOverview.ts

@@ -0,0 +1,136 @@
+import ScriptUtils from "./ScriptUtils";
+import {Utils} from "../Utils";
+import {lstatSync, readdirSync, readFileSync, writeFileSync} from "fs";
+
+Utils.runningFromConsole = true
+import LayerConfig from "../Customizations/JSON/LayerConfig";
+import {error} from "util";
+import * as licenses from "../assets/generated/license_info.json"
+import SmallLicense from "../Models/smallLicense";
+import LayoutConfig from "../Customizations/JSON/LayoutConfig";
+import {LayerConfigJson} from "../Customizations/JSON/LayerConfigJson";
+import {Layer} from "leaflet";
+// This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files.
+// It spits out an overview of those to be used to load them
+
+
+// First, remove the old file. It might be buggy!
+writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
+    "layers": [],
+    "themes": []
+}))
+const layerFiles = ScriptUtils.readDirRecSync("./assets/layers")
+    .filter(path => path.indexOf(".json") > 0)
+    .filter(path => path.indexOf("license_info.json") < 0)
+    .map(path => {
+        try {
+            const parsed = JSON.parse(readFileSync(path, "UTF8"));
+            return parsed
+        } catch (e) {
+            console.error("Could not parse file ", path, "due to ", e)
+        }
+    })
+const themeFiles: any[] = ScriptUtils.readDirRecSync("./assets/themes")
+    .filter(path => path.indexOf(".json") > 0)
+    .filter(path => path.indexOf("license_info.json") < 0)
+    .map(path => {
+        return JSON.parse(readFileSync(path, "UTF8"));
+    })
+writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
+    "layers": layerFiles,
+    "themes": themeFiles
+}))
+
+
+console.log("Discovered ", layerFiles.length, "layers and ", themeFiles.length, "themes\n")
+console.log("   ---------- VALIDATING ---------")
+// ------------- VALIDATION --------------
+const licensePaths = []
+for (const i in licenses) {
+    licensePaths.push(licenses[i].path)
+}
+const knownPaths = new Set<string>(licensePaths)
+
+function validateLayer(layerJson: LayerConfigJson, context?: string): string[] {
+    let errorCount = [];
+    if (layerJson["overpassTags"] !== undefined) {
+        errorCount.push("CRIT! Layer ", layerJson.id, "still uses the old 'overpassTags'-format. Please use 'source: {osmTags: <tags>}' instead")
+    }
+    try {
+        const layer = new LayerConfig(layerJson, "test", true)
+        const images = Array.from(layer.ExtractImages())
+        const remoteImages = images.filter(img => img.indexOf("http") == 0)
+        for (const remoteImage of remoteImages) {
+            errorCount.push("Found a remote image: "+ remoteImage+ " in layer "+ layer.id)
+        }
+        for (const image of images) {
+            if (!knownPaths.has(image)) {
+                const ctx = context === undefined ? "" : ` in a layer defined in the theme ${context}`
+                errorCount.push(`Image with path ${image} not found or not attributed; it is used in ${layer.id}${ctx}`)
+            }
+        }
+
+    } catch (e) {
+        return [`Layer ${layerJson.id}` ?? JSON.stringify(layerJson).substring(0, 50)+" is invalid: "+ e]
+    }
+    return errorCount
+}
+
+let layerErrorCount = []
+const knownLayerIds = new Set<string>();
+for (const layerFile of layerFiles) {
+    knownLayerIds.add(layerFile.id)
+    layerErrorCount .push(...validateLayer(layerFile))
+}
+
+let themeErrorCount = []
+for (const themeFile of themeFiles) {
+
+    for (const layer of themeFile.layers) {
+        if (typeof layer === "string") {
+            if (!knownLayerIds.has(layer)) {
+                themeErrorCount.push("Unknown layer id: "+ layer)
+            }
+        } else {
+            if (layer.builtin !== undefined) {
+                if (!knownLayerIds.has(layer.builtin)) {
+                    themeErrorCount.push("Unknown layer id: "+ layer.builtin+ "(which uses inheritance)")
+                } 
+            } else {
+                // layer.builtin contains layer overrides - we can skip those
+                layerErrorCount .push(...validateLayer(layer, themeFile.id))
+            }
+        }
+    }
+
+    themeFile.layers = themeFile.layers
+        .filter(l => typeof l != "string") // We remove all the builtin layer references as they don't work with ts-node for some weird reason
+        .filter(l => l.builtin === undefined)
+
+    try {
+        const theme = new LayoutConfig(themeFile, true, "test")
+        if (theme.id !== theme.id.toLowerCase()) {
+            themeErrorCount.push("Theme ids should be in lowercase, but it is "+ theme.id)
+        }
+    } catch (e) {
+        themeErrorCount.push("Could not parse theme "+ themeFile["id"]+ "due to", e)
+    }
+}
+
+console.log("LE", layerErrorCount)
+
+if (layerErrorCount.length + themeErrorCount.length == 0) {
+    console.log("All good!")
+} else {
+    const errors = layerErrorCount.concat(themeErrorCount).join("\n")
+    console.log(errors)
+    const msg = (`Found ${errors.length} errors in the layers; ${themeErrorCount} errors in the themes`)
+    if(process.argv.indexOf("--no-fail") >= 0) {
+        console.log(msg)
+    }else if(process.argv.indexOf("--report") >= 0){
+        writeFileSync("layer_report.txt", errors)
+    }else{
+        
+        throw msg;
+    }
+}

scripts/generateLayouts.ts

@@ -3,13 +3,13 @@ import {Utils} from "../Utils";
 Utils.runningFromConsole = true;
 
 import LayoutConfig from "../Customizations/JSON/LayoutConfig";
-import {AllKnownLayouts} from "../Customizations/AllKnownLayouts";
 import {existsSync, mkdirSync, readFileSync, writeFile, writeFileSync} from "fs";
 import Locale from "../UI/i18n/Locale";
 import Translations from "../UI/i18n/Translations";
 import {Translation} from "../UI/i18n/Translation";
 import Constants from "../Models/Constants";
-
+import * as all_known_layouts from "../assets/generated/known_layers_and_themes.json"
+import {LayoutConfigJson} from "../Customizations/JSON/LayoutConfigJson";
 const sharp = require('sharp');
 
 
@@ -234,9 +234,12 @@ if (!existsSync(generatedDir)) {
 }
 
 const blacklist = ["", "test", ".", "..", "manifest", "index", "land", "preferences", "account", "openstreetmap", "custom"]
-const all = AllKnownLayouts.allSets;
+const all : LayoutConfigJson[] = all_known_layouts.themes;
 
-for (const layoutName in all) {
+for (const i in all) {
+    const layoutConfigJson : LayoutConfigJson = all[i]
+    const layout = new LayoutConfig(layoutConfigJson, true, "generating layouts")
+    const layoutName = layout.id
     if (blacklist.indexOf(layoutName.toLowerCase()) >= 0) {
         console.log(`Skipping a layout with name${layoutName}, it is on the blacklist`);
         continue;
@@ -246,7 +249,6 @@ for (const layoutName in all) {
             console.log("Could not write manifest for ", layoutName, " because ", err)
         }
     };
-    const layout = all[layoutName];
     validate(layout)
     createManifest(layout, "").then(manifObj => {
         const manif = JSON.stringify(manifObj, undefined, 2);

scripts/generateLicenseInfo.ts

@@ -0,0 +1,213 @@
+import {Utils} from "../Utils";
+import {lstatSync, readdirSync, readFileSync, writeFileSync, unlinkSync} from "fs";
+import SmallLicense from "../Models/smallLicense";
+import ScriptUtils from "./ScriptUtils";
+
+Utils.runningFromConsole = true;
+
+/**
+ * Sweeps the entire 'assets/' (except assets/generated) directory for image files and any 'license_info.json'-file.
+ * Checks that the license info is included for each of them and generates a compiles license_info.json for those
+ */
+
+function generateLicenseInfos(paths: string[]): SmallLicense[] {
+    const licenses = []
+    for (const path of paths) {
+        const parsed = JSON.parse(readFileSync(path, "UTF-8"))
+        if (Array.isArray(parsed)) {
+            const l: SmallLicense[] = parsed
+            for (const smallLicens of l) {
+                smallLicens.path = path.substring(0, path.length - "license_info.json".length) + smallLicens.path
+            }
+            licenses.push(...l)
+        } else {
+            const smallLicens: SmallLicense = parsed;
+            /*if(parsed.license === "CC-BY"){
+                console.log("Rewriting ", path)
+                parsed.license === "CC-BY 4.0"
+                writeFileSync(path, JSON.stringify(smallLicens, null, "  "))
+            }*/
+            
+            smallLicens.path = path.substring(0, 1 + path.lastIndexOf("/")) + smallLicens.path
+            licenses.push(smallLicens)
+        }
+
+    }
+    return licenses
+}
+
+function missingLicenseInfos(licenseInfos: SmallLicense[], allIcons: string[]) {
+    const missing = []
+
+    const knownPaths = new Set<string>()
+    for (const licenseInfo of licenseInfos) {
+        knownPaths.add(licenseInfo.path)
+    }
+
+    for (const iconPath of allIcons) {
+        if (iconPath.indexOf("license_info.json") >= 0) {
+            continue;
+        }
+        if (knownPaths.has(iconPath)) {
+            continue;
+        }
+        missing.push(iconPath)
+    }
+    return missing;
+}
+
+const prompt = require('prompt-sync')();
+
+const knownLicenses = new Map<string, SmallLicense>()
+knownLicenses.set("cf", {
+    authors: ["Pieter Fiers", "Thibault Declercq", "Pierre Barban", "Joost Schouppe", "Pieter Vander Vennet"],
+    path: undefined,
+    license: "CC-BY-SA",
+    sources: ["https://osoc.be/editions/2020/cyclofix"]
+})
+knownLicenses.set("me", {
+    authors: ["Pieter Vander Vennet"],
+    path: undefined,
+    license: "CC0",
+    sources: []
+})
+
+knownLicenses.set("t", {
+    authors: [],
+    path: undefined,
+    license: "CC0; trivial",
+    sources: []
+})
+
+knownLicenses.set("na", {
+    authors: [],
+    path: undefined,
+    license: "CC0",
+    sources: []
+})
+
+knownLicenses.set("chrn", {
+    authors: ["Christian Neumann"],
+    path: undefined,
+    license: "CC-BY-SA 3.0",
+    sources: ["https://utopicode.de/", "https://github.com/chrneumann/MapComplete"]
+})
+
+knownLicenses.set("klimaan", {
+    authors: ["Klimaan VZW"],
+    path: undefined,
+    license: "CC-BY-SA 3.0",
+    sources: ["https://klimaan.be/"]
+})
+
+function promptLicenseFor(path): SmallLicense {
+    console.log("License abbreviations:")
+    knownLicenses.forEach((value, key) => {
+        console.log(key, " => ", value)
+    })
+    const author = prompt("What is the author for artwork " + path + "? (or: [Q]uit, [S]kip)  > ")
+    path = path.substring(path.lastIndexOf("/") + 1)
+
+    if (knownLicenses.has(author)) {
+        const license = knownLicenses.get(author);
+        license.path = path;
+        return license;
+    }
+
+    if (author == "s") {
+        return null;
+    }
+    if (author == "Q" || author == "q" || author == "") {
+        throw "Quitting now!"
+    }
+    let authors = author.split(";")
+    if (author.toLowerCase() == "none") {
+        authors = []
+    }
+    return {
+        authors: author.split(";"),
+        path: path,
+        license: prompt("What is the license for artwork " + path + "?  > "),
+        sources: prompt("Where was this artwork found?  > ").split(";")
+    }
+}
+
+function createLicenseInfoFor(path): void {
+    const li = promptLicenseFor(path);
+    if (li == null) {
+        return;
+    }
+    writeFileSync(path + ".license_info.json", JSON.stringify(li, null, "  "))
+}
+
+function cleanLicenseInfo(allPaths: string[], allLicenseInfos: SmallLicense[]){
+    // Read the license info file from the generated assets, creates a compiled license info in every directory
+    // Note: this removes all the old license infos
+    for (const licensePath of licensePaths) {
+        unlinkSync(licensePath)
+    }
+    
+    const perDirectory = new Map<string, SmallLicense[]>()
+
+    for (const license of allLicenseInfos) {
+        const p = license.path
+        const dir = p.substring(0, p.lastIndexOf("/"))
+        license.path = p.substring(dir.length + 1)
+        if(!perDirectory.has(dir)){
+            perDirectory.set(dir, [])
+        }
+        perDirectory.get(dir).push(license)
+    }
+    
+    perDirectory.forEach((licenses, dir) => {
+        writeFileSync( dir+"/license_info.json", JSON.stringify(licenses, null, 2))
+    })
+    
+}
+
+function queryMissingLicenses(missingLicenses: string[])  {
+    process.on('SIGINT', function () {
+        console.log("Aborting... Bye!");
+        process.exit();
+    });
+
+    let i = 1;
+    for (const missingLicens of missingLicenses) {
+        console.log(i + " / " + missingLicenses.length)
+        i++;
+        if (i < missingLicenses.length - 5) {
+            //    continue
+        }
+        createLicenseInfoFor(missingLicens)
+    }
+
+    console.log("You're through!")
+}
+
+console.log("Checking and compiling license info")
+const contents = ScriptUtils.readDirRecSync("./assets")
+    .filter(entry => entry.indexOf("./assets/generated") != 0)
+const licensePaths = contents.filter(entry => entry.indexOf("license_info.json") >= 0)
+const licenseInfos = generateLicenseInfos(licensePaths);
+writeFileSync("./assets/generated/license_info.json", JSON.stringify(licenseInfos, null, "  "))
+
+const artwork = contents.filter(pth => pth.match(/(.svg|.png|.jpg)$/i) != null)
+const missingLicenses = missingLicenseInfos(licenseInfos, artwork)
+
+
+if(process.argv.indexOf("--prompt") >= 0 || process.argv.indexOf("--query") >= 0 ) {
+    queryMissingLicenses(missingLicenses)
+}
+if(missingLicenses.length > 0){
+    const msg = `There are ${missingLicenses.length} licenses missing.`
+    if(process.argv.indexOf("--no-fail") >= 0){
+        console.log(msg)
+    }else if(process.argv.indexOf("--report") >= 0){
+        writeFileSync("missing_licenses.txt", missingLicenses.join("\n"))
+    } else{
+
+        throw msg
+    }
+}
+
+cleanLicenseInfo(licensePaths, licenseInfos)