Scripts(nsi): stabilize and make scripts more log-friendly

scripts/ScriptUtils.ts

@@ -7,7 +7,11 @@ import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
 import xml2js from "xml2js"
 
 export default class ScriptUtils {
-    public static fixUtils() {
+
+    public static verbose = true
+
+    public static fixUtils(verbose = true) {
+        ScriptUtils.verbose = verbose
         Utils.externalDownloadFunction = ScriptUtils.Download
     }
 
@@ -54,16 +58,26 @@ export default class ScriptUtils {
     }
 
     public static DownloadFileTo(url, targetFilePath: string): Promise<void> {
+        if (this.verbose) {
             ScriptUtils.erasableLog("Downloading", url, "to", targetFilePath)
-        return new Promise<void>((resolve) => {
+        }
-            https.get(url, (res) => {
+        return new Promise<void>((resolve, reject) => {
+            const req = https.get(url, (res) => {
+                if (res.statusCode === 429) {
+                    console.error(`RATE LIMITED on ${url}`)
+                    ScriptUtils.sleep(50000).then(() => reject("rate limited"))
+                }
+                if (res.statusCode !== 200) {
+                    req.destroy()
+                    return reject(new Error(`HTTP ${res.statusCode} for ${url}`))
+                }
                 const filePath = fs.createWriteStream(targetFilePath)
                 res.pipe(filePath)
                 filePath.on("finish", () => {
                     filePath.close()
                     resolve()
                 })
-            })
+            }).on("error", (e) => reject(e))
         })
     }
 
@@ -222,7 +236,9 @@ export default class ScriptUtils {
                 if (!headers.Accept) {
                     headers.accept ??= "application/json"
                 }
+                if (ScriptUtils.verbose) {
                     ScriptUtils.erasableLog(" > ScriptUtils.Download(", url, ")")
+                }
                 const urlObj = new URL(url)
                 const request = https.get(
                     {

scripts/generateNsiStats.ts

@@ -146,15 +146,10 @@ class GenerateNsiStats extends Script {
         const batchSize = 16
         const f = (stats) => stats.data.find((t) => t.type === "all").count
 
+        const start = new Date().getTime()
+        let lastBatchStart = start
+        const preloaded = Object.keys(allBrands).length
         for (let i = 0; i < allBrandNames.length; i += batchSize) {
-            console.warn(
-                "Downloading ",
-                batchSize,
-                "occurence counts, items: ",
-                i + "/" + allBrandNames.length
-            )
-            let downloaded = 0
-
             await Promise.all(
                 Utils.timesT(batchSize, async (j) => {
                     const brand = allBrandNames[i + j]
@@ -162,18 +157,30 @@ class GenerateNsiStats extends Script {
                         allBrands[brand] = {}
                     }
                     const writeInto = allBrands[brand]
-                    const dloaded = await TagInfo.getGlobalDistributionsFor(
+                    await TagInfo.getGlobalDistributionsFor(
                         writeInto,
                         f,
                         type,
                         brand
                     )
-                    downloaded += dloaded
                 })
             )
-            console.log("Downloaded ", downloaded, " values this batch")
             writeFileSync(path, JSON.stringify(allBrands), "utf8")
-            console.log("Checkpointed", path)
+
+            if (i > preloaded && (i / batchSize) % 10 === 0) {
+                const now = new Date().getTime()
+                const elapsed = (now - start) / 1000
+                const speed = (i - preloaded) / elapsed
+                const restingItems = allBrandNames.length - i
+                const restingSeconds = restingItems / speed
+
+                const elapsedB = (now - lastBatchStart) / 1000
+
+                console.log(`Downloaded ${i}/${allBrandNames.length} of category ${type} (from checkpoint: ${preloaded}), elapsed ${Utils.toHumanTime(elapsed)} (last batch: ${elapsedB}); speed: ${Math.floor(speed * 1000)}items/millisec (last batch: ${Math.floor((i - preloaded) / (elapsedB * 1000))}); estimated left: ${Utils.toHumanTime(restingSeconds)}`)
+                lastBatchStart = new Date().getTime()
+            } else {
+                process.stdout.write(".")
+            }
         }
         console.log("Written:", path)
         writeFileSync(path, JSON.stringify(allBrands), "utf8")
@@ -181,11 +188,13 @@ class GenerateNsiStats extends Script {
 
     constructor() {
         super(
-            "Downloads stats on osmSource-tags and keys from tagInfo. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index "
+            "Downloads stats on osmSource-tags and keys from tagInfo; generates 'key_totals.json' and '*.summarized.json'. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index ",
         )
     }
 
     async main() {
+        ScriptUtils.verbose = false
+
         const target = "./public/assets/data/nsi/"
         const basepath = target + "stats/"
         {

scripts/nsiLogos.ts

@@ -12,10 +12,12 @@ import { FilterConfigOptionJson } from "../src/Models/ThemeConfig/Json/FilterCon
 import { TagUtils } from "../src/Logic/Tags/TagUtils"
 import { openSync, readSync } from "node:fs"
 import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
+import Constants from "../src/Models/Constants"
 
 class NsiLogos extends Script {
     constructor() {
         super("Contains various subcommands for NSI logo maintainance")
+        ScriptUtils.verbose = false
     }
 
     private async downloadLogo(
@@ -38,7 +40,7 @@ class NsiLogos extends Script {
         }
     }
 
-    private async downloadLogoUnsafe(nsiItem: NSIItem, type: string, basePath: string) {
+    private async downloadLogoUnsafe(nsiItem: NSIItem, type: string, basePath: string) : Promise<true | false | "error">{
         if (nsiItem === undefined) {
             return false
         }
@@ -86,7 +88,7 @@ class NsiLogos extends Script {
                 }
                 if ((<string>logos.wikidata).toLowerCase().endsWith(".svg")) {
                     if (!path.endsWith(".svg")) {
-                        throw "Undetected svg path:" + logos.wikidata
+                       return false // We don't supports svgs
                     }
                     writeFileSync(path, dloaded["content"], "utf8")
                     return true
@@ -242,6 +244,7 @@ class NsiLogos extends Script {
     }
 
     private async download() {
+        ScriptUtils.verbose = false
         const types = ["brand", "operator"]
         let dload = 0
         let failed = 0
@@ -393,7 +396,7 @@ class NsiLogos extends Script {
     }
 
     private commands: Record<string, { f: () => Promise<void>; doc?: string }> = {
-        download: { f: () => this.download(), doc: "Download all icons" },
+        download: { f: () => this.download(), doc: "Download all icons. Gets the index from "+Constants.nsiLogosEndpoint+" first, then fetches the corresponding logos" },
         generateRenderings: {
             f: () => this.generateRenderings(),
             doc: "Generates the layer files 'nsi_brand.json' and 'nsi_operator.json' which allows to reuse the icons in renderings",
@@ -405,7 +408,7 @@ class NsiLogos extends Script {
         },
         patch: {
             f: () => this.patchNsiFile(),
-            doc: "Reads nsi.min.json, adds the 'ext' (extension) field to every relevant entry",
+            doc: "Modifies nsi.min.json, adds the 'ext' (extension) field to every relevant entry",
         },
         all: {
             doc: "Run `download`, `generateRenderings`, `prune`  and `addExtensions`",

scripts/prepare-build.sh

@@ -17,6 +17,7 @@ sed "s/= \"0.0.0\"/= \"$VERSION\"/" -i public/service-worker.js
 
 # This script ends every line with '&&' to chain everything. A failure will thus stop the build
 npm run download:editor-layer-index &&
+npm i name-suggestion-index && npx vite-node scripts/nsiLogos.ts -- generateRenderings
 npm run prep:layeroverview &&
 # Done by 'deploy_hosted':
 # npm run generate:licenses; npm run generate:images; npm run generate:charging-stations; npm run generate:service-worker; npm run download:editor-layer-index

src/Utils.ts

@@ -127,8 +127,16 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
         return str.substring(0, 1).toUpperCase() + str.substring(1)
     }
 
+    /**
+     * Always prints (at least) digits
+     *
+     * Utils.twoDigits(5) // => "05"
+     * Utils.twoDigits(0) // => "00"
+     * Utils.twoDigits(128) // => "128"
+     * Utils.twoDigits(-5) // => "-5"
+     */
     public static twoDigits(i: number) {
-        if (i < 10) {
+        if (i < 10 && i >= 0) {
             return "0" + i
         }
         return "" + i
@@ -1058,7 +1066,16 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
         })
     }
 
-    public static toHumanTime(seconds): string {
+    /**
+     * Converts a number of seconds into a human readable format
+     *
+     * Utils.toHumanTime(45) // => "0:00:45"
+     * Utils.toHumanTime(120) // => "0:02:00"
+     * Utils.toHumanTime(3610) // => "1:00:10"
+     * Utils.toHumanTime(25 * 3600) // => "1days 1h"
+     *
+     */
+    public static toHumanTime(seconds:number): string {
         seconds = Math.floor(seconds)
         let minutes = Math.floor(seconds / 60)
         seconds = seconds % 60