From 2b10c715b0b733349acc652725e5febedab99ae0 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Thu, 16 Oct 2025 00:43:06 +0200 Subject: [PATCH] Scripts(nsi): stabilize and make scripts more log-friendly --- scripts/ScriptUtils.ts | 28 ++++++++++++++++++++++------ scripts/generateNsiStats.ts | 35 ++++++++++++++++++++++------------- scripts/nsiLogos.ts | 11 +++++++---- scripts/prepare-build.sh | 1 + src/Utils.ts | 21 +++++++++++++++++++-- 5 files changed, 71 insertions(+), 25 deletions(-) diff --git a/scripts/ScriptUtils.ts b/scripts/ScriptUtils.ts index 4adeaf6c4..5707d38f1 100644 --- a/scripts/ScriptUtils.ts +++ b/scripts/ScriptUtils.ts @@ -7,7 +7,11 @@ import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson" import xml2js from "xml2js" export default class ScriptUtils { - public static fixUtils() { + + public static verbose = true + + public static fixUtils(verbose = true) { + ScriptUtils.verbose = verbose Utils.externalDownloadFunction = ScriptUtils.Download } @@ -54,16 +58,26 @@ export default class ScriptUtils { } public static DownloadFileTo(url, targetFilePath: string): Promise { - ScriptUtils.erasableLog("Downloading", url, "to", targetFilePath) - return new Promise((resolve) => { - https.get(url, (res) => { + if (this.verbose) { + ScriptUtils.erasableLog("Downloading", url, "to", targetFilePath) + } + return new Promise((resolve, reject) => { + const req = https.get(url, (res) => { + if (res.statusCode === 429) { + console.error(`RATE LIMITED on ${url}`) + ScriptUtils.sleep(50000).then(() => reject("rate limited")) + } + if (res.statusCode !== 200) { + req.destroy() + return reject(new Error(`HTTP ${res.statusCode} for ${url}`)) + } const filePath = fs.createWriteStream(targetFilePath) res.pipe(filePath) filePath.on("finish", () => { filePath.close() resolve() }) - }) + }).on("error", (e) => reject(e)) }) } @@ -222,7 +236,9 @@ export default class ScriptUtils { if (!headers.Accept) { headers.accept ??= "application/json" } - ScriptUtils.erasableLog(" > ScriptUtils.Download(", url, ")") + if (ScriptUtils.verbose) { + ScriptUtils.erasableLog(" > ScriptUtils.Download(", url, ")") + } const urlObj = new URL(url) const request = https.get( { diff --git a/scripts/generateNsiStats.ts b/scripts/generateNsiStats.ts index ae5d97636..edfdb6f0e 100644 --- a/scripts/generateNsiStats.ts +++ b/scripts/generateNsiStats.ts @@ -146,15 +146,10 @@ class GenerateNsiStats extends Script { const batchSize = 16 const f = (stats) => stats.data.find((t) => t.type === "all").count + const start = new Date().getTime() + let lastBatchStart = start + const preloaded = Object.keys(allBrands).length for (let i = 0; i < allBrandNames.length; i += batchSize) { - console.warn( - "Downloading ", - batchSize, - "occurence counts, items: ", - i + "/" + allBrandNames.length - ) - let downloaded = 0 - await Promise.all( Utils.timesT(batchSize, async (j) => { const brand = allBrandNames[i + j] @@ -162,18 +157,30 @@ class GenerateNsiStats extends Script { allBrands[brand] = {} } const writeInto = allBrands[brand] - const dloaded = await TagInfo.getGlobalDistributionsFor( + await TagInfo.getGlobalDistributionsFor( writeInto, f, type, brand ) - downloaded += dloaded }) ) - console.log("Downloaded ", downloaded, " values this batch") writeFileSync(path, JSON.stringify(allBrands), "utf8") - console.log("Checkpointed", path) + + if (i > preloaded && (i / batchSize) % 10 === 0) { + const now = new Date().getTime() + const elapsed = (now - start) / 1000 + const speed = (i - preloaded) / elapsed + const restingItems = allBrandNames.length - i + const restingSeconds = restingItems / speed + + const elapsedB = (now - lastBatchStart) / 1000 + + console.log(`Downloaded ${i}/${allBrandNames.length} of category ${type} (from checkpoint: ${preloaded}), elapsed ${Utils.toHumanTime(elapsed)} (last batch: ${elapsedB}); speed: ${Math.floor(speed * 1000)}items/millisec (last batch: ${Math.floor((i - preloaded) / (elapsedB * 1000))}); estimated left: ${Utils.toHumanTime(restingSeconds)}`) + lastBatchStart = new Date().getTime() + } else { + process.stdout.write(".") + } } console.log("Written:", path) writeFileSync(path, JSON.stringify(allBrands), "utf8") @@ -181,11 +188,13 @@ class GenerateNsiStats extends Script { constructor() { super( - "Downloads stats on osmSource-tags and keys from tagInfo. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index " + "Downloads stats on osmSource-tags and keys from tagInfo; generates 'key_totals.json' and '*.summarized.json'. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index ", ) } async main() { + ScriptUtils.verbose = false + const target = "./public/assets/data/nsi/" const basepath = target + "stats/" { diff --git a/scripts/nsiLogos.ts b/scripts/nsiLogos.ts index aa109a784..5848ce365 100644 --- a/scripts/nsiLogos.ts +++ b/scripts/nsiLogos.ts @@ -12,10 +12,12 @@ import { FilterConfigOptionJson } from "../src/Models/ThemeConfig/Json/FilterCon import { TagUtils } from "../src/Logic/Tags/TagUtils" import { openSync, readSync } from "node:fs" import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson" +import Constants from "../src/Models/Constants" class NsiLogos extends Script { constructor() { super("Contains various subcommands for NSI logo maintainance") + ScriptUtils.verbose = false } private async downloadLogo( @@ -38,7 +40,7 @@ class NsiLogos extends Script { } } - private async downloadLogoUnsafe(nsiItem: NSIItem, type: string, basePath: string) { + private async downloadLogoUnsafe(nsiItem: NSIItem, type: string, basePath: string) : Promise{ if (nsiItem === undefined) { return false } @@ -86,7 +88,7 @@ class NsiLogos extends Script { } if ((logos.wikidata).toLowerCase().endsWith(".svg")) { if (!path.endsWith(".svg")) { - throw "Undetected svg path:" + logos.wikidata + return false // We don't supports svgs } writeFileSync(path, dloaded["content"], "utf8") return true @@ -242,6 +244,7 @@ class NsiLogos extends Script { } private async download() { + ScriptUtils.verbose = false const types = ["brand", "operator"] let dload = 0 let failed = 0 @@ -393,7 +396,7 @@ class NsiLogos extends Script { } private commands: Record Promise; doc?: string }> = { - download: { f: () => this.download(), doc: "Download all icons" }, + download: { f: () => this.download(), doc: "Download all icons. Gets the index from "+Constants.nsiLogosEndpoint+" first, then fetches the corresponding logos" }, generateRenderings: { f: () => this.generateRenderings(), doc: "Generates the layer files 'nsi_brand.json' and 'nsi_operator.json' which allows to reuse the icons in renderings", @@ -405,7 +408,7 @@ class NsiLogos extends Script { }, patch: { f: () => this.patchNsiFile(), - doc: "Reads nsi.min.json, adds the 'ext' (extension) field to every relevant entry", + doc: "Modifies nsi.min.json, adds the 'ext' (extension) field to every relevant entry", }, all: { doc: "Run `download`, `generateRenderings`, `prune` and `addExtensions`", diff --git a/scripts/prepare-build.sh b/scripts/prepare-build.sh index 4c4140608..fc5a4ac5a 100755 --- a/scripts/prepare-build.sh +++ b/scripts/prepare-build.sh @@ -17,6 +17,7 @@ sed "s/= \"0.0.0\"/= \"$VERSION\"/" -i public/service-worker.js # This script ends every line with '&&' to chain everything. A failure will thus stop the build npm run download:editor-layer-index && +npm i name-suggestion-index && npx vite-node scripts/nsiLogos.ts -- generateRenderings npm run prep:layeroverview && # Done by 'deploy_hosted': # npm run generate:licenses; npm run generate:images; npm run generate:charging-stations; npm run generate:service-worker; npm run download:editor-layer-index diff --git a/src/Utils.ts b/src/Utils.ts index d0542dbdb..3faafc21c 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -127,8 +127,16 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be return str.substring(0, 1).toUpperCase() + str.substring(1) } + /** + * Always prints (at least) digits + * + * Utils.twoDigits(5) // => "05" + * Utils.twoDigits(0) // => "00" + * Utils.twoDigits(128) // => "128" + * Utils.twoDigits(-5) // => "-5" + */ public static twoDigits(i: number) { - if (i < 10) { + if (i < 10 && i >= 0) { return "0" + i } return "" + i @@ -1058,7 +1066,16 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be }) } - public static toHumanTime(seconds): string { + /** + * Converts a number of seconds into a human readable format + * + * Utils.toHumanTime(45) // => "0:00:45" + * Utils.toHumanTime(120) // => "0:02:00" + * Utils.toHumanTime(3610) // => "1:00:10" + * Utils.toHumanTime(25 * 3600) // => "1days 1h" + * + */ + public static toHumanTime(seconds:number): string { seconds = Math.floor(seconds) let minutes = Math.floor(seconds / 60) seconds = seconds % 60