Scripts(nsi): stabilize and make scripts more log-friendly

This commit is contained in:
Pieter Vander Vennet 2025-10-16 00:43:06 +02:00
parent 53106cc0bf
commit 2b10c715b0
5 changed files with 71 additions and 25 deletions

View file

@ -7,7 +7,11 @@ import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
import xml2js from "xml2js" import xml2js from "xml2js"
export default class ScriptUtils { export default class ScriptUtils {
public static fixUtils() {
public static verbose = true
public static fixUtils(verbose = true) {
ScriptUtils.verbose = verbose
Utils.externalDownloadFunction = ScriptUtils.Download Utils.externalDownloadFunction = ScriptUtils.Download
} }
@ -54,16 +58,26 @@ export default class ScriptUtils {
} }
public static DownloadFileTo(url, targetFilePath: string): Promise<void> { public static DownloadFileTo(url, targetFilePath: string): Promise<void> {
if (this.verbose) {
ScriptUtils.erasableLog("Downloading", url, "to", targetFilePath) ScriptUtils.erasableLog("Downloading", url, "to", targetFilePath)
return new Promise<void>((resolve) => { }
https.get(url, (res) => { return new Promise<void>((resolve, reject) => {
const req = https.get(url, (res) => {
if (res.statusCode === 429) {
console.error(`RATE LIMITED on ${url}`)
ScriptUtils.sleep(50000).then(() => reject("rate limited"))
}
if (res.statusCode !== 200) {
req.destroy()
return reject(new Error(`HTTP ${res.statusCode} for ${url}`))
}
const filePath = fs.createWriteStream(targetFilePath) const filePath = fs.createWriteStream(targetFilePath)
res.pipe(filePath) res.pipe(filePath)
filePath.on("finish", () => { filePath.on("finish", () => {
filePath.close() filePath.close()
resolve() resolve()
}) })
}) }).on("error", (e) => reject(e))
}) })
} }
@ -222,7 +236,9 @@ export default class ScriptUtils {
if (!headers.Accept) { if (!headers.Accept) {
headers.accept ??= "application/json" headers.accept ??= "application/json"
} }
if (ScriptUtils.verbose) {
ScriptUtils.erasableLog(" > ScriptUtils.Download(", url, ")") ScriptUtils.erasableLog(" > ScriptUtils.Download(", url, ")")
}
const urlObj = new URL(url) const urlObj = new URL(url)
const request = https.get( const request = https.get(
{ {

View file

@ -146,15 +146,10 @@ class GenerateNsiStats extends Script {
const batchSize = 16 const batchSize = 16
const f = (stats) => stats.data.find((t) => t.type === "all").count const f = (stats) => stats.data.find((t) => t.type === "all").count
const start = new Date().getTime()
let lastBatchStart = start
const preloaded = Object.keys(allBrands).length
for (let i = 0; i < allBrandNames.length; i += batchSize) { for (let i = 0; i < allBrandNames.length; i += batchSize) {
console.warn(
"Downloading ",
batchSize,
"occurence counts, items: ",
i + "/" + allBrandNames.length
)
let downloaded = 0
await Promise.all( await Promise.all(
Utils.timesT(batchSize, async (j) => { Utils.timesT(batchSize, async (j) => {
const brand = allBrandNames[i + j] const brand = allBrandNames[i + j]
@ -162,18 +157,30 @@ class GenerateNsiStats extends Script {
allBrands[brand] = {} allBrands[brand] = {}
} }
const writeInto = allBrands[brand] const writeInto = allBrands[brand]
const dloaded = await TagInfo.getGlobalDistributionsFor( await TagInfo.getGlobalDistributionsFor(
writeInto, writeInto,
f, f,
type, type,
brand brand
) )
downloaded += dloaded
}) })
) )
console.log("Downloaded ", downloaded, " values this batch")
writeFileSync(path, JSON.stringify(allBrands), "utf8") writeFileSync(path, JSON.stringify(allBrands), "utf8")
console.log("Checkpointed", path)
if (i > preloaded && (i / batchSize) % 10 === 0) {
const now = new Date().getTime()
const elapsed = (now - start) / 1000
const speed = (i - preloaded) / elapsed
const restingItems = allBrandNames.length - i
const restingSeconds = restingItems / speed
const elapsedB = (now - lastBatchStart) / 1000
console.log(`Downloaded ${i}/${allBrandNames.length} of category ${type} (from checkpoint: ${preloaded}), elapsed ${Utils.toHumanTime(elapsed)} (last batch: ${elapsedB}); speed: ${Math.floor(speed * 1000)}items/millisec (last batch: ${Math.floor((i - preloaded) / (elapsedB * 1000))}); estimated left: ${Utils.toHumanTime(restingSeconds)}`)
lastBatchStart = new Date().getTime()
} else {
process.stdout.write(".")
}
} }
console.log("Written:", path) console.log("Written:", path)
writeFileSync(path, JSON.stringify(allBrands), "utf8") writeFileSync(path, JSON.stringify(allBrands), "utf8")
@ -181,11 +188,13 @@ class GenerateNsiStats extends Script {
constructor() { constructor() {
super( super(
"Downloads stats on osmSource-tags and keys from tagInfo. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index " "Downloads stats on osmSource-tags and keys from tagInfo; generates 'key_totals.json' and '*.summarized.json'. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index ",
) )
} }
async main() { async main() {
ScriptUtils.verbose = false
const target = "./public/assets/data/nsi/" const target = "./public/assets/data/nsi/"
const basepath = target + "stats/" const basepath = target + "stats/"
{ {

View file

@ -12,10 +12,12 @@ import { FilterConfigOptionJson } from "../src/Models/ThemeConfig/Json/FilterCon
import { TagUtils } from "../src/Logic/Tags/TagUtils" import { TagUtils } from "../src/Logic/Tags/TagUtils"
import { openSync, readSync } from "node:fs" import { openSync, readSync } from "node:fs"
import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson" import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import Constants from "../src/Models/Constants"
class NsiLogos extends Script { class NsiLogos extends Script {
constructor() { constructor() {
super("Contains various subcommands for NSI logo maintainance") super("Contains various subcommands for NSI logo maintainance")
ScriptUtils.verbose = false
} }
private async downloadLogo( private async downloadLogo(
@ -38,7 +40,7 @@ class NsiLogos extends Script {
} }
} }
private async downloadLogoUnsafe(nsiItem: NSIItem, type: string, basePath: string) { private async downloadLogoUnsafe(nsiItem: NSIItem, type: string, basePath: string) : Promise<true | false | "error">{
if (nsiItem === undefined) { if (nsiItem === undefined) {
return false return false
} }
@ -86,7 +88,7 @@ class NsiLogos extends Script {
} }
if ((<string>logos.wikidata).toLowerCase().endsWith(".svg")) { if ((<string>logos.wikidata).toLowerCase().endsWith(".svg")) {
if (!path.endsWith(".svg")) { if (!path.endsWith(".svg")) {
throw "Undetected svg path:" + logos.wikidata return false // We don't supports svgs
} }
writeFileSync(path, dloaded["content"], "utf8") writeFileSync(path, dloaded["content"], "utf8")
return true return true
@ -242,6 +244,7 @@ class NsiLogos extends Script {
} }
private async download() { private async download() {
ScriptUtils.verbose = false
const types = ["brand", "operator"] const types = ["brand", "operator"]
let dload = 0 let dload = 0
let failed = 0 let failed = 0
@ -393,7 +396,7 @@ class NsiLogos extends Script {
} }
private commands: Record<string, { f: () => Promise<void>; doc?: string }> = { private commands: Record<string, { f: () => Promise<void>; doc?: string }> = {
download: { f: () => this.download(), doc: "Download all icons" }, download: { f: () => this.download(), doc: "Download all icons. Gets the index from "+Constants.nsiLogosEndpoint+" first, then fetches the corresponding logos" },
generateRenderings: { generateRenderings: {
f: () => this.generateRenderings(), f: () => this.generateRenderings(),
doc: "Generates the layer files 'nsi_brand.json' and 'nsi_operator.json' which allows to reuse the icons in renderings", doc: "Generates the layer files 'nsi_brand.json' and 'nsi_operator.json' which allows to reuse the icons in renderings",
@ -405,7 +408,7 @@ class NsiLogos extends Script {
}, },
patch: { patch: {
f: () => this.patchNsiFile(), f: () => this.patchNsiFile(),
doc: "Reads nsi.min.json, adds the 'ext' (extension) field to every relevant entry", doc: "Modifies nsi.min.json, adds the 'ext' (extension) field to every relevant entry",
}, },
all: { all: {
doc: "Run `download`, `generateRenderings`, `prune` and `addExtensions`", doc: "Run `download`, `generateRenderings`, `prune` and `addExtensions`",

View file

@ -17,6 +17,7 @@ sed "s/= \"0.0.0\"/= \"$VERSION\"/" -i public/service-worker.js
# This script ends every line with '&&' to chain everything. A failure will thus stop the build # This script ends every line with '&&' to chain everything. A failure will thus stop the build
npm run download:editor-layer-index && npm run download:editor-layer-index &&
npm i name-suggestion-index && npx vite-node scripts/nsiLogos.ts -- generateRenderings
npm run prep:layeroverview && npm run prep:layeroverview &&
# Done by 'deploy_hosted': # Done by 'deploy_hosted':
# npm run generate:licenses; npm run generate:images; npm run generate:charging-stations; npm run generate:service-worker; npm run download:editor-layer-index # npm run generate:licenses; npm run generate:images; npm run generate:charging-stations; npm run generate:service-worker; npm run download:editor-layer-index

View file

@ -127,8 +127,16 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
return str.substring(0, 1).toUpperCase() + str.substring(1) return str.substring(0, 1).toUpperCase() + str.substring(1)
} }
/**
* Always prints (at least) digits
*
* Utils.twoDigits(5) // => "05"
* Utils.twoDigits(0) // => "00"
* Utils.twoDigits(128) // => "128"
* Utils.twoDigits(-5) // => "-5"
*/
public static twoDigits(i: number) { public static twoDigits(i: number) {
if (i < 10) { if (i < 10 && i >= 0) {
return "0" + i return "0" + i
} }
return "" + i return "" + i
@ -1058,7 +1066,16 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
}) })
} }
public static toHumanTime(seconds): string { /**
* Converts a number of seconds into a human readable format
*
* Utils.toHumanTime(45) // => "0:00:45"
* Utils.toHumanTime(120) // => "0:02:00"
* Utils.toHumanTime(3610) // => "1:00:10"
* Utils.toHumanTime(25 * 3600) // => "1days 1h"
*
*/
public static toHumanTime(seconds:number): string {
seconds = Math.floor(seconds) seconds = Math.floor(seconds)
let minutes = Math.floor(seconds / 60) let minutes = Math.floor(seconds / 60)
seconds = seconds % 60 seconds = seconds % 60