Osmwithspace/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Scripts(nsi): stabilize and make scripts more log-friendly

Browse source
This commit is contained in:
Pieter Vander Vennet 2025-10-16 00:43:06 +02:00
parent 53106cc0bf
commit 2b10c715b0
5 changed files with 71 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

28
scripts/ScriptUtils.ts
View file

@ -7,7 +7,11 @@ import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
import xml2js from "xml2js"
export default class ScriptUtils {
public static fixUtils() {
public static verbose = true
public static fixUtils(verbose = true) {
ScriptUtils.verbose = verbose
Utils.externalDownloadFunction = ScriptUtils.Download
}
@ -54,16 +58,26 @@ export default class ScriptUtils {
}
public static DownloadFileTo(url, targetFilePath: string): Promise<void> {
ScriptUtils.erasableLog("Downloading", url, "to", targetFilePath)
return new Promise<void>((resolve) => {
https.get(url, (res) => {
if (this.verbose) {
ScriptUtils.erasableLog("Downloading", url, "to", targetFilePath)
}
return new Promise<void>((resolve, reject) => {
const req = https.get(url, (res) => {
if (res.statusCode === 429) {
console.error(`RATE LIMITED on ${url}`)
ScriptUtils.sleep(50000).then(() => reject("rate limited"))
}
if (res.statusCode !== 200) {
req.destroy()
return reject(new Error(`HTTP ${res.statusCode} for ${url}`))
}
const filePath = fs.createWriteStream(targetFilePath)
res.pipe(filePath)
filePath.on("finish", () => {
filePath.close()
resolve()
})
})
}).on("error", (e) => reject(e))
})
}
@ -222,7 +236,9 @@ export default class ScriptUtils {
if (!headers.Accept) {
headers.accept ??= "application/json"
}
ScriptUtils.erasableLog(" > ScriptUtils.Download(", url, ")")
if (ScriptUtils.verbose) {
ScriptUtils.erasableLog(" > ScriptUtils.Download(", url, ")")
}
const urlObj = new URL(url)
const request = https.get(
{

35
scripts/generateNsiStats.ts
View file

@ -146,15 +146,10 @@ class GenerateNsiStats extends Script {
const batchSize = 16
const f = (stats) => stats.data.find((t) => t.type === "all").count
const start = new Date().getTime()
let lastBatchStart = start
const preloaded = Object.keys(allBrands).length
for (let i = 0; i < allBrandNames.length; i += batchSize) {
console.warn(
"Downloading ",
batchSize,
"occurence counts, items: ",
i + "/" + allBrandNames.length
)
let downloaded = 0
await Promise.all(
Utils.timesT(batchSize, async (j) => {
const brand = allBrandNames[i + j]
@ -162,18 +157,30 @@ class GenerateNsiStats extends Script {
allBrands[brand] = {}
}
const writeInto = allBrands[brand]
const dloaded = await TagInfo.getGlobalDistributionsFor(
await TagInfo.getGlobalDistributionsFor(
writeInto,
f,
type,
brand
)
downloaded += dloaded
})
)
console.log("Downloaded ", downloaded, " values this batch")
writeFileSync(path, JSON.stringify(allBrands), "utf8")
console.log("Checkpointed", path)
if (i > preloaded && (i / batchSize) % 10 === 0) {
const now = new Date().getTime()
const elapsed = (now - start) / 1000
const speed = (i - preloaded) / elapsed
const restingItems = allBrandNames.length - i
const restingSeconds = restingItems / speed
const elapsedB = (now - lastBatchStart) / 1000
console.log(`Downloaded ${i}/${allBrandNames.length} of category ${type} (from checkpoint: ${preloaded}), elapsed ${Utils.toHumanTime(elapsed)} (last batch: ${elapsedB}); speed: ${Math.floor(speed * 1000)}items/millisec (last batch: ${Math.floor((i - preloaded) / (elapsedB * 1000))}); estimated left: ${Utils.toHumanTime(restingSeconds)}`)
lastBatchStart = new Date().getTime()
} else {
process.stdout.write(".")
}
}
console.log("Written:", path)
writeFileSync(path, JSON.stringify(allBrands), "utf8")
@ -181,11 +188,13 @@ class GenerateNsiStats extends Script {
constructor() {
super(
"Downloads stats on osmSource-tags and keys from tagInfo. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index "
"Downloads stats on osmSource-tags and keys from tagInfo; generates 'key_totals.json' and '*.summarized.json'. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index ",
)
}
async main() {
ScriptUtils.verbose = false
const target = "./public/assets/data/nsi/"
const basepath = target + "stats/"
{

11
scripts/nsiLogos.ts
View file

@ -12,10 +12,12 @@ import { FilterConfigOptionJson } from "../src/Models/ThemeConfig/Json/FilterCon
import { TagUtils } from "../src/Logic/Tags/TagUtils"
import { openSync, readSync } from "node:fs"
import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import Constants from "../src/Models/Constants"
class NsiLogos extends Script {
constructor() {
super("Contains various subcommands for NSI logo maintainance")
ScriptUtils.verbose = false
}
private async downloadLogo(
@ -38,7 +40,7 @@ class NsiLogos extends Script {
}
}
private async downloadLogoUnsafe(nsiItem: NSIItem, type: string, basePath: string) {
private async downloadLogoUnsafe(nsiItem: NSIItem, type: string, basePath: string) : Promise<true | false | "error">{
if (nsiItem === undefined) {
return false
}
@ -86,7 +88,7 @@ class NsiLogos extends Script {
}
if ((<string>logos.wikidata).toLowerCase().endsWith(".svg")) {
if (!path.endsWith(".svg")) {
throw "Undetected svg path:" + logos.wikidata
return false // We don't supports svgs
}
writeFileSync(path, dloaded["content"], "utf8")
return true
@ -242,6 +244,7 @@ class NsiLogos extends Script {
}
private async download() {
ScriptUtils.verbose = false
const types = ["brand", "operator"]
let dload = 0
let failed = 0
@ -393,7 +396,7 @@ class NsiLogos extends Script {
}
private commands: Record<string, { f: () => Promise<void>; doc?: string }> = {
download: { f: () => this.download(), doc: "Download all icons" },
download: { f: () => this.download(), doc: "Download all icons. Gets the index from "+Constants.nsiLogosEndpoint+" first, then fetches the corresponding logos" },
generateRenderings: {
f: () => this.generateRenderings(),
doc: "Generates the layer files 'nsi_brand.json' and 'nsi_operator.json' which allows to reuse the icons in renderings",
@ -405,7 +408,7 @@ class NsiLogos extends Script {
},
patch: {
f: () => this.patchNsiFile(),
doc: "Reads nsi.min.json, adds the 'ext' (extension) field to every relevant entry",
doc: "Modifies nsi.min.json, adds the 'ext' (extension) field to every relevant entry",
},
all: {
doc: "Run `download`, `generateRenderings`, `prune` and `addExtensions`",

1
scripts/prepare-build.sh
View file

@ -17,6 +17,7 @@ sed "s/= \"0.0.0\"/= \"$VERSION\"/" -i public/service-worker.js
# This script ends every line with '&&' to chain everything. A failure will thus stop the build
npm run download:editor-layer-index &&
npm i name-suggestion-index && npx vite-node scripts/nsiLogos.ts -- generateRenderings
npm run prep:layeroverview &&
# Done by 'deploy_hosted':
# npm run generate:licenses; npm run generate:images; npm run generate:charging-stations; npm run generate:service-worker; npm run download:editor-layer-index

21
src/Utils.ts
View file

@ -127,8 +127,16 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
return str.substring(0, 1).toUpperCase() + str.substring(1)
}
/**
* Always prints (at least) digits
*
* Utils.twoDigits(5) // => "05"
* Utils.twoDigits(0) // => "00"
* Utils.twoDigits(128) // => "128"
* Utils.twoDigits(-5) // => "-5"
*/
public static twoDigits(i: number) {
if (i < 10) {
if (i < 10 && i >= 0) {
return "0" + i
}
return "" + i
@ -1058,7 +1066,16 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
})
}
public static toHumanTime(seconds): string {
/**
* Converts a number of seconds into a human readable format
*
* Utils.toHumanTime(45) // => "0:00:45"
* Utils.toHumanTime(120) // => "0:02:00"
* Utils.toHumanTime(3610) // => "1:00:10"
* Utils.toHumanTime(25 * 3600) // => "1days 1h"
*
*/
public static toHumanTime(seconds:number): string {
seconds = Math.floor(seconds)
let minutes = Math.floor(seconds / 60)
seconds = seconds % 60