Osmwithspace/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Scripts(nsi): stabilize and make scripts more log-friendly

Browse source
This commit is contained in:
Pieter Vander Vennet 2025-10-16 00:43:06 +02:00
parent 53106cc0bf
commit 2b10c715b0
5 changed files with 71 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

35
scripts/generateNsiStats.ts
View file

@ -146,15 +146,10 @@ class GenerateNsiStats extends Script {
const batchSize = 16
const f = (stats) => stats.data.find((t) => t.type === "all").count
const start = new Date().getTime()
let lastBatchStart = start
const preloaded = Object.keys(allBrands).length
for (let i = 0; i < allBrandNames.length; i += batchSize) {
console.warn(
"Downloading ",
batchSize,
"occurence counts, items: ",
i + "/" + allBrandNames.length
)
let downloaded = 0
await Promise.all(
Utils.timesT(batchSize, async (j) => {
const brand = allBrandNames[i + j]
@ -162,18 +157,30 @@ class GenerateNsiStats extends Script {
allBrands[brand] = {}
}
const writeInto = allBrands[brand]
const dloaded = await TagInfo.getGlobalDistributionsFor(
await TagInfo.getGlobalDistributionsFor(
writeInto,
f,
type,
brand
)
downloaded += dloaded
})
)
console.log("Downloaded ", downloaded, " values this batch")
writeFileSync(path, JSON.stringify(allBrands), "utf8")
console.log("Checkpointed", path)
if (i > preloaded && (i / batchSize) % 10 === 0) {
const now = new Date().getTime()
const elapsed = (now - start) / 1000
const speed = (i - preloaded) / elapsed
const restingItems = allBrandNames.length - i
const restingSeconds = restingItems / speed
const elapsedB = (now - lastBatchStart) / 1000
console.log(`Downloaded ${i}/${allBrandNames.length} of category ${type} (from checkpoint: ${preloaded}), elapsed ${Utils.toHumanTime(elapsed)} (last batch: ${elapsedB}); speed: ${Math.floor(speed * 1000)}items/millisec (last batch: ${Math.floor((i - preloaded) / (elapsedB * 1000))}); estimated left: ${Utils.toHumanTime(restingSeconds)}`)
lastBatchStart = new Date().getTime()
} else {
process.stdout.write(".")
}
}
console.log("Written:", path)
writeFileSync(path, JSON.stringify(allBrands), "utf8")
@ -181,11 +188,13 @@ class GenerateNsiStats extends Script {
constructor() {
super(
"Downloads stats on osmSource-tags and keys from tagInfo. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index "
"Downloads stats on osmSource-tags and keys from tagInfo; generates 'key_totals.json' and '*.summarized.json'. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index ",
)
}
async main() {
ScriptUtils.verbose = false
const target = "./public/assets/data/nsi/"
const basepath = target + "stats/"
{