ilja_space/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Add error recovery to downloadNSI-script

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-07-08 23:13:57 +02:00
parent bf4f5ab256
commit e64d8884a3
2 changed files with 19 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -111,7 +111,7 @@
"housekeeping": "git pull && npx update-browserslist-db@latest && npm run weblate-fix-heavy && npm run generate && npm run generate:docs && npm run generate:contributor-list && vite-node scripts/fetchLanguages.ts && vite-node scripts/generateSunnyUnlabeled.ts && npm run format && git add assets/ langs/ Docs/ **/*.ts Docs/* src/* && git commit -m 'chore: automated housekeeping...'",
"reuse-compliance": "reuse lint",
"backup:images": "vite-node scripts/generateImageAnalysis.ts -- ~/data/imgur-image-backup/",
"downloadNsiLogos": "vite-node scripts/downloadNsiLogos.ts",
"downloadNsiLogos": "vite-node scripts/downloadNsiLogos.ts || npm run downloadNsiLogos # This script crashes often without the possibility to correct - hence the auto retry with OR",
"dloadVelopark": "vite-node scripts/velopark/veloParkToGeojson.ts ",
"compareVelopark": "vite-node scripts/velopark/compare.ts -- velopark_nonsynced_.geojson ~/Projecten/OSM/Fietsberaad/2024-02-02\\ Fietsenstallingen_OSM_met_velopark_ref.geojson\n",
"scrapeWebsites": "vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/",

23
scripts/downloadNsiLogos.ts
View file

@ -15,7 +15,7 @@ class DownloadNsiLogos extends Script {
return await this.downloadLogoUnsafe(nsiItem, type, basePath)
} catch (e) {
console.error("Could not download", nsiItem.displayName, "due to", e)
return false
return "error"
}
}
@ -39,7 +39,7 @@ class DownloadNsiLogos extends Script {
return false
}
if (logos.facebook) {
// Facebook logo's are generally better and square
// Facebook's logos are generally better and square
await ScriptUtils.DownloadFileTo(logos.facebook, path)
return true
}
@ -88,19 +88,32 @@ class DownloadNsiLogos extends Script {
const items = NameSuggestionIndex.allPossible(type)
const basePath = "./public/assets/data/nsi/logos/"
let downloadCount = 0
const stepcount = 100
const stepcount = 5
for (let i = 0; i < items.length; i += stepcount) {
if (i % 100 === 0) {
if (downloadCount > 0 || i % 200 === 0) {
console.log(i + "/" + items.length, "downloaded " + downloadCount)
}
await Promise.all(
const results = await Promise.all(
Utils.TimesT(stepcount, (j) => j).map(async (j) => {
const downloaded = await this.downloadLogo(items[i + j], type, basePath)
if (downloaded) {
downloadCount++
}
return downloaded
})
)
for (let j = 0; j < results.length; j++) {
let didDownload = results[j]
if(didDownload !== "error"){
continue
}
console.log("Retrying", items[i + j].id, type)
didDownload = await this.downloadLogo(items[i + j], type, basePath)
if(didDownload === "error"){
console.log("Failed again:", items[i + j].id)
}
}
}
}
}