NSI: add script to download logos and statistics, dynamically inject extra mappings, hide low-priority mappings if applicable

scripts/downloadNsiLogos.ts

@@ -0,0 +1,113 @@
+import Script from "./Script"
+import NameSuggestionIndex, { NSIItem } from "../src/Logic/Web/NameSuggestionIndex"
+import * as nsiWD from "../node_modules/name-suggestion-index/dist/wikidata.min.json"
+import { existsSync, writeFileSync } from "fs"
+import ScriptUtils from "./ScriptUtils"
+import { Utils } from "../src/Utils"
+import { WikimediaImageProvider } from "../src/Logic/ImageProviders/WikimediaImageProvider"
+import { renameSync } from "node:fs"
+
+export default class DownloadNsiLogos extends Script {
+    constructor() {
+        super("Downloads all images of the NSI")
+    }
+
+    private async getWikimediaUrl(startUrl: string) {
+        if (!startUrl) {
+            return startUrl
+        }
+
+
+    }
+
+    private async downloadLogo(nsiItem: NSIItem, type: string, basePath: string) {
+        try {
+            return await this.downloadLogoUnsafe(nsiItem, type, basePath)
+        } catch (e) {
+            console.error("Could not download", nsiItem.displayName, "due to", e)
+            return false
+        }
+    }
+
+    private async downloadLogoUnsafe(nsiItem: NSIItem, type: string, basePath: string) {
+        if (nsiItem === undefined) {
+            return false
+        }
+        let path = basePath + nsiItem.id
+
+        const logos = nsiWD["wikidata"][nsiItem?.tags?.[type + ":wikidata"]]?.logos
+
+        if (NameSuggestionIndex.isSvg(nsiItem, type)) {
+            path = path + ".svg"
+        }
+
+        if (existsSync(path)) {
+            return false
+        }
+
+
+        if (!logos) {
+            return false
+        }
+        if (logos.facebook) {
+            // Facebook logo's are generally better and square
+            await ScriptUtils.DownloadFileTo(logos.facebook, path)
+            return true
+        }
+        if (logos.wikidata) {
+            let url: string = logos.wikidata
+            console.log("Downloading", url)
+            let ttl = 10
+            do {
+                ttl--
+                const dloaded = await Utils.downloadAdvanced(url, {
+                    "User-Agent": "MapComplete NSI scraper/0.1 (https://github.com/pietervdvn/MapComplete; pietervdvn@posteo.net)"
+                })
+                const redirect: string | undefined = dloaded["redirect"]
+                if (redirect) {
+                    console.log("Got a redirect from", url, "to", redirect)
+                    url = redirect
+                    continue
+                }
+                if ((<string>logos.wikidata).toLowerCase().endsWith(".svg")) {
+                    console.log("Written SVG", path)
+                    if(!path.endsWith(".svg")){
+                        throw "Undetected svg path:"+logos.wikidata
+                    }
+                    writeFileSync(path, dloaded["content"], "utf8")
+                    return true
+                }
+
+                console.log("Got data from", url, "-->", path)
+                await ScriptUtils.DownloadFileTo(url, path)
+                return true
+            } while (ttl > 0)
+
+            return false
+        }
+
+        return false
+
+    }
+
+    async main(args: string[]): Promise<void> {
+        const type = "brand"
+        const items = NameSuggestionIndex.allPossible(type)
+        const basePath = "./public/assets/data/nsi/logos/"
+        let downloadCount = 0
+        const stepcount = 100
+        for (let i = 0; i < items.length; i += stepcount) {
+            if (i % 100 === 0) {
+                console.log(i + "/" + items.length, "downloaded " + downloadCount)
+            }
+            await Promise.all(Utils.TimesT(stepcount, j => j).map(async j => {
+                const downloaded = await this.downloadLogo(items[i + j], type, basePath)
+                if (downloaded) {
+                    downloadCount++
+                }
+            }))
+        }
+    }
+}
+
+new DownloadNsiLogos().run()