Refactoring: put all special visualisations into their own class, add their location into the documentation

scripts/SourceOverview.ts

@@ -0,0 +1,168 @@
+import { RasterLayerProperties } from "../src/Models/RasterLayerProperties"
+import { AvailableRasterLayers, RasterLayerPolygon } from "../src/Models/RasterLayers"
+import { Utils } from "../src/Utils"
+import * as eli from "../public/assets/data/editor-layer-index.json"
+import * as layers_global from "../src/assets/global-raster-layers.json"
+import eli_global from "../src/assets/generated/editor-layer-index-global.json"
+import bing from "../src/assets/bing.json"
+import Constants from "../src/Models/Constants"
+import ThemeConfig from "../src/Models/ThemeConfig/ThemeConfig"
+import { ThemeConfigJson } from "../src/Models/ThemeConfig/Json/ThemeConfigJson"
+import SpecialVisualizations from "../src/UI/SpecialVisualizations"
+import ValidationUtils from "../src/Models/ThemeConfig/Conversion/ValidationUtils"
+import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
+import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
+import { ImmutableStore } from "../src/Logic/UIEventSource"
+
+export interface SourceInfo {
+    host: string
+}
+
+/**
+ * Generates what URLs the webapp might access, and why
+ */
+export class SourceOverview {
+    private eliUrlsCached: string[]
+
+    public async getOverview( layout: ThemeConfig,
+                              layoutJson: ThemeConfigJson,
+                              options: {
+                                  scriptSrcs: string[]
+                              }): Promise<string[]> {
+        const apiUrls: string[] = [
+            ...Constants.allServers,
+            "https://www.openstreetmap.org",
+            "https://api.openstreetmap.org",
+            "https://pietervdvn.goatcounter.com",
+            "https://api.panoramax.xyz",
+            "https://panoramax.mapcomplete.org",
+            "https://data.velopark.be",
+            "https://data.mapcomplete.org",
+        ].concat(...(await this.eliUrls()))
+
+        for (const sv of SpecialVisualizations.specialVisualizations) {
+            if (typeof sv.needsUrls === "function") {
+                // Handled below
+                continue
+            }
+            apiUrls.push(...(sv.needsUrls ?? []))
+        }
+
+
+        const usedSpecialVisualisations = [].concat(
+            ...layoutJson.layers.map((l) =>
+                ValidationUtils.getAllSpecialVisualisations(
+                    <QuestionableTagRenderingConfigJson[]>(<LayerConfigJson>l).tagRenderings ?? []
+                )
+            )
+        )
+        for (const usedSpecialVisualisation of usedSpecialVisualisations) {
+            if (typeof usedSpecialVisualisation === "string") {
+                continue
+            }
+            const neededUrls = usedSpecialVisualisation.func.needsUrls ?? []
+            if (typeof neededUrls === "function") {
+                let needed: string | string[] = neededUrls(usedSpecialVisualisation.args)
+                if (typeof needed === "string") {
+                    needed = [needed]
+                }
+                apiUrls.push(...needed)
+            }
+        }
+
+        apiUrls.push("https://schema.org")
+        const eliLayers: RasterLayerPolygon[] = AvailableRasterLayers.layersAvailableAt(
+            new ImmutableStore({ lon: 0, lat: 0 })
+        ).store.data
+        {
+            const vectorLayers = eliLayers.filter((l) => l.properties.type === "vector")
+            const vectorSources = vectorLayers.map((l) => l.properties.url)
+            vectorSources.push(...vectorLayers.map((l) => l.properties.style))
+            apiUrls.push(
+                ...vectorSources.map((url) => {
+                    if (url?.startsWith("pmtiles://")) {
+                        return url.substring("pmtiles://".length)
+                    }
+                    return url
+                })
+            )
+        }
+        const geojsonSources: string[] = layout.layers.map((l) => l.source?.geojsonSource)
+
+        return apiUrls.concat(...geojsonSources)
+
+    }
+
+   private  async eliUrls(): Promise<string[]> {
+        if (this.eliUrlsCached) {
+            return this.eliUrlsCached
+        }
+        const urls: string[] = []
+        const regex = /{switch:([^}]+)}/
+        const rasterLayers: { properties: RasterLayerProperties }[] = [
+            AvailableRasterLayers.defaultBackgroundLayer,
+            ...eli.features,
+            bing,
+            ...eli_global.map((properties) => ({ properties })),
+            ...layers_global.layers.map((properties) => ({ properties })),
+        ]
+        for (const feature of rasterLayers) {
+            const f = <RasterLayerPolygon>feature
+            const url = f.properties.url
+            const match = url.match(regex)
+            if (match) {
+                const domains = match[1].split(",")
+                const subpart = match[0]
+                urls.push(...domains.map((d) => url.replace(subpart, d)))
+            } else {
+                urls.push(url)
+            }
+
+            if (f.properties.type === "vector") {
+                // We also need to whitelist eventual sources
+                let url = f.properties.url
+                urls.push(...(f.properties["connect-src"] ?? []))
+                if (url.startsWith("pmtiles://")) {
+                    url = url.substring("pmtiles://".length)
+                }
+                if (url.endsWith(".pmtiles")) {
+                    continue
+                }
+                try {
+                    console.log("Downloading ", url)
+                    const styleSpec = await Utils.downloadJsonCached(url, 1000 * 120, {
+                        Origin: "https://mapcomplete.org",
+                    })
+                    for (const key of Object.keys(styleSpec?.["sources"] ?? {})) {
+                        const url = styleSpec["sources"][key].url
+                        if (!url) {
+                            continue
+                        }
+                        let urlClipped = url
+                        if (url.indexOf("?") > 0) {
+                            urlClipped = url?.substring(0, url.indexOf("?"))
+                        }
+                        console.log("Source url ", key, url)
+                        urls.push(url)
+                        if (urlClipped.endsWith(".json")) {
+                            const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120, {
+                                Origin: "https://mapcomplete.org",
+                            })
+                            urls.push(tileInfo["tiles"] ?? [])
+                        }
+                    }
+                    urls.push(...(styleSpec["tiles"] ?? []))
+                    urls.push(styleSpec["sprite"])
+                    urls.push(styleSpec["glyphs"])
+                } catch (e) {
+                    console.error(
+                        "ERROR: could not download a resource, some sprites might not be whitelisted and thus not load"
+                    )
+                }
+            }
+        }
+        this.eliUrlsCached = urls
+        return Utils.NoNull(urls).sort()
+    }
+
+}