Refactoring: put all special visualisations into their own class, add their location into the documentation

scripts/generateLayouts.ts

@@ -7,21 +7,10 @@ import ThemeConfig from "../src/Models/ThemeConfig/ThemeConfig"
 import xml2js from "xml2js"
 import ScriptUtils from "./ScriptUtils"
 import { Utils } from "../src/Utils"
-import SpecialVisualizations from "../src/UI/SpecialVisualizations"
 import Constants from "../src/Models/Constants"
-import { AvailableRasterLayers, RasterLayerPolygon } from "../src/Models/RasterLayers"
-import { ImmutableStore } from "../src/Logic/UIEventSource"
-import * as eli from "../public/assets/data/editor-layer-index.json"
-import * as layers_global from "../src/assets/global-raster-layers.json"
-import eli_global from "../src/assets/generated/editor-layer-index-global.json"
-import bing from "../src/assets/bing.json"
-
-import ValidationUtils from "../src/Models/ThemeConfig/Conversion/ValidationUtils"
-import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
-import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
 import Script from "./Script"
 import crypto from "crypto"
-import { RasterLayerProperties } from "../src/Models/RasterLayerProperties"
+import { SourceOverview } from "./SourceOverview"
 
 const sharp = require("sharp")
 
@@ -36,8 +25,6 @@ class GenerateLayouts extends Script {
         .join("\n")
     private readonly removeOtherLanguagesHash =
         "sha256-" + crypto.createHash("sha256").update(this.removeOtherLanguages).digest("base64")
-    private previousSrc: Set<string> = new Set<string>()
-    private eliUrlsCached: string[]
     private date = new Date().toISOString()
     private branchName: string = undefined
     private alreadyWritten: Set<string> = new Set<string>()
@@ -265,78 +252,7 @@ class GenerateLayouts extends Script {
         return values.join("\n")
     }
 
-    async eliUrls(): Promise<string[]> {
-        if (this.eliUrlsCached) {
-            return this.eliUrlsCached
-        }
-        const urls: string[] = []
-        const regex = /{switch:([^}]+)}/
-        const rasterLayers: { properties: RasterLayerProperties }[] = [
-            AvailableRasterLayers.defaultBackgroundLayer,
-            ...eli.features,
-            bing,
-            ...eli_global.map((properties) => ({ properties })),
-            ...layers_global.layers.map((properties) => ({ properties })),
-        ]
-        for (const feature of rasterLayers) {
-            const f = <RasterLayerPolygon>feature
-            const url = f.properties.url
-            const match = url.match(regex)
-            if (match) {
-                const domains = match[1].split(",")
-                const subpart = match[0]
-                urls.push(...domains.map((d) => url.replace(subpart, d)))
-            } else {
-                urls.push(url)
-            }
-
-            if (f.properties.type === "vector") {
-                // We also need to whitelist eventual sources
-                let url = f.properties.url
-                urls.push(...(f.properties["connect-src"] ?? []))
-                if (url.startsWith("pmtiles://")) {
-                    url = url.substring("pmtiles://".length)
-                }
-                if (url.endsWith(".pmtiles")) {
-                    continue
-                }
-                try {
-                    console.log("Downloading ", url)
-                    const styleSpec = await Utils.downloadJsonCached(url, 1000 * 120, {
-                        Origin: "https://mapcomplete.org",
-                    })
-                    for (const key of Object.keys(styleSpec?.["sources"] ?? {})) {
-                        const url = styleSpec["sources"][key].url
-                        if (!url) {
-                            continue
-                        }
-                        let urlClipped = url
-                        if (url.indexOf("?") > 0) {
-                            urlClipped = url?.substring(0, url.indexOf("?"))
-                        }
-                        console.log("Source url ", key, url)
-                        urls.push(url)
-                        if (urlClipped.endsWith(".json")) {
-                            const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120, {
-                                Origin: "https://mapcomplete.org",
-                            })
-                            urls.push(tileInfo["tiles"] ?? [])
-                        }
-                    }
-                    urls.push(...(styleSpec["tiles"] ?? []))
-                    urls.push(styleSpec["sprite"])
-                    urls.push(styleSpec["glyphs"])
-                } catch (e) {
-                    console.error(
-                        "ERROR: could not download a resource, some sprites might not be whitelisted and thus not load"
-                    )
-                }
-            }
-        }
-        this.eliUrlsCached = urls
-        return Utils.NoNull(urls).sort()
-    }
-
+    private readonly sourceOverview = new SourceOverview()
     async generateCsp(
         layout: ThemeConfig,
         layoutJson: ThemeConfigJson,
@@ -344,66 +260,10 @@ class GenerateLayouts extends Script {
             scriptSrcs: string[]
         }
     ): Promise<string> {
-        const apiUrls: string[] = [
-            ...Constants.allServers,
-            "https://www.openstreetmap.org",
-            "https://api.openstreetmap.org",
-            "https://pietervdvn.goatcounter.com",
-            "https://api.panoramax.xyz",
-            "https://panoramax.mapcomplete.org",
-            "https://data.velopark.be",
-            "https://data.mapcomplete.org",
-        ].concat(...(await this.eliUrls()))
-
-        SpecialVisualizations.specialVisualizations.forEach((sv) => {
-            if (typeof sv.needsUrls === "function") {
-                // Handled below
-                return
-            }
-            apiUrls.push(...(sv.needsUrls ?? []))
-        })
-
-        const usedSpecialVisualisations = [].concat(
-            ...layoutJson.layers.map((l) =>
-                ValidationUtils.getAllSpecialVisualisations(
-                    <QuestionableTagRenderingConfigJson[]>(<LayerConfigJson>l).tagRenderings ?? []
-                )
-            )
-        )
-        for (const usedSpecialVisualisation of usedSpecialVisualisations) {
-            if (typeof usedSpecialVisualisation === "string") {
-                continue
-            }
-            const neededUrls = usedSpecialVisualisation.func.needsUrls ?? []
-            if (typeof neededUrls === "function") {
-                let needed: string | string[] = neededUrls(usedSpecialVisualisation.args)
-                if (typeof needed === "string") {
-                    needed = [needed]
-                }
-                apiUrls.push(...needed)
-            }
-        }
-
-        const geojsonSources: string[] = layout.layers.map((l) => l.source?.geojsonSource)
+        const urls = await this.sourceOverview.getOverview(layout, layoutJson, options)
         const hosts = new Set<string>()
-        hosts.add("https://schema.org")
-        const eliLayers: RasterLayerPolygon[] = AvailableRasterLayers.layersAvailableAt(
-            new ImmutableStore({ lon: 0, lat: 0 })
-        ).store.data
-        {
-            const vectorLayers = eliLayers.filter((l) => l.properties.type === "vector")
-            const vectorSources = vectorLayers.map((l) => l.properties.url)
-            vectorSources.push(...vectorLayers.map((l) => l.properties.style))
-            apiUrls.push(
-                ...vectorSources.map((url) => {
-                    if (url?.startsWith("pmtiles://")) {
-                        return url.substring("pmtiles://".length)
-                    }
-                    return url
-                })
-            )
-        }
-        for (let connectSource of apiUrls.concat(geojsonSources)) {
+
+        for (let connectSource of urls) {
             if (!connectSource) {
                 continue
             }
@@ -425,17 +285,12 @@ class GenerateLayouts extends Script {
         }
 
         const connectSrc = Array.from(hosts).sort()
-
-        const newSrcs = connectSrc.filter((newItem) => !this.previousSrc.has(newItem))
-
         console.log(
             "Got",
             hosts.size,
             "connect-src items for theme",
             layout.id,
-            newSrcs.length > 0 ? "(extra sources: " + newSrcs.join(" ") + ")" : ""
         )
-        this.previousSrc = hosts
 
         const csp: Record<string, string> = {
             "default-src": "'self'",