MapComplete/scripts/SourceOverview.ts

import { RasterLayerProperties } from "../src/Models/RasterLayerProperties"
import { AvailableRasterLayers, RasterLayerPolygon } from "../src/Models/RasterLayers"
import { Utils } from "../src/Utils"
import * as eli from "../public/assets/data/editor-layer-index.json"
import * as layers_global from "../src/assets/global-raster-layers.json"
import eli_global from "../src/assets/generated/editor-layer-index-global.json"
import bing from "../src/assets/bing.json"
import Constants from "../src/Models/Constants"
import ThemeConfig from "../src/Models/ThemeConfig/ThemeConfig"
import { ThemeConfigJson } from "../src/Models/ThemeConfig/Json/ThemeConfigJson"
import SpecialVisualizations from "../src/UI/SpecialVisualizations"
import ValidationUtils from "../src/Models/ThemeConfig/Conversion/ValidationUtils"
import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
import { ImmutableStore } from "../src/Logic/UIEventSource"

export interface SourceInfo {
    host: string
}

/**
 * Generates what URLs the webapp might access, and why
 */
export class SourceOverview {
    private eliUrlsCached: string[]

    public async getOverview( layout: ThemeConfig,
                              layoutJson: ThemeConfigJson,
                              options: {
                                  scriptSrcs: string[]
                              }): Promise<string[]> {
        const apiUrls: string[] = [
            ...Constants.allServers,
            "https://www.openstreetmap.org",
            "https://api.openstreetmap.org",
            "https://pietervdvn.goatcounter.com",
            "https://api.panoramax.xyz",
            "https://panoramax.mapcomplete.org",
            "https://data.velopark.be",
            "https://data.mapcomplete.org",
        ].concat(...(await this.eliUrls()))

        for (const sv of SpecialVisualizations.specialVisualizations) {
            if (typeof sv.needsUrls === "function") {
                // Handled below
                continue
            }
            apiUrls.push(...(sv.needsUrls ?? []))
        }


        const usedSpecialVisualisations = [].concat(
            ...layoutJson.layers.map((l) =>
                ValidationUtils.getAllSpecialVisualisations(
                    <QuestionableTagRenderingConfigJson[]>(<LayerConfigJson>l).tagRenderings ?? []
                )
            )
        )
        for (const usedSpecialVisualisation of usedSpecialVisualisations) {
            if (typeof usedSpecialVisualisation === "string") {
                continue
            }
            const neededUrls = usedSpecialVisualisation.func.needsUrls ?? []
            if (typeof neededUrls === "function") {
                let needed: string | string[] = neededUrls(usedSpecialVisualisation.args)
                if (typeof needed === "string") {
                    needed = [needed]
                }
                apiUrls.push(...needed)
            }
        }

        apiUrls.push("https://schema.org")
        const eliLayers: RasterLayerPolygon[] = AvailableRasterLayers.layersAvailableAt(
            new ImmutableStore({ lon: 0, lat: 0 })
        ).store.data
        {
            const vectorLayers = eliLayers.filter((l) => l.properties.type === "vector")
            const vectorSources = vectorLayers.map((l) => l.properties.url)
            vectorSources.push(...vectorLayers.map((l) => l.properties.style))
            apiUrls.push(
                ...vectorSources.map((url) => {
                    if (url?.startsWith("pmtiles://")) {
                        return url.substring("pmtiles://".length)
                    }
                    return url
                })
            )
        }
        const geojsonSources: string[] = layout.layers.map((l) => l.source?.geojsonSource)

        return apiUrls.concat(...geojsonSources)

    }

   private  async eliUrls(): Promise<string[]> {
        if (this.eliUrlsCached) {
            return this.eliUrlsCached
        }
        const urls: string[] = []
        const regex = /{switch:([^}]+)}/
        const rasterLayers: { properties: RasterLayerProperties }[] = [
            AvailableRasterLayers.defaultBackgroundLayer,
            ...eli.features,
            bing,
            ...eli_global.map((properties) => ({ properties })),
            ...layers_global.layers.map((properties) => ({ properties })),
        ]
        for (const feature of rasterLayers) {
            const f = <RasterLayerPolygon>feature
            const url = f.properties.url
            const match = url.match(regex)
            if (match) {
                const domains = match[1].split(",")
                const subpart = match[0]
                urls.push(...domains.map((d) => url.replace(subpart, d)))
            } else {
                urls.push(url)
            }

            if (f.properties.type === "vector") {
                // We also need to whitelist eventual sources
                let url = f.properties.url
                urls.push(...(f.properties["connect-src"] ?? []))
                if (url.startsWith("pmtiles://")) {
                    url = url.substring("pmtiles://".length)
                }
                if (url.endsWith(".pmtiles")) {
                    continue
                }
                try {
                    console.log("Downloading ", url)
                    const styleSpec = await Utils.downloadJsonCached(url, 1000 * 120, {
                        Origin: "https://mapcomplete.org",
                    })
                    for (const key of Object.keys(styleSpec?.["sources"] ?? {})) {
                        const url = styleSpec["sources"][key].url
                        if (!url) {
                            continue
                        }
                        let urlClipped = url
                        if (url.indexOf("?") > 0) {
                            urlClipped = url?.substring(0, url.indexOf("?"))
                        }
                        console.log("Source url ", key, url)
                        urls.push(url)
                        if (urlClipped.endsWith(".json")) {
                            const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120, {
                                Origin: "https://mapcomplete.org",
                            })
                            urls.push(tileInfo["tiles"] ?? [])
                        }
                    }
                    urls.push(...(styleSpec["tiles"] ?? []))
                    urls.push(styleSpec["sprite"])
                    urls.push(styleSpec["glyphs"])
                } catch (e) {
                    console.error(
                        "ERROR: could not download a resource, some sprites might not be whitelisted and thus not load"
                    )
                }
            }
        }
        this.eliUrlsCached = urls
        return Utils.NoNull(urls).sort()
    }

}