MapComplete/src/Models/SourceOverview.ts

import { RasterLayerProperties } from "./RasterLayerProperties"
import { AvailableRasterLayers, RasterLayerPolygon } from "./RasterLayers"
import { Utils } from "../Utils"
import * as eli from "../../public/assets/data/editor-layer-index.json"
import * as layers_global from "../../src/assets/global-raster-layers.json"
import eli_global from "../../src/assets/generated/editor-layer-index-global.json"
import bing from "../../src/assets/bing.json"
import Constants from "../../src/Models/Constants"
import ThemeConfig from "../../src/Models/ThemeConfig/ThemeConfig"
import { ThemeConfigJson } from "../../src/Models/ThemeConfig/Json/ThemeConfigJson"
import SpecialVisualizations from "../../src/UI/SpecialVisualizations"
import ValidationUtils from "../../src/Models/ThemeConfig/Conversion/ValidationUtils"
import {
    QuestionableTagRenderingConfigJson
} from "../../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import { LayerConfigJson } from "../../src/Models/ThemeConfig/Json/LayerConfigJson"

export interface ServerSourceInfo {
    url: string
    description: string
    category: "core" | "feature" | "maplayer"
    selfhostable?: boolean | string
    sourceAvailable?: boolean | string
    openData?: boolean | string
    trigger?: ("always" | "specific_theme" | "specific_feature" | "clear_consent" | string)[]
    moreInfo?: string[]
    logging?: "yes" | "probably" | "no"
}

/**
 * Generates what URLs the webapp might access, and why
 */
export class SourceOverview {
    private eliUrlsCached: ServerSourceInfo[]

    public async getOverview(
        layout?: ThemeConfig,
        layoutJson?: ThemeConfigJson
    ): Promise<(ServerSourceInfo | string)[]> {
        const apiUrls: (ServerSourceInfo | string)[] = [
            ...Constants.allServers,
            <ServerSourceInfo>{
                url: "https://data.mapcomplete.org/nsi",
                description: "Contains a copy and the logos of the Name Suggestion Index",
                category: "core",
            },
        ]

        apiUrls.push(...(await this.eliUrls()))

        for (const sv of SpecialVisualizations.specialVisualizations) {
            if (typeof sv.needsUrls === "function") {
                // Handled below
                continue
            }
            apiUrls.push(...(sv.needsUrls ?? []))
        }

        const usedSpecialVisualisations = layoutJson?.layers?.flatMap((l) =>
            ValidationUtils.getAllSpecialVisualisations(
                <QuestionableTagRenderingConfigJson[]>(<LayerConfigJson>l).tagRenderings ?? []
            )
        )

        for (const usedSpecialVisualisation of usedSpecialVisualisations ?? []) {
            if (typeof usedSpecialVisualisation === "string") {
                continue
            }
            const neededUrls = usedSpecialVisualisation.func.needsUrls ?? []
            if (typeof neededUrls === "function") {
                const needed: string | string[] | ServerSourceInfo | ServerSourceInfo[] = neededUrls(
                    usedSpecialVisualisation.args
                )
                if (Array.isArray(needed)) {
                    apiUrls.push(...needed)
                } else {
                    apiUrls.push(needed)
                }
            }
        }

        const geojsonSources: string[] = layout?.layers?.map((l) => l.source?.geojsonSource) ?? []

        return Utils.NoNull(apiUrls.concat(...geojsonSources)).filter((item) => {
            if (typeof item === "string") {
                return true
            }
            return item.url?.trim()?.length > 0
        })
    }

    private async eliUrls(): Promise<ServerSourceInfo[]> {
        if (this.eliUrlsCached) {
            return this.eliUrlsCached
        }
        let urls: ServerSourceInfo[] = []
        const regex = /{switch:([^}]+)}/
        const rasterLayers: { properties: RasterLayerProperties }[] = [
            AvailableRasterLayers.defaultBackgroundLayer,
            ...eli.features,
            bing,
            ...eli_global.map((properties) => ({ properties })),
            ...layers_global.layers.map((properties) => ({ properties })),
        ]
        for (const feature of rasterLayers) {
            const f = <RasterLayerPolygon>feature
            const url = f.properties.url
            const match = url.match(regex)

            const packageInInfo: (url: (string | string[])) => ServerSourceInfo[] = (urls: string | string[]) => {
                if (typeof urls === "string") {
                    urls = [urls]
                }
                return urls.map(url => <ServerSourceInfo>{
                    url,
                    description:
                        "Background layer source or supporting sources for " + f.properties.id,
                    trigger: ["specific_feature"],
                    category: "maplayer",
                    moreInfo: Utils.NoEmpty([
                        "https://github.com/osmlab/editor-layer-index",
                        f.properties?.attribution?.url,
                    ]),
                })
            }

            const packageInInfoD: (url: (string | string[])) => (ServerSourceInfo[]) = (url: string | string[]) => {
                if (!url) {
                    return []
                }
                return packageInInfo(url)
            }

            if (match) {
                const domains = match[1].split(",")
                const subpart = match[0]
                const info: ServerSourceInfo[] = domains.flatMap((d) =>
                    packageInInfo(url.replace(subpart, d))
                )
                urls.push(...info)
            } else {
                urls.push(...packageInInfo(url))
            }

            if (f.properties.type === "vector") {
                // We also need to whitelist eventual sources
                urls.push(...(f.properties["connect-src"] ?? []).map(packageInInfo))
                let url = f.properties.url
                if (url.startsWith("pmtiles://")) {
                    url = url.substring("pmtiles://".length)
                }
                if (url.endsWith(".pmtiles")) {
                    continue
                }
                try {
                    console.log("Downloading ", url)
                    urls.push(...packageInInfo(url))
                    const styleSpec = await Utils.downloadJsonCached(url, 1000 * 120, {
                        Origin: "https://mapcomplete.org",
                    })
                    for (const key of Object.keys(styleSpec?.["sources"] ?? {})) {
                        let url = styleSpec["sources"][key].url
                        if (!url) {
                            continue
                        }
                        console.log(">>> SPlitting", url)
                        const split = url.split(",https://")
                        console.log("Multisplit:", split)
                        url = split[0]
                        let urlClipped = url
                        if (url.indexOf("?") > 0) {
                            urlClipped = url?.substring(0, url.indexOf("?"))
                        }
                        urls.push(...packageInInfo(url))
                        if (urlClipped.endsWith(".json")) {
                            const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120, {
                                Origins: "https://mapcomplete.org"
                            })
                            urls.push(...packageInInfo(tileInfo["tiles"] ?? []))
                        }
                    }
                    urls.push(
                        ...(styleSpec["tiles"] ?? [])
                            .flatMap((ls) => ls)
                            .map((url) => packageInInfoD(url))
                    )
                    urls.push(...packageInInfoD(styleSpec["sprite"]))
                    urls.push(...packageInInfoD(styleSpec["glyphs"]))
                } catch (e) {
                    console.error(e)
                    console.error(
                        "ERROR: could not download a resource: " + url + "\nSome sprites might not be whitelisted and thus not load")
                }
            }
        }
        urls = urls.filter((item) => !!item?.url)
        urls.sort((a, b) => (a < b ? -1 : 1))
        urls = Utils.DedupOnId(urls, (item) => item.url)
        this.eliUrlsCached = urls
        return urls
    }
}