Docs: create overview of online services (for F-Droid acceptance); inline ELI again

src/Models/SourceOverview.ts

@@ -0,0 +1,186 @@
+import { RasterLayerProperties } from "./RasterLayerProperties"
+import { AvailableRasterLayers, RasterLayerPolygon } from "./RasterLayers"
+import { Utils } from "../Utils"
+import * as eli from "../../public/assets/data/editor-layer-index.json"
+import * as layers_global from "../../src/assets/global-raster-layers.json"
+import eli_global from "../../src/assets/generated/editor-layer-index-global.json"
+import bing from "../../src/assets/bing.json"
+import Constants from "../../src/Models/Constants"
+import ThemeConfig from "../../src/Models/ThemeConfig/ThemeConfig"
+import { ThemeConfigJson } from "../../src/Models/ThemeConfig/Json/ThemeConfigJson"
+import SpecialVisualizations from "../../src/UI/SpecialVisualizations"
+import ValidationUtils from "../../src/Models/ThemeConfig/Conversion/ValidationUtils"
+import {
+    QuestionableTagRenderingConfigJson,
+} from "../../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
+import { LayerConfigJson } from "../../src/Models/ThemeConfig/Json/LayerConfigJson"
+import { ImmutableStore } from "../../src/Logic/UIEventSource"
+
+export interface ServerSourceInfo {
+    url: string
+    description: string
+    category: "core" | "feature" | "maplayer"
+    selfhostable?: boolean | string
+    sourceAvailable?: boolean | string
+    openData?: boolean | string
+    trigger?: ("always" | "specific_theme" | "specific_feature" | "clear_consent" | string)[]
+    moreInfo?: string[],
+    logging?: "yes" | "probably" | "no"
+}
+
+/**
+ * Generates what URLs the webapp might access, and why
+ */
+export class SourceOverview {
+    private eliUrlsCached: ServerSourceInfo[]
+
+    public async getOverview(layout?: ThemeConfig,
+                             layoutJson?: ThemeConfigJson): Promise<(ServerSourceInfo | string)[]> {
+        const apiUrls: (ServerSourceInfo | string)[] = [
+            ...Constants.allServers,
+
+            "https://www.openstreetmap.org",
+            "https://api.openstreetmap.org",
+            "https://panoramax.mapcomplete.org",
+
+            <ServerSourceInfo>{
+                url: "https://data.mapcomplete.org/nsi",
+                description: "Contains a copy and the logos of the Name Suggestion Index",
+                category: "core",
+            },
+        ]
+
+        apiUrls.push(...(await this.eliUrls()))
+
+        for (const sv of SpecialVisualizations.specialVisualizations) {
+            if (typeof sv.needsUrls === "function") {
+                // Handled below
+                continue
+            }
+            apiUrls.push(...(sv.needsUrls ?? []))
+        }
+
+
+        const usedSpecialVisualisations = layoutJson?.layers?.flatMap((l) =>
+            ValidationUtils.getAllSpecialVisualisations(
+                <QuestionableTagRenderingConfigJson[]>(<LayerConfigJson>l).tagRenderings ?? [],
+            ))
+
+        for (const usedSpecialVisualisation of usedSpecialVisualisations ?? []) {
+            if (typeof usedSpecialVisualisation === "string") {
+                continue
+            }
+            const neededUrls = usedSpecialVisualisation.func.needsUrls ?? []
+            if (typeof neededUrls === "function") {
+                let needed: string | string[] | ServerSourceInfo | ServerSourceInfo[] = neededUrls(usedSpecialVisualisation.args)
+                if (Array.isArray(needed)) {
+                    apiUrls.push(...needed)
+                } else {
+                    apiUrls.push(needed)
+                }
+            }
+        }
+
+        const geojsonSources: string[] = layout?.layers?.map((l) => l.source?.geojsonSource) ?? []
+
+        return Utils.NoNull(apiUrls.concat(...geojsonSources)).filter(item => {
+            if (typeof item === "string") {
+                return true
+            }
+            console.log(item)
+            return item.url?.trim()?.length > 0
+        })
+
+    }
+
+    private async eliUrls(): Promise<ServerSourceInfo[]> {
+        if (this.eliUrlsCached) {
+            return this.eliUrlsCached
+        }
+        let urls: ServerSourceInfo[] = []
+        const regex = /{switch:([^}]+)}/
+        const rasterLayers: { properties: RasterLayerProperties }[] = [
+            AvailableRasterLayers.defaultBackgroundLayer,
+            ...eli.features,
+            bing,
+            ...eli_global.map((properties) => ({ properties })),
+            ...layers_global.layers.map((properties) => ({ properties })),
+        ]
+        for (const feature of rasterLayers) {
+            const f = <RasterLayerPolygon>feature
+            const url = f.properties.url
+            const match = url.match(regex)
+
+            const packageInInfo = (url: string) => {
+                if(typeof url !== "string"){
+                    throw "invalid url"+url
+                }
+                return (<ServerSourceInfo>{
+                    url,
+                    description: "Background layer source or supporting sources for " + f.properties.id,
+                    trigger: ["specific_feature"],
+                    category: "maplayer",
+                    moreInfo: Utils.NoEmpty(["https://github.com/osmlab/editor-layer-index", f.properties?.attribution?.url]),
+                })
+            }
+
+            if (match) {
+                const domains = match[1].split(",")
+                const subpart = match[0]
+                const info: ServerSourceInfo[] = domains.map((d) => packageInInfo(url.replace(subpart, d)))
+                urls.push(...info)
+            } else {
+                urls.push(packageInInfo(url))
+            }
+
+            if (f.properties.type === "vector") {
+                // We also need to whitelist eventual sources
+                urls.push(...(f.properties["connect-src"] ?? []).map(packageInInfo))
+                let url = f.properties.url
+                if (url.startsWith("pmtiles://")) {
+                    url = url.substring("pmtiles://".length)
+                }
+                if (url.endsWith(".pmtiles")) {
+                    continue
+                }
+                try {
+                    console.log("Downloading ", url)
+                    const styleSpec = await Utils.downloadJsonCached(url, 1000 * 120, {
+                        Origin: "https://mapcomplete.org",
+                    })
+                    for (const key of Object.keys(styleSpec?.["sources"] ?? {})) {
+                        const url = styleSpec["sources"][key].url
+                        if (!url) {
+                            continue
+                        }
+                        let urlClipped = url
+                        if (url.indexOf("?") > 0) {
+                            urlClipped = url?.substring(0, url.indexOf("?"))
+                        }
+                        console.log("Source url ", key, url)
+                        urls.push(packageInInfo(url))
+                        if (urlClipped.endsWith(".json")) {
+                            const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120, {
+                                Origin: "https://mapcomplete.org",
+                            })
+                            urls.push(packageInInfo(tileInfo["tiles"] ?? []))
+                        }
+                    }
+                    urls.push(...(styleSpec["tiles"] ?? []).flatMap(ls => ls).map(url => packageInInfo(url)))
+                    urls.push(packageInInfo(styleSpec["sprite"]))
+                    urls.push(packageInInfo(styleSpec["glyphs"]))
+                } catch (e) {
+                    console.error(
+                        "ERROR: could not download a resource, some sprites might not be whitelisted and thus not load"
+                    )
+                }
+            }
+        }
+        urls = urls.filter(item => !!item.url)
+        urls.sort((a, b) => a < b ? -1 : 1)
+        urls = Utils.DedupOnId(urls, item => item.url)
+        this.eliUrlsCached = urls
+        return urls
+    }
+
+}