Docs: create overview of online services (for F-Droid acceptance); inline ELI again

src/Models/Constants.ts

@@ -1,7 +1,7 @@
 import * as packagefile from "../../package.json"
 import * as extraconfig from "../../config.json"
-import { Utils } from "../Utils"
 import { AuthConfig } from "../Logic/Osm/AuthConfig"
+import { ServerSourceInfo } from "./SourceOverview"
 
 export type PriviligedLayerType = (typeof Constants.priviliged_layers)[number]
 export type DefaultPinIcon = (typeof Constants._defaultPinIcons)[number]
@@ -55,7 +55,14 @@ export default class Constants {
         token: string
         sequence: string
         testsequence: string
-    } = packagefile.config.panoramax
+    } & ServerSourceInfo = { ...packagefile.config.panoramax,
+    description: "The panoramax-server that MapComplete uploads to",
+        category: "core",
+        sourceAvailable: true,
+        openData: true,
+        selfhostable: true,
+        moreInfo: ["https://wiki.openstreetmap.org/wiki/Panoramax"]
+    }
 
     // The user journey states thresholds when a new feature gets unlocked
     public static userJourney = {
@@ -134,15 +141,68 @@ export default class Constants {
     public static readonly mapillary_client_token_v4 = Constants.config.api_keys.mapillary_v4
     public static defaultOverpassUrls = Constants.config.default_overpass_urls
     public static countryCoderEndpoint: string = Constants.config.country_coder_host
+    public static countryCoderInfo: ServerSourceInfo = {
+        url: this.countryCoderEndpoint,
+        trigger: ["always"],
+        openData: true,
+        category: "core",
+        logging: "no",
+        selfhostable: true,
+        sourceAvailable: true,
+        moreInfo:["https://source.mapcomplete.org/MapComplete/latlon2country"],
+        description: "For quite some functions, we need to know in what _country_ a feature is located. LatLon2Country is a static dataset, which, by cleverly encoding the data, can quickly tell in what country a feature is located."
+    }
     public static communityIndexHost: string = Constants.config.community_index_host
 
-    public static osmAuthConfig: AuthConfig = Constants.config.oauth_credentials
+    private static osmServerInfo: Omit<ServerSourceInfo, "url"> = { trigger: ["always"],
+        description: "Login service, by OpenStreetMap.org",
+        sourceAvailable: true,
+        openData: true,
+        selfhostable: "partially - a copy can be hosted, but this would be useless",
+        category: "core",
+        moreInfo: ["https://www.openstreetmap.org/copyright","https://www.openstreetmap.org/about", "https://osmfoundation.org/wiki/Privacy_Policy"]
+    }
+
+    public static osmAuthConfig: AuthConfig & ServerSourceInfo= {... Constants.config.oauth_credentials,
+        ...this.osmServerInfo
+    }
     public static nominatimEndpoint: string = Constants.config.nominatimEndpoint
+    public static nominatimEndpointInfo: ServerSourceInfo = {
+        url: this.nominatimEndpoint,
+        description: "Nominatim search engine endpoint, used when searching",
+        selfhostable: true,
+        openData: true,
+        sourceAvailable: true,
+        category: "core",
+        trigger:["specific_feature"],
+        moreInfo: ["https://wiki.openstreetmap.org/wiki/Nominatim"]
+    }
     public static photonEndpoint: string = Constants.config.photonEndpoint
+    public static photonEndpointInfo: ServerSourceInfo = {
+        url: this.photonEndpoint,
+        description: "Endpoint for search with photon",
+        sourceAvailable: true,
+        selfhostable: true,
+        openData: true,
+        category: "core",
+        trigger: ["specific_feature"],
+        moreInfo: ["https://wiki.openstreetmap.org/wiki/Photon"]
+    }
     public static nsiLogosEndpoint: string = Constants.config.nsi_logos_server ?? null
     public static weblate: string = "https://translate.mapcomplete.org/"
 
     public static linkedDataProxy: string = Constants.config["jsonld-proxy"]
+    public static linkedDataProxyInfo: ServerSourceInfo = {
+        url: Constants.config["jsonld-proxy"],
+        trigger: ["specific_feature"],
+        category: "core",
+        openData: true,
+        description: "This proxy queries websites to detect if they contain linked open data and gives this data back. Triggered by opening a feature",
+        sourceAvailable: true,
+        selfhostable: true,
+        moreInfo: ["https://source.mapcomplete.org/MapComplete/MapComplete/src/branch/develop/scripts/serverLdScrape.ts"]
+    }
+
     /**
      * These are the values that are allowed to use as 'backdrop' icon for a map pin
      */
@@ -208,22 +268,65 @@ export default class Constants {
      * This is a MapLibre/MapBox vector tile server which hosts vector tiles for every (official) layer
      */
     public static VectorTileServer: string | undefined = Constants.config.mvt_layer_server
+    public static vectorTileServerInfo: ServerSourceInfo = {
+        url: this.VectorTileServer,
+        description: "The vectortileserver is a cache of OSM data and can be used as an alternative for overpass to actually show data, esp on low zoom levels",
+        selfhostable: true,
+        openData: true,
+        category: "core",
+        sourceAvailable: true,
+        trigger: ["always"],
+        moreInfo: ["https://source.mapcomplete.org/MapComplete/MapComplete/src/branch/develop/Docs/SettingUpPSQL.md"]
+    }
     public static GeoIpServer: string | undefined = Constants.config.geoip_server
+    public static geoIpServerInfo: ServerSourceInfo = {
+        url: this.GeoIpServer,
+        category: "core",
+        description: "When opening MapComplete for the first time, we try to set the map at a relevant location. For this, we try to determine the location based on the IP-address; this is done by this service.",
+        selfhostable: true,
+        openData: false,
+        trigger: ["always"],
+        sourceAvailable: true
+    }
     public static ErrorReportServer: string | undefined = Constants.config.error_server
-
+    public static errorReportServerInfo: ServerSourceInfo = {
+        url: this.ErrorReportServer,
+        logging: "yes",
+        category: "core",
+        selfhostable: "yes",
+        openData: "no (privacy)",
+        trigger: ["on_failure"],
+        description: "If a severe error occurs in MapComplete, this is logged on this server - this mostly concerns errors where making a change to OpenStreetMap failed. Data is handled confidentially and _only_ to replay the change and fix the root cause."
+    }
     public static readonly SummaryServer: string = Constants.config.summary_server
-
-    public static allServers: string[] = [
-        Constants.SummaryServer,
-        Constants.VectorTileServer,
-        Constants.GeoIpServer,
-        Constants.ErrorReportServer,
-        Constants.countryCoderEndpoint,
-        Constants.osmAuthConfig.url,
-        Constants.nominatimEndpoint,
-        Constants.photonEndpoint,
-        Constants.linkedDataProxy,
-        ...Constants.defaultOverpassUrls,
+    public static readonly summaryServerInfo: ServerSourceInfo = {
+        url: this.SummaryServer,
+        trigger: ["always"],
+        category: "core",
+        selfhostable: true,
+        sourceAvailable: true,
+        openData: true,
+        description: "This server indicates how much items there are (according to OpenStreetMap) at a given slippy tile coordinate"
+    }
+    public static allServers: ServerSourceInfo[] = [
+        Constants.summaryServerInfo,
+        Constants.vectorTileServerInfo,
+        Constants.geoIpServerInfo,
+        Constants.errorReportServerInfo,
+        Constants.osmAuthConfig,
+        Constants.countryCoderInfo,
+        Constants.nominatimEndpointInfo,
+        Constants.photonEndpointInfo,
+        ...Constants.defaultOverpassUrls.map(url => (<ServerSourceInfo>{
+            url,
+                openData: true,
+            selfhostable: true,
+            trigger: ["always"],
+            sourceAvailable: true,
+            category: "core",
+            description: "Overpass is a query service where OpenStreetMap-data can be retrieved. Various overpass-servers are used to query this data",
+            moreInfo: ["https://wiki.openstreetmap.org/wiki/Overpass_turbo"]
+        })),
     ]
 
     private static priviligedLayerSet = new Set<string>(Constants.priviliged_layers)

src/Models/RasterLayers.ts

@@ -9,25 +9,14 @@ import { Store, Stores, UIEventSource } from "../Logic/UIEventSource"
 import { GeoOperations } from "../Logic/GeoOperations"
 import { EliCategory, RasterLayerProperties } from "./RasterLayerProperties"
 import { Utils } from "../Utils"
-
+import {default as ELI} from "../../public/assets/data/editor-layer-index.json"
 export type EditorLayerIndex = (Feature<Polygon, EditorLayerIndexProperties> & RasterLayerPolygon)[]
 
 export class AvailableRasterLayers {
-    private static _editorLayerIndex: EditorLayerIndex = undefined
-    private static _editorLayerIndexStore: UIEventSource<EditorLayerIndex> =
-        new UIEventSource<EditorLayerIndex>(undefined)
+    private static _editorLayerIndex: EditorLayerIndex = <EditorLayerIndex> ELI.features
 
-    public static async editorLayerIndex(): Promise<EditorLayerIndex> {
-        if (AvailableRasterLayers._editorLayerIndex !== undefined) {
-            return AvailableRasterLayers._editorLayerIndex
-        }
-        console.debug("Downloading ELI")
-        const eli = await Utils.downloadJson<{ features: EditorLayerIndex }>(
-            "./assets/data/editor-layer-index.json"
-        )
-        this._editorLayerIndex = eli.features?.filter((l) => l.properties.id !== "Bing") ?? []
-        this._editorLayerIndexStore.set(this._editorLayerIndex)
-        return this._editorLayerIndex
+    public static editorLayerIndex(): EditorLayerIndex {
+        return AvailableRasterLayers._editorLayerIndex
     }
 
     public static readonly globalLayers: ReadonlyArray<RasterLayerPolygon> =
@@ -101,18 +90,13 @@ export class AvailableRasterLayers {
         location: Store<{ lon: number; lat: number }>,
         enableBing?: Store<boolean>
     ): Store<RasterLayerPolygon[]> {
-        this.editorLayerIndex() // start the download
         const availableLayersBboxes = Stores.ListStabilized(
             location.mapD(
                 (loc) => {
-                    const eli = AvailableRasterLayers._editorLayerIndexStore.data
-                    if (!eli) {
-                        return []
-                    }
+                    const eli = AvailableRasterLayers._editorLayerIndex
                     const lonlat: [number, number] = [loc.lon, loc.lat]
                     return eli.filter((eliPolygon) => BBox.get(eliPolygon).contains(lonlat))
-                },
-                [AvailableRasterLayers._editorLayerIndexStore]
+                }
             )
         )
         return Stores.ListStabilized(

src/Models/SourceOverview.ts

@@ -0,0 +1,186 @@
+import { RasterLayerProperties } from "./RasterLayerProperties"
+import { AvailableRasterLayers, RasterLayerPolygon } from "./RasterLayers"
+import { Utils } from "../Utils"
+import * as eli from "../../public/assets/data/editor-layer-index.json"
+import * as layers_global from "../../src/assets/global-raster-layers.json"
+import eli_global from "../../src/assets/generated/editor-layer-index-global.json"
+import bing from "../../src/assets/bing.json"
+import Constants from "../../src/Models/Constants"
+import ThemeConfig from "../../src/Models/ThemeConfig/ThemeConfig"
+import { ThemeConfigJson } from "../../src/Models/ThemeConfig/Json/ThemeConfigJson"
+import SpecialVisualizations from "../../src/UI/SpecialVisualizations"
+import ValidationUtils from "../../src/Models/ThemeConfig/Conversion/ValidationUtils"
+import {
+    QuestionableTagRenderingConfigJson,
+} from "../../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
+import { LayerConfigJson } from "../../src/Models/ThemeConfig/Json/LayerConfigJson"
+import { ImmutableStore } from "../../src/Logic/UIEventSource"
+
+export interface ServerSourceInfo {
+    url: string
+    description: string
+    category: "core" | "feature" | "maplayer"
+    selfhostable?: boolean | string
+    sourceAvailable?: boolean | string
+    openData?: boolean | string
+    trigger?: ("always" | "specific_theme" | "specific_feature" | "clear_consent" | string)[]
+    moreInfo?: string[],
+    logging?: "yes" | "probably" | "no"
+}
+
+/**
+ * Generates what URLs the webapp might access, and why
+ */
+export class SourceOverview {
+    private eliUrlsCached: ServerSourceInfo[]
+
+    public async getOverview(layout?: ThemeConfig,
+                             layoutJson?: ThemeConfigJson): Promise<(ServerSourceInfo | string)[]> {
+        const apiUrls: (ServerSourceInfo | string)[] = [
+            ...Constants.allServers,
+
+            "https://www.openstreetmap.org",
+            "https://api.openstreetmap.org",
+            "https://panoramax.mapcomplete.org",
+
+            <ServerSourceInfo>{
+                url: "https://data.mapcomplete.org/nsi",
+                description: "Contains a copy and the logos of the Name Suggestion Index",
+                category: "core",
+            },
+        ]
+
+        apiUrls.push(...(await this.eliUrls()))
+
+        for (const sv of SpecialVisualizations.specialVisualizations) {
+            if (typeof sv.needsUrls === "function") {
+                // Handled below
+                continue
+            }
+            apiUrls.push(...(sv.needsUrls ?? []))
+        }
+
+
+        const usedSpecialVisualisations = layoutJson?.layers?.flatMap((l) =>
+            ValidationUtils.getAllSpecialVisualisations(
+                <QuestionableTagRenderingConfigJson[]>(<LayerConfigJson>l).tagRenderings ?? [],
+            ))
+
+        for (const usedSpecialVisualisation of usedSpecialVisualisations ?? []) {
+            if (typeof usedSpecialVisualisation === "string") {
+                continue
+            }
+            const neededUrls = usedSpecialVisualisation.func.needsUrls ?? []
+            if (typeof neededUrls === "function") {
+                let needed: string | string[] | ServerSourceInfo | ServerSourceInfo[] = neededUrls(usedSpecialVisualisation.args)
+                if (Array.isArray(needed)) {
+                    apiUrls.push(...needed)
+                } else {
+                    apiUrls.push(needed)
+                }
+            }
+        }
+
+        const geojsonSources: string[] = layout?.layers?.map((l) => l.source?.geojsonSource) ?? []
+
+        return Utils.NoNull(apiUrls.concat(...geojsonSources)).filter(item => {
+            if (typeof item === "string") {
+                return true
+            }
+            console.log(item)
+            return item.url?.trim()?.length > 0
+        })
+
+    }
+
+    private async eliUrls(): Promise<ServerSourceInfo[]> {
+        if (this.eliUrlsCached) {
+            return this.eliUrlsCached
+        }
+        let urls: ServerSourceInfo[] = []
+        const regex = /{switch:([^}]+)}/
+        const rasterLayers: { properties: RasterLayerProperties }[] = [
+            AvailableRasterLayers.defaultBackgroundLayer,
+            ...eli.features,
+            bing,
+            ...eli_global.map((properties) => ({ properties })),
+            ...layers_global.layers.map((properties) => ({ properties })),
+        ]
+        for (const feature of rasterLayers) {
+            const f = <RasterLayerPolygon>feature
+            const url = f.properties.url
+            const match = url.match(regex)
+
+            const packageInInfo = (url: string) => {
+                if(typeof url !== "string"){
+                    throw "invalid url"+url
+                }
+                return (<ServerSourceInfo>{
+                    url,
+                    description: "Background layer source or supporting sources for " + f.properties.id,
+                    trigger: ["specific_feature"],
+                    category: "maplayer",
+                    moreInfo: Utils.NoEmpty(["https://github.com/osmlab/editor-layer-index", f.properties?.attribution?.url]),
+                })
+            }
+
+            if (match) {
+                const domains = match[1].split(",")
+                const subpart = match[0]
+                const info: ServerSourceInfo[] = domains.map((d) => packageInInfo(url.replace(subpart, d)))
+                urls.push(...info)
+            } else {
+                urls.push(packageInInfo(url))
+            }
+
+            if (f.properties.type === "vector") {
+                // We also need to whitelist eventual sources
+                urls.push(...(f.properties["connect-src"] ?? []).map(packageInInfo))
+                let url = f.properties.url
+                if (url.startsWith("pmtiles://")) {
+                    url = url.substring("pmtiles://".length)
+                }
+                if (url.endsWith(".pmtiles")) {
+                    continue
+                }
+                try {
+                    console.log("Downloading ", url)
+                    const styleSpec = await Utils.downloadJsonCached(url, 1000 * 120, {
+                        Origin: "https://mapcomplete.org",
+                    })
+                    for (const key of Object.keys(styleSpec?.["sources"] ?? {})) {
+                        const url = styleSpec["sources"][key].url
+                        if (!url) {
+                            continue
+                        }
+                        let urlClipped = url
+                        if (url.indexOf("?") > 0) {
+                            urlClipped = url?.substring(0, url.indexOf("?"))
+                        }
+                        console.log("Source url ", key, url)
+                        urls.push(packageInInfo(url))
+                        if (urlClipped.endsWith(".json")) {
+                            const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120, {
+                                Origin: "https://mapcomplete.org",
+                            })
+                            urls.push(packageInInfo(tileInfo["tiles"] ?? []))
+                        }
+                    }
+                    urls.push(...(styleSpec["tiles"] ?? []).flatMap(ls => ls).map(url => packageInInfo(url)))
+                    urls.push(packageInInfo(styleSpec["sprite"]))
+                    urls.push(packageInInfo(styleSpec["glyphs"]))
+                } catch (e) {
+                    console.error(
+                        "ERROR: could not download a resource, some sprites might not be whitelisted and thus not load"
+                    )
+                }
+            }
+        }
+        urls = urls.filter(item => !!item.url)
+        urls.sort((a, b) => a < b ? -1 : 1)
+        urls = Utils.DedupOnId(urls, item => item.url)
+        this.eliUrlsCached = urls
+        return urls
+    }
+
+}

src/Models/ThemeConfig/TagRenderingConfig.ts

@@ -698,7 +698,7 @@ export default class TagRenderingConfig {
                 values,
             }
         } catch (e) {
-            console.error("Could not create FreeformValues for tagrendering", this.id)
+            console.error("Could not create FreeformValues for tagrendering", this.id, e)
             return undefined
         }
     }