Docs: create overview of online services (for F-Droid acceptance); inline ELI again

scripts/generateDocs.ts

@@ -33,6 +33,7 @@ import { AvailableRasterLayers } from "../src/Models/RasterLayers"
 import { ImmutableStore } from "../src/Logic/UIEventSource"
 import * as unitUsage from "../Docs/Schemas/UnitConfigJson.schema.json"
 import { ThemeConfigJson } from "../src/Models/ThemeConfig/Json/ThemeConfigJson"
+import { ServerSourceInfo, SourceOverview } from "../src/Models/SourceOverview"
 
 /**
  * Converts a markdown-file into a .json file, which a walkthrough/slideshow element can use
@@ -169,6 +170,7 @@ export class GenerateDocs extends Script {
         this.generateOverviewsForAllSingleLayer()
         this.generateLayerOverviewText()
         this.generateBuiltinUnits()
+        await this.generateSourcesOverview()
 
         Array.from(AllKnownLayouts.allKnownLayouts.values()).map((theme) => {
             this.generateForTheme(theme)
@@ -189,7 +191,7 @@ export class GenerateDocs extends Script {
             "src/Logic/Osm/Changes.ts",
             "src/Logic/Osm/ChangesetHandler.ts",
         ])
-        const eli = await AvailableRasterLayers.editorLayerIndex()
+        const eli = AvailableRasterLayers.editorLayerIndex()
         this.WriteMarkdownFile(
             "./Docs/ELI-overview.md",
             [
@@ -536,6 +538,106 @@ export class GenerateDocs extends Script {
         )
     }
 
+    private async generateSourcesOverview() {
+        console.log("Generating the sources overview - this might take a bit")
+        const sources = await new SourceOverview().getOverview()
+        const md = [
+            "# Overview of used online services",
+        ]
+
+        const serverInfos = <ServerSourceInfo[]>sources.filter(s => typeof s !== "string")
+            .filter(item => typeof item === "string" || item.url.startsWith("https://") || item.url.startsWith("pmtiles://"))
+        const titles = Utils.Dedup(Utils.NoEmpty(serverInfos.map(s => s.category)))
+        titles.sort()
+
+        function getHost(item: ServerSourceInfo) {
+            let url = item.url
+            if (url.startsWith("pmtiles://")) {
+                url = url.slice("pmtiles://".length)
+            }
+            const host = new URL(url).host.split(".")
+
+            return (host.at(-2) + "." + host.at(-1)).toLowerCase()
+        }
+
+        const categoryExplanation: Record<ServerSourceInfo["category"], string> = {
+            core: ["Core features are always active and will be contacted as soon as you visit any map",
+                "### About displayed images",
+                "MapComplete will read the 'image' attribute from OpenStreetMap-data when a POI is opened and will attempt to display this image (and thus download it). Those 'images' can be spread all over the internet and thus leak the IP address of the visitor",
+            ].join("\n\n"),
+            feature: "These are only enabled for certain maps or certain features",
+            maplayer: ["MapLayers are integrated from the [Editor Layer Index](https://github.com/osmlab/editor-layer-index). A map layer listed here will only be contacted if the user decides to use this map background. This list changes over time, as new background layers are published and old background layers go offline. For all map layers, we have permission to use them to improve OpenStreetMap",
+                "A full listing can be found in [ELI-overview](ELI-overview.md)",
+            ].join("\n\n"),
+        }
+
+        for (const title of titles) {
+            md.push("## " + title)
+            const items = serverInfos.filter(info => info.category.toLowerCase() === title.toLowerCase())
+            md.push(items.length + " items")
+            md.push(categoryExplanation[title])
+
+            const hosts = Utils.Dedup(items.map(getHost))
+            hosts.sort()
+            if (title === "maplayer") {
+                md.push(MarkdownUtils.list(hosts))
+                continue
+            }
+
+            for (const host of hosts) {
+                md.push("### " + host)
+                const itemsForHost = items.filter(info => getHost(info) === host)
+                const identicalDescription = itemsForHost.every(item => item.description === itemsForHost[0].description)
+                if (identicalDescription) {
+                    md.push(itemsForHost[0].description)
+                }
+                const table = MarkdownUtils.table(
+                    ["source", "description", "license;selfhosting;more info"],
+                    itemsForHost.map(item => {
+                        let selfHostable = ""
+                        if (item.selfhostable) {
+                            if (typeof item.selfhostable === "string") {
+                                selfHostable = item.selfhostable
+                            } else {
+                                selfHostable = "self hostable"
+                            }
+                        }
+                        let sourceAvailable = ""
+                        if (item.sourceAvailable) {
+                            if (typeof item.sourceAvailable === "string") {
+                                sourceAvailable = item.sourceAvailable
+                            } else {
+                                sourceAvailable = "source available"
+                            }
+                        }
+                        return [
+                            item.url,
+                            identicalDescription ? "" : item.description,
+                            Utils.NoEmpty([(item.openData ? "OpenData" : ""),
+                                sourceAvailable,
+                                selfHostable,
+                                item.moreInfo?.join(" , "),
+                            ]).join(", "),
+                        ]
+                    }),
+                    {
+                        dropEmptyColumns: true,
+                    },
+                )
+                md.push(table)
+            }
+
+
+        }
+
+        md.push("## No category")
+        const urls: string[] = <string[]>sources.filter(s => typeof s === "string")
+        md.push(urls.length + " items")
+        md.push(MarkdownUtils.list(urls))
+
+        this.WriteMarkdownFile("./Docs/OnlineServicesOverview.md", md.join("\n\n"), ["src/Models/SourceOverview.ts"], { tocMaxDepth: 2 })
+    }
+
     /**
      * Generates the documentation for the layers overview page
      * @constructor