import { RasterLayerProperties } from "./RasterLayerProperties" import { AvailableRasterLayers, RasterLayerPolygon } from "./RasterLayers" import { Utils } from "../Utils" import * as eli from "../../public/assets/data/editor-layer-index.json" import * as layers_global from "../../src/assets/global-raster-layers.json" import eli_global from "../../src/assets/generated/editor-layer-index-global.json" import bing from "../../src/assets/bing.json" import Constants from "../../src/Models/Constants" import ThemeConfig from "../../src/Models/ThemeConfig/ThemeConfig" import { ThemeConfigJson } from "../../src/Models/ThemeConfig/Json/ThemeConfigJson" import SpecialVisualizations from "../../src/UI/SpecialVisualizations" import ValidationUtils from "../../src/Models/ThemeConfig/Conversion/ValidationUtils" import { QuestionableTagRenderingConfigJson } from "../../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson" import { LayerConfigJson } from "../../src/Models/ThemeConfig/Json/LayerConfigJson" export interface ServerSourceInfo { url: string description: string category: "core" | "feature" | "maplayer" selfhostable?: boolean | string sourceAvailable?: boolean | string openData?: boolean | string trigger?: ("always" | "specific_theme" | "specific_feature" | "clear_consent" | string)[] moreInfo?: string[] logging?: "yes" | "probably" | "no" } /** * Generates what URLs the webapp might access, and why */ export class SourceOverview { private eliUrlsCached: ServerSourceInfo[] public async getOverview( layout?: ThemeConfig, layoutJson?: ThemeConfigJson ): Promise<(ServerSourceInfo | string)[]> { const apiUrls: (ServerSourceInfo | string)[] = [ ...Constants.allServers, { url: "https://data.mapcomplete.org/nsi", description: "Contains a copy and the logos of the Name Suggestion Index", category: "core", }, ] apiUrls.push(...(await this.eliUrls())) for (const sv of SpecialVisualizations.specialVisualizations) { if (typeof sv.needsUrls === "function") { // Handled below continue } apiUrls.push(...(sv.needsUrls ?? [])) } const usedSpecialVisualisations = layoutJson?.layers?.flatMap((l) => ValidationUtils.getAllSpecialVisualisations( (l).tagRenderings ?? [] ) ) for (const usedSpecialVisualisation of usedSpecialVisualisations ?? []) { if (typeof usedSpecialVisualisation === "string") { continue } const neededUrls = usedSpecialVisualisation.func.needsUrls ?? [] if (typeof neededUrls === "function") { const needed: string | string[] | ServerSourceInfo | ServerSourceInfo[] = neededUrls( usedSpecialVisualisation.args ) if (Array.isArray(needed)) { apiUrls.push(...needed) } else { apiUrls.push(needed) } } } const geojsonSources: string[] = layout?.layers?.map((l) => l.source?.geojsonSource) ?? [] return Utils.NoNull(apiUrls.concat(...geojsonSources)).filter((item) => { if (typeof item === "string") { return true } return item.url?.trim()?.length > 0 }) } private async eliUrls(): Promise { if (this.eliUrlsCached) { return this.eliUrlsCached } let urls: ServerSourceInfo[] = [] const regex = /{switch:([^}]+)}/ const rasterLayers: { properties: RasterLayerProperties }[] = [ AvailableRasterLayers.defaultBackgroundLayer, ...eli.features, bing, ...eli_global.map((properties) => ({ properties })), ...layers_global.layers.map((properties) => ({ properties })), ] for (const feature of rasterLayers) { const f = feature const url = f.properties.url const match = url.match(regex) const packageInInfo: (url: (string | string[])) => ServerSourceInfo[] = (urls: string | string[]) => { if (typeof urls === "string") { urls = [urls] } return urls.map(url => { url, description: "Background layer source or supporting sources for " + f.properties.id, trigger: ["specific_feature"], category: "maplayer", moreInfo: Utils.NoEmpty([ "https://github.com/osmlab/editor-layer-index", f.properties?.attribution?.url, ]), }) } const packageInInfoD: (url: (string | string[])) => (ServerSourceInfo[]) = (url: string | string[]) => { if (!url) { return [] } return packageInInfo(url) } if (match) { const domains = match[1].split(",") const subpart = match[0] const info: ServerSourceInfo[] = domains.flatMap((d) => packageInInfo(url.replace(subpart, d)) ) urls.push(...info) } else { urls.push(...packageInInfo(url)) } if (f.properties.type === "vector") { // We also need to whitelist eventual sources urls.push(...(f.properties["connect-src"] ?? []).map(packageInInfo)) let url = f.properties.url if (url.startsWith("pmtiles://")) { url = url.substring("pmtiles://".length) } if (url.endsWith(".pmtiles")) { continue } try { console.log("Downloading ", url) urls.push(...packageInInfo(url)) const styleSpec = await Utils.downloadJsonCached(url, 1000 * 120, { Origin: "https://mapcomplete.org", }) for (const key of Object.keys(styleSpec?.["sources"] ?? {})) { let url = styleSpec["sources"][key].url if (!url) { continue } console.log(">>> SPlitting", url) const split = url.split(",https://") console.log("Multisplit:", split) url = split[0] let urlClipped = url if (url.indexOf("?") > 0) { urlClipped = url?.substring(0, url.indexOf("?")) } urls.push(...packageInInfo(url)) if (urlClipped.endsWith(".json")) { const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120, { Origins: "https://mapcomplete.org" }) urls.push(...packageInInfo(tileInfo["tiles"] ?? [])) } } urls.push( ...(styleSpec["tiles"] ?? []) .flatMap((ls) => ls) .map((url) => packageInInfoD(url)) ) urls.push(...packageInInfoD(styleSpec["sprite"])) urls.push(...packageInInfoD(styleSpec["glyphs"])) } catch (e) { console.error(e) console.error( "ERROR: could not download a resource: " + url + "\nSome sprites might not be whitelisted and thus not load") } } } urls = urls.filter((item) => !!item?.url) urls.sort((a, b) => (a < b ? -1 : 1)) urls = Utils.DedupOnId(urls, (item) => item.url) this.eliUrlsCached = urls return urls } }