import { RasterLayerProperties } from "./RasterLayerProperties" import { AvailableRasterLayers, RasterLayerPolygon } from "./RasterLayers" import { Utils } from "../Utils" import * as eli from "../../public/assets/data/editor-layer-index.json" import * as layers_global from "../../src/assets/global-raster-layers.json" import eli_global from "../../src/assets/generated/editor-layer-index-global.json" import bing from "../../src/assets/bing.json" import Constants from "../../src/Models/Constants" import ThemeConfig from "../../src/Models/ThemeConfig/ThemeConfig" import { ThemeConfigJson } from "../../src/Models/ThemeConfig/Json/ThemeConfigJson" import SpecialVisualizations from "../../src/UI/SpecialVisualizations" import ValidationUtils from "../../src/Models/ThemeConfig/Conversion/ValidationUtils" import { QuestionableTagRenderingConfigJson, } from "../../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson" import { LayerConfigJson } from "../../src/Models/ThemeConfig/Json/LayerConfigJson" export interface ServerSourceInfo { url: string description: string category: "core" | "feature" | "maplayer" selfhostable?: boolean | string sourceAvailable?: boolean | string openData?: boolean | string trigger?: ("always" | "specific_theme" | "specific_feature" | "clear_consent" | string)[] moreInfo?: string[], logging?: "yes" | "probably" | "no" } /** * Generates what URLs the webapp might access, and why */ export class SourceOverview { private eliUrlsCached: ServerSourceInfo[] public async getOverview(layout?: ThemeConfig, layoutJson?: ThemeConfigJson): Promise<(ServerSourceInfo | string)[]> { const apiUrls: (ServerSourceInfo | string)[] = [ ...Constants.allServers, { url: "https://data.mapcomplete.org/nsi", description: "Contains a copy and the logos of the Name Suggestion Index", category: "core", }, ] apiUrls.push(...(await this.eliUrls())) for (const sv of SpecialVisualizations.specialVisualizations) { if (typeof sv.needsUrls === "function") { // Handled below continue } apiUrls.push(...(sv.needsUrls ?? [])) } const usedSpecialVisualisations = layoutJson?.layers?.flatMap((l) => ValidationUtils.getAllSpecialVisualisations( (l).tagRenderings ?? [], )) for (const usedSpecialVisualisation of usedSpecialVisualisations ?? []) { if (typeof usedSpecialVisualisation === "string") { continue } const neededUrls = usedSpecialVisualisation.func.needsUrls ?? [] if (typeof neededUrls === "function") { let needed: string | string[] | ServerSourceInfo | ServerSourceInfo[] = neededUrls(usedSpecialVisualisation.args) if (Array.isArray(needed)) { apiUrls.push(...needed) } else { apiUrls.push(needed) } } } const geojsonSources: string[] = layout?.layers?.map((l) => l.source?.geojsonSource) ?? [] return Utils.NoNull(apiUrls.concat(...geojsonSources)).filter(item => { if (typeof item === "string") { return true } return item.url?.trim()?.length > 0 }) } private async eliUrls(): Promise { if (this.eliUrlsCached) { return this.eliUrlsCached } let urls: ServerSourceInfo[] = [] const regex = /{switch:([^}]+)}/ const rasterLayers: { properties: RasterLayerProperties }[] = [ AvailableRasterLayers.defaultBackgroundLayer, ...eli.features, bing, ...eli_global.map((properties) => ({ properties })), ...layers_global.layers.map((properties) => ({ properties })), ] for (const feature of rasterLayers) { const f = feature const url = f.properties.url const match = url.match(regex) const packageInInfo = (url: string) => { if(typeof url !== "string"){ throw "invalid url"+url } return ({ url, description: "Background layer source or supporting sources for " + f.properties.id, trigger: ["specific_feature"], category: "maplayer", moreInfo: Utils.NoEmpty(["https://github.com/osmlab/editor-layer-index", f.properties?.attribution?.url]), }) } if (match) { const domains = match[1].split(",") const subpart = match[0] const info: ServerSourceInfo[] = domains.map((d) => packageInInfo(url.replace(subpart, d))) urls.push(...info) } else { urls.push(packageInInfo(url)) } if (f.properties.type === "vector") { // We also need to whitelist eventual sources urls.push(...(f.properties["connect-src"] ?? []).map(packageInInfo)) let url = f.properties.url if (url.startsWith("pmtiles://")) { url = url.substring("pmtiles://".length) } if (url.endsWith(".pmtiles")) { continue } try { console.log("Downloading ", url) const styleSpec = await Utils.downloadJsonCached(url, 1000 * 120, { Origin: "https://mapcomplete.org", }) for (const key of Object.keys(styleSpec?.["sources"] ?? {})) { const url = styleSpec["sources"][key].url if (!url) { continue } let urlClipped = url if (url.indexOf("?") > 0) { urlClipped = url?.substring(0, url.indexOf("?")) } console.log("Source url ", key, url) urls.push(packageInInfo(url)) if (urlClipped.endsWith(".json")) { const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120, { Origin: "https://mapcomplete.org", }) urls.push(packageInInfo(tileInfo["tiles"] ?? [])) } } urls.push(...(styleSpec["tiles"] ?? []).flatMap(ls => ls).map(url => packageInInfo(url))) urls.push(packageInInfo(styleSpec["sprite"])) urls.push(packageInInfo(styleSpec["glyphs"])) } catch (e) { console.error( "ERROR: could not download a resource, some sprites might not be whitelisted and thus not load" ) } } } urls = urls.filter(item => !!item.url) urls.sort((a, b) => a < b ? -1 : 1) urls = Utils.DedupOnId(urls, item => item.url) this.eliUrlsCached = urls return urls } }