import { RasterLayerProperties } from "../src/Models/RasterLayerProperties" import { AvailableRasterLayers, RasterLayerPolygon } from "../src/Models/RasterLayers" import { Utils } from "../src/Utils" import * as eli from "../public/assets/data/editor-layer-index.json" import * as layers_global from "../src/assets/global-raster-layers.json" import eli_global from "../src/assets/generated/editor-layer-index-global.json" import bing from "../src/assets/bing.json" import Constants from "../src/Models/Constants" import ThemeConfig from "../src/Models/ThemeConfig/ThemeConfig" import { ThemeConfigJson } from "../src/Models/ThemeConfig/Json/ThemeConfigJson" import SpecialVisualizations from "../src/UI/SpecialVisualizations" import ValidationUtils from "../src/Models/ThemeConfig/Conversion/ValidationUtils" import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson" import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson" import { ImmutableStore } from "../src/Logic/UIEventSource" export interface SourceInfo { host: string } /** * Generates what URLs the webapp might access, and why */ export class SourceOverview { private eliUrlsCached: string[] public async getOverview( layout: ThemeConfig, layoutJson: ThemeConfigJson, options: { scriptSrcs: string[] }): Promise { const apiUrls: string[] = [ ...Constants.allServers, "https://www.openstreetmap.org", "https://api.openstreetmap.org", "https://pietervdvn.goatcounter.com", "https://api.panoramax.xyz", "https://panoramax.mapcomplete.org", "https://data.velopark.be", "https://data.mapcomplete.org", ].concat(...(await this.eliUrls())) for (const sv of SpecialVisualizations.specialVisualizations) { if (typeof sv.needsUrls === "function") { // Handled below continue } apiUrls.push(...(sv.needsUrls ?? [])) } const usedSpecialVisualisations = [].concat( ...layoutJson.layers.map((l) => ValidationUtils.getAllSpecialVisualisations( (l).tagRenderings ?? [] ) ) ) for (const usedSpecialVisualisation of usedSpecialVisualisations) { if (typeof usedSpecialVisualisation === "string") { continue } const neededUrls = usedSpecialVisualisation.func.needsUrls ?? [] if (typeof neededUrls === "function") { let needed: string | string[] = neededUrls(usedSpecialVisualisation.args) if (typeof needed === "string") { needed = [needed] } apiUrls.push(...needed) } } apiUrls.push("https://schema.org") const eliLayers: RasterLayerPolygon[] = AvailableRasterLayers.layersAvailableAt( new ImmutableStore({ lon: 0, lat: 0 }) ).store.data { const vectorLayers = eliLayers.filter((l) => l.properties.type === "vector") const vectorSources = vectorLayers.map((l) => l.properties.url) vectorSources.push(...vectorLayers.map((l) => l.properties.style)) apiUrls.push( ...vectorSources.map((url) => { if (url?.startsWith("pmtiles://")) { return url.substring("pmtiles://".length) } return url }) ) } const geojsonSources: string[] = layout.layers.map((l) => l.source?.geojsonSource) return apiUrls.concat(...geojsonSources) } private async eliUrls(): Promise { if (this.eliUrlsCached) { return this.eliUrlsCached } const urls: string[] = [] const regex = /{switch:([^}]+)}/ const rasterLayers: { properties: RasterLayerProperties }[] = [ AvailableRasterLayers.defaultBackgroundLayer, ...eli.features, bing, ...eli_global.map((properties) => ({ properties })), ...layers_global.layers.map((properties) => ({ properties })), ] for (const feature of rasterLayers) { const f = feature const url = f.properties.url const match = url.match(regex) if (match) { const domains = match[1].split(",") const subpart = match[0] urls.push(...domains.map((d) => url.replace(subpart, d))) } else { urls.push(url) } if (f.properties.type === "vector") { // We also need to whitelist eventual sources let url = f.properties.url urls.push(...(f.properties["connect-src"] ?? [])) if (url.startsWith("pmtiles://")) { url = url.substring("pmtiles://".length) } if (url.endsWith(".pmtiles")) { continue } try { console.log("Downloading ", url) const styleSpec = await Utils.downloadJsonCached(url, 1000 * 120, { Origin: "https://mapcomplete.org", }) for (const key of Object.keys(styleSpec?.["sources"] ?? {})) { const url = styleSpec["sources"][key].url if (!url) { continue } let urlClipped = url if (url.indexOf("?") > 0) { urlClipped = url?.substring(0, url.indexOf("?")) } console.log("Source url ", key, url) urls.push(url) if (urlClipped.endsWith(".json")) { const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120, { Origin: "https://mapcomplete.org", }) urls.push(tileInfo["tiles"] ?? []) } } urls.push(...(styleSpec["tiles"] ?? [])) urls.push(styleSpec["sprite"]) urls.push(styleSpec["glyphs"]) } catch (e) { console.error( "ERROR: could not download a resource, some sprites might not be whitelisted and thus not load" ) } } } this.eliUrlsCached = urls return Utils.NoNull(urls).sort() } }