diff --git a/scripts/ScriptUtils.ts b/scripts/ScriptUtils.ts index 043cfee1f2..36e3b55214 100644 --- a/scripts/ScriptUtils.ts +++ b/scripts/ScriptUtils.ts @@ -205,6 +205,10 @@ export default class ScriptUtils { headers?: any, timeoutSecs?: number ): Promise<{ content: string } | { redirect: string } | "timeout"> { + if (url === undefined || url === null) { + console.trace("URL is null or undefined") + throw ("Invalid url: " + url) + } if (url.startsWith("./assets")) { return Promise.resolve({ content: readFileSync("./public/" + url, "utf8") }) } diff --git a/src/Models/SourceOverview.ts b/src/Models/SourceOverview.ts index 831e7f0a22..47efe999e6 100644 --- a/src/Models/SourceOverview.ts +++ b/src/Models/SourceOverview.ts @@ -10,7 +10,9 @@ import ThemeConfig from "../../src/Models/ThemeConfig/ThemeConfig" import { ThemeConfigJson } from "../../src/Models/ThemeConfig/Json/ThemeConfigJson" import SpecialVisualizations from "../../src/UI/SpecialVisualizations" import ValidationUtils from "../../src/Models/ThemeConfig/Conversion/ValidationUtils" -import { QuestionableTagRenderingConfigJson } from "../../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson" +import { + QuestionableTagRenderingConfigJson +} from "../../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson" import { LayerConfigJson } from "../../src/Models/ThemeConfig/Json/LayerConfigJson" export interface ServerSourceInfo { @@ -66,7 +68,7 @@ export class SourceOverview { } const neededUrls = usedSpecialVisualisation.func.needsUrls ?? [] if (typeof neededUrls === "function") { - let needed: string | string[] | ServerSourceInfo | ServerSourceInfo[] = neededUrls( + const needed: string | string[] | ServerSourceInfo | ServerSourceInfo[] = neededUrls( usedSpecialVisualisation.args ) if (Array.isArray(needed)) { @@ -105,11 +107,11 @@ export class SourceOverview { const url = f.properties.url const match = url.match(regex) - const packageInInfo = (url: string) => { - if (typeof url !== "string") { - throw "invalid url" + url + const packageInInfo: (url: (string | string[])) => ServerSourceInfo[] = (urls: string | string[]) => { + if (typeof urls === "string") { + urls = [urls] } - return { + return urls.map(url => { url, description: "Background layer source or supporting sources for " + f.properties.id, @@ -119,18 +121,25 @@ export class SourceOverview { "https://github.com/osmlab/editor-layer-index", f.properties?.attribution?.url, ]), + }) + } + + const packageInInfoD: (url: (string | string[])) => (ServerSourceInfo[]) = (url: string | string[]) => { + if (!url) { + return [] } + return packageInInfo(url) } if (match) { const domains = match[1].split(",") const subpart = match[0] - const info: ServerSourceInfo[] = domains.map((d) => + const info: ServerSourceInfo[] = domains.flatMap((d) => packageInInfo(url.replace(subpart, d)) ) urls.push(...info) } else { - urls.push(packageInInfo(url)) + urls.push(...packageInInfo(url)) } if (f.properties.type === "vector") { @@ -145,42 +154,46 @@ export class SourceOverview { } try { console.log("Downloading ", url) + urls.push(...packageInInfo(url)) const styleSpec = await Utils.downloadJsonCached(url, 1000 * 120, { Origin: "https://mapcomplete.org", }) for (const key of Object.keys(styleSpec?.["sources"] ?? {})) { - const url = styleSpec["sources"][key].url + let url = styleSpec["sources"][key].url if (!url) { continue } + console.log(">>> SPlitting", url) + const split = url.split(",https://") + console.log("Multisplit:", split) + url = split[0] let urlClipped = url if (url.indexOf("?") > 0) { urlClipped = url?.substring(0, url.indexOf("?")) } - console.log("Source url ", key, url) - urls.push(packageInInfo(url)) + urls.push(...packageInInfo(url)) if (urlClipped.endsWith(".json")) { const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120, { - Origin: "https://mapcomplete.org", + Origins: "https://mapcomplete.org" }) - urls.push(packageInInfo(tileInfo["tiles"] ?? [])) + urls.push(...packageInInfo(tileInfo["tiles"] ?? [])) } } urls.push( ...(styleSpec["tiles"] ?? []) .flatMap((ls) => ls) - .map((url) => packageInInfo(url)) + .map((url) => packageInInfoD(url)) ) - urls.push(packageInInfo(styleSpec["sprite"])) - urls.push(packageInInfo(styleSpec["glyphs"])) + urls.push(...packageInInfoD(styleSpec["sprite"])) + urls.push(...packageInInfoD(styleSpec["glyphs"])) } catch (e) { + console.error(e) console.error( - "ERROR: could not download a resource, some sprites might not be whitelisted and thus not load" - ) + "ERROR: could not download a resource: " + url + "\nSome sprites might not be whitelisted and thus not load") } } } - urls = urls.filter((item) => !!item.url) + urls = urls.filter((item) => !!item?.url) urls.sort((a, b) => (a < b ? -1 : 1)) urls = Utils.DedupOnId(urls, (item) => item.url) this.eliUrlsCached = urls