diff --git a/scripts/generateLayouts.ts b/scripts/generateLayouts.ts index e7e8d676fd..17805270d8 100644 --- a/scripts/generateLayouts.ts +++ b/scripts/generateLayouts.ts @@ -1,20 +1,20 @@ -import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFile, writeFileSync } from "fs"; -import Locale from "../src/UI/i18n/Locale"; -import Translations from "../src/UI/i18n/Translations"; -import { Translation } from "../src/UI/i18n/Translation"; -import all_known_layouts from "../src/assets/generated/known_themes.json"; -import { LayoutConfigJson } from "../src/Models/ThemeConfig/Json/LayoutConfigJson"; -import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig"; -import xml2js from "xml2js"; -import ScriptUtils from "./ScriptUtils"; -import { Utils } from "../src/Utils"; -import SpecialVisualizations from "../src/UI/SpecialVisualizations"; -import Constants from "../src/Models/Constants"; -import { AvailableRasterLayers, RasterLayerPolygon } from "../src/Models/RasterLayers"; -import { ImmutableStore } from "../src/Logic/UIEventSource"; -import * as crypto from "crypto"; -import * as eli from "../src/assets/editor-layer-index.json"; -import * as eli_global from "../src/assets/global-raster-layers.json"; +import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFile, writeFileSync } from "fs" +import Locale from "../src/UI/i18n/Locale" +import Translations from "../src/UI/i18n/Translations" +import { Translation } from "../src/UI/i18n/Translation" +import all_known_layouts from "../src/assets/generated/known_themes.json" +import { LayoutConfigJson } from "../src/Models/ThemeConfig/Json/LayoutConfigJson" +import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig" +import xml2js from "xml2js" +import ScriptUtils from "./ScriptUtils" +import { Utils } from "../src/Utils" +import SpecialVisualizations from "../src/UI/SpecialVisualizations" +import Constants from "../src/Models/Constants" +import { AvailableRasterLayers, RasterLayerPolygon } from "../src/Models/RasterLayers" +import { ImmutableStore } from "../src/Logic/UIEventSource" +import * as crypto from "crypto" +import * as eli from "../src/assets/editor-layer-index.json" +import * as eli_global from "../src/assets/global-raster-layers.json" const sharp = require("sharp") const template = readFileSync("theme.html", "utf8") @@ -61,9 +61,9 @@ async function createSocialImage(layout: LayoutConfig, template: "" | "Wide"): P if (!layout.icon.endsWith(".svg")) { console.warn( "Not creating a social image for " + - layout.id + - " as it is _not_ a .svg: " + - layout.icon + layout.id + + " as it is _not_ a .svg: " + + layout.icon, ) return undefined } @@ -85,7 +85,7 @@ async function createSocialImage(layout: LayoutConfig, template: "" | "Wide"): P delete svg["defs"] delete svg["$"] let templateSvg = await ScriptUtils.ReadSvg( - "./public/assets/SocialImageTemplate" + template + ".svg" + "./public/assets/SocialImageTemplate" + template + ".svg", ) templateSvg = Utils.WalkJson( templateSvg, @@ -104,7 +104,7 @@ async function createSocialImage(layout: LayoutConfig, template: "" | "Wide"): P return false } return mightBeTokenToReplace.circle[0]?.$?.style?.indexOf("fill:#ff00ff") >= 0 - } + }, ) const builder = new xml2js.Builder() @@ -116,7 +116,7 @@ async function createSocialImage(layout: LayoutConfig, template: "" | "Wide"): P async function createManifest( layout: LayoutConfig, - alreadyWritten: string[] + alreadyWritten: string[], ): Promise<{ manifest: any whiteIcons: string[] @@ -210,15 +210,17 @@ function asLangSpan(t: Translation, tag = "span"): string { let previousSrc: Set = new Set() let eliUrlsCached: string[] -function eliUrls(): string[] { + +async function eliUrls(): Promise { if (eliUrlsCached) { return eliUrlsCached } const urls: string[] = [] const regex = /{switch:([^}]+)}/ - const rasterLayers = [...AvailableRasterLayers.vectorLayers, ...eli.features, ...eli_global.layers.map(properties => ({properties})) ] - for (const feature of rasterLayers) { - const url = (feature).properties.url + const rasterLayers = [...AvailableRasterLayers.vectorLayers, ...eli.features, ...eli_global.layers.map(properties => ({ properties }))] + for (const feature of rasterLayers) { + const f = feature + const url = f.properties.url const match = url.match(regex) if (match) { const domains = match[1].split(",") @@ -227,17 +229,28 @@ function eliUrls(): string[] { } else { urls.push(url) } + + if (f.properties.type === "vector") { + // We also need to whitelist eventual sources + const styleSpec = await Utils.downloadJsonCached(f.properties.url, 1000 * 120) + for (const key in styleSpec.sources) { + const url = styleSpec.sources[key].url + urls.push(url) + } + urls.push(...(styleSpec["tiles"] ?? [])) + } + } eliUrlsCached = urls return urls } -function generateCsp( +async function generateCsp( layout: LayoutConfig, options: { scriptSrcs: string[] - } -): string { + }, +): Promise { const apiUrls: string[] = [ "'self'", ...Constants.defaultOverpassUrls, @@ -247,12 +260,12 @@ function generateCsp( "https://pietervdvn.goatcounter.com", ] .concat(...SpecialVisualizations.specialVisualizations.map((sv) => sv.needsUrls)) - .concat(...eliUrls()) + .concat(...await eliUrls()) const geojsonSources: string[] = layout.layers.map((l) => l.source?.geojsonSource) const hosts = new Set() const eliLayers: RasterLayerPolygon[] = AvailableRasterLayers.layersAvailableAt( - new ImmutableStore({ lon: 0, lat: 0 }) + new ImmutableStore({ lon: 0, lat: 0 }), ).data const vectorLayers = eliLayers.filter((l) => l.properties.type === "vector") const vectorSources = vectorLayers.map((l) => l.properties.url) @@ -279,14 +292,14 @@ function generateCsp( "connect-src items for theme", layout.id, "(extra sources: ", - newSrcs.join(" ") + ")" + newSrcs.join(" ") + ")", ) previousSrc = hosts const csp: Record = { "default-src": "'self'", "script-src": ["'self'", "https://gc.zgo.at/count.js", ...(options?.scriptSrcs ?? [])].join( - " " + " ", ), "img-src": "* data:", // maplibre depends on 'data:' to load "connect-src": connectSrc.join(" "), @@ -316,12 +329,12 @@ const removeOtherLanguagesHash = crypto async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alreadyWritten) { Locale.language.setData(layout.language[0]) const targetLanguage = layout.language[0] - const ogTitle = Translations.T(layout.title).textFor(targetLanguage).replace(/"/g, '\\"') + const ogTitle = Translations.T(layout.title).textFor(targetLanguage).replace(/"/g, "\\\"") const ogDescr = Translations.T( - layout.shortDescription ?? "Easily add and edit geodata with OpenStreetMap" + layout.shortDescription ?? "Easily add and edit geodata with OpenStreetMap", ) .textFor(targetLanguage) - .replace(/"/g, '\\"') + .replace(/"/g, "\\\"") let ogImage = layout.socialImage let twitterImage = ogImage if (ogImage === LayoutConfig.defaultSocialImage && layout.official) { @@ -386,34 +399,34 @@ async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alr const loadingText = Translations.t.general.loadingTheme.Subs({ theme: layout.title }) const templateLines = template.split("\n") const removeOtherLanguagesReference = templateLines.find( - (line) => line.indexOf("./src/UI/RemoveOtherLanguages.js") >= 0 + (line) => line.indexOf("./src/UI/RemoveOtherLanguages.js") >= 0, ) let output = template .replace("Loading MapComplete, hang on...", asLangSpan(loadingText, "h1")) .replace( "Made with OpenStreetMap", - Translations.t.general.poweredByOsm.textFor(targetLanguage) + Translations.t.general.poweredByOsm.textFor(targetLanguage), ) .replace(/.*/s, themeSpecific) .replace( //, - generateCsp(layout, { + await generateCsp(layout, { scriptSrcs: [`'sha256-${removeOtherLanguagesHash}'`], - }) + }), ) .replace(removeOtherLanguagesReference, "") .replace( /.*/s, - asLangSpan(layout.shortDescription) + asLangSpan(layout.shortDescription), ) .replace( /.*/s, - "" + "", ) .replace( /.*\/src\/index\.ts.*/, - `` + ``, ) return output @@ -504,13 +517,14 @@ async function main(): Promise { title: { en: "MapComplete" }, description: { en: "A thematic map viewer and editor based on OpenStreetMap" }, }), - alreadyWritten + alreadyWritten, ) const manif = JSON.stringify(manifest, undefined, 2) writeFileSync("public/index.webmanifest", manif) } +ScriptUtils.fixUtils() main().then(() => { console.log("All done!") })