MapComplete/MapComplete
MapComplete/MapComplete
2
1
Fork 3
Code Issues 181 Pull requests 2 Activity Actions

Fix: include 'source' and tile URLs from vector tile sources into CSP, see #1652

Browse source
This commit is contained in:
Pieter Vander Vennet 2023-10-11 19:04:31 +02:00
parent d997c90352
commit 09504e18ec
1 changed files with 59 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

98
scripts/generateLayouts.ts
View file

@ -1,20 +1,20 @@
import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFile, writeFileSync } from "fs"; import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFile, writeFileSync } from "fs"
import Locale from "../src/UI/i18n/Locale"; import Locale from "../src/UI/i18n/Locale"
import Translations from "../src/UI/i18n/Translations"; import Translations from "../src/UI/i18n/Translations"
import { Translation } from "../src/UI/i18n/Translation"; import { Translation } from "../src/UI/i18n/Translation"
import all_known_layouts from "../src/assets/generated/known_themes.json"; import all_known_layouts from "../src/assets/generated/known_themes.json"
import { LayoutConfigJson } from "../src/Models/ThemeConfig/Json/LayoutConfigJson"; import { LayoutConfigJson } from "../src/Models/ThemeConfig/Json/LayoutConfigJson"
import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig"; import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig"
import xml2js from "xml2js"; import xml2js from "xml2js"
import ScriptUtils from "./ScriptUtils"; import ScriptUtils from "./ScriptUtils"
import { Utils } from "../src/Utils"; import { Utils } from "../src/Utils"
import SpecialVisualizations from "../src/UI/SpecialVisualizations"; import SpecialVisualizations from "../src/UI/SpecialVisualizations"
import Constants from "../src/Models/Constants"; import Constants from "../src/Models/Constants"
import { AvailableRasterLayers, RasterLayerPolygon } from "../src/Models/RasterLayers"; import { AvailableRasterLayers, RasterLayerPolygon } from "../src/Models/RasterLayers"
import { ImmutableStore } from "../src/Logic/UIEventSource"; import { ImmutableStore } from "../src/Logic/UIEventSource"
import * as crypto from "crypto"; import * as crypto from "crypto"
import * as eli from "../src/assets/editor-layer-index.json"; import * as eli from "../src/assets/editor-layer-index.json"
import * as eli_global from "../src/assets/global-raster-layers.json"; import * as eli_global from "../src/assets/global-raster-layers.json"
const sharp = require("sharp") const sharp = require("sharp")
const template = readFileSync("theme.html", "utf8") const template = readFileSync("theme.html", "utf8")
@ -63,7 +63,7 @@ async function createSocialImage(layout: LayoutConfig, template: "" | "Wide"): P
"Not creating a social image for " + "Not creating a social image for " +
layout.id + layout.id +
" as it is _not_ a .svg: " + " as it is _not_ a .svg: " +
layout.icon layout.icon,
) )
return undefined return undefined
} }
@ -85,7 +85,7 @@ async function createSocialImage(layout: LayoutConfig, template: "" | "Wide"): P
delete svg["defs"] delete svg["defs"]
delete svg["$"] delete svg["$"]
let templateSvg = await ScriptUtils.ReadSvg( let templateSvg = await ScriptUtils.ReadSvg(
"./public/assets/SocialImageTemplate" + template + ".svg" "./public/assets/SocialImageTemplate" + template + ".svg",
) )
templateSvg = Utils.WalkJson( templateSvg = Utils.WalkJson(
templateSvg, templateSvg,
@ -104,7 +104,7 @@ async function createSocialImage(layout: LayoutConfig, template: "" | "Wide"): P
return false return false
} }
return mightBeTokenToReplace.circle[0]?.$?.style?.indexOf("fill:#ff00ff") >= 0 return mightBeTokenToReplace.circle[0]?.$?.style?.indexOf("fill:#ff00ff") >= 0
} },
) )
const builder = new xml2js.Builder() const builder = new xml2js.Builder()
@ -116,7 +116,7 @@ async function createSocialImage(layout: LayoutConfig, template: "" | "Wide"): P
async function createManifest( async function createManifest(
layout: LayoutConfig, layout: LayoutConfig,
alreadyWritten: string[] alreadyWritten: string[],
): Promise<{ ): Promise<{
manifest: any manifest: any
whiteIcons: string[] whiteIcons: string[]
@ -210,15 +210,17 @@ function asLangSpan(t: Translation, tag = "span"): string {
let previousSrc: Set<string> = new Set<string>() let previousSrc: Set<string> = new Set<string>()
let eliUrlsCached: string[] let eliUrlsCached: string[]
function eliUrls(): string[] {
async function eliUrls(): Promise<string[]> {
if (eliUrlsCached) { if (eliUrlsCached) {
return eliUrlsCached return eliUrlsCached
} }
const urls: string[] = [] const urls: string[] = []
const regex = /{switch:([^}]+)}/ const regex = /{switch:([^}]+)}/
const rasterLayers = [...AvailableRasterLayers.vectorLayers, ...eli.features, ...eli_global.layers.map(properties => ({properties})) ] const rasterLayers = [...AvailableRasterLayers.vectorLayers, ...eli.features, ...eli_global.layers.map(properties => ({ properties }))]
for (const feature of rasterLayers) { for (const feature of rasterLayers) {
const url = (<RasterLayerPolygon>feature).properties.url const f = <RasterLayerPolygon>feature
const url = f.properties.url
const match = url.match(regex) const match = url.match(regex)
if (match) { if (match) {
const domains = match[1].split(",") const domains = match[1].split(",")
@ -227,17 +229,28 @@ function eliUrls(): string[] {
} else { } else {
urls.push(url) urls.push(url)
} }
if (f.properties.type === "vector") {
// We also need to whitelist eventual sources
const styleSpec = await Utils.downloadJsonCached(f.properties.url, 1000 * 120)
for (const key in styleSpec.sources) {
const url = styleSpec.sources[key].url
urls.push(url)
}
urls.push(...(styleSpec["tiles"] ?? []))
}
} }
eliUrlsCached = urls eliUrlsCached = urls
return urls return urls
} }
function generateCsp( async function generateCsp(
layout: LayoutConfig, layout: LayoutConfig,
options: { options: {
scriptSrcs: string[] scriptSrcs: string[]
} },
): string { ): Promise<string> {
const apiUrls: string[] = [ const apiUrls: string[] = [
"'self'", "'self'",
...Constants.defaultOverpassUrls, ...Constants.defaultOverpassUrls,
@ -247,12 +260,12 @@ function generateCsp(
"https://pietervdvn.goatcounter.com", "https://pietervdvn.goatcounter.com",
] ]
.concat(...SpecialVisualizations.specialVisualizations.map((sv) => sv.needsUrls)) .concat(...SpecialVisualizations.specialVisualizations.map((sv) => sv.needsUrls))
.concat(...eliUrls()) .concat(...await eliUrls())
const geojsonSources: string[] = layout.layers.map((l) => l.source?.geojsonSource) const geojsonSources: string[] = layout.layers.map((l) => l.source?.geojsonSource)
const hosts = new Set<string>() const hosts = new Set<string>()
const eliLayers: RasterLayerPolygon[] = AvailableRasterLayers.layersAvailableAt( const eliLayers: RasterLayerPolygon[] = AvailableRasterLayers.layersAvailableAt(
new ImmutableStore({ lon: 0, lat: 0 }) new ImmutableStore({ lon: 0, lat: 0 }),
).data ).data
const vectorLayers = eliLayers.filter((l) => l.properties.type === "vector") const vectorLayers = eliLayers.filter((l) => l.properties.type === "vector")
const vectorSources = vectorLayers.map((l) => l.properties.url) const vectorSources = vectorLayers.map((l) => l.properties.url)
@ -279,14 +292,14 @@ function generateCsp(
"connect-src items for theme", "connect-src items for theme",
layout.id, layout.id,
"(extra sources: ", "(extra sources: ",
newSrcs.join(" ") + ")" newSrcs.join(" ") + ")",
) )
previousSrc = hosts previousSrc = hosts
const csp: Record<string, string> = { const csp: Record<string, string> = {
"default-src": "'self'", "default-src": "'self'",
"script-src": ["'self'", "https://gc.zgo.at/count.js", ...(options?.scriptSrcs ?? [])].join( "script-src": ["'self'", "https://gc.zgo.at/count.js", ...(options?.scriptSrcs ?? [])].join(
" " " ",
), ),
"img-src": "* data:", // maplibre depends on 'data:' to load "img-src": "* data:", // maplibre depends on 'data:' to load
"connect-src": connectSrc.join(" "), "connect-src": connectSrc.join(" "),
@ -316,12 +329,12 @@ const removeOtherLanguagesHash = crypto
async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alreadyWritten) { async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alreadyWritten) {
Locale.language.setData(layout.language[0]) Locale.language.setData(layout.language[0])
const targetLanguage = layout.language[0] const targetLanguage = layout.language[0]
const ogTitle = Translations.T(layout.title).textFor(targetLanguage).replace(/"/g, '\\"') const ogTitle = Translations.T(layout.title).textFor(targetLanguage).replace(/"/g, "\\\"")
const ogDescr = Translations.T( const ogDescr = Translations.T(
layout.shortDescription ?? "Easily add and edit geodata with OpenStreetMap" layout.shortDescription ?? "Easily add and edit geodata with OpenStreetMap",
) )
.textFor(targetLanguage) .textFor(targetLanguage)
.replace(/"/g, '\\"') .replace(/"/g, "\\\"")
let ogImage = layout.socialImage let ogImage = layout.socialImage
let twitterImage = ogImage let twitterImage = ogImage
if (ogImage === LayoutConfig.defaultSocialImage && layout.official) { if (ogImage === LayoutConfig.defaultSocialImage && layout.official) {
@ -386,34 +399,34 @@ async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alr
const loadingText = Translations.t.general.loadingTheme.Subs({ theme: layout.title }) const loadingText = Translations.t.general.loadingTheme.Subs({ theme: layout.title })
const templateLines = template.split("\n") const templateLines = template.split("\n")
const removeOtherLanguagesReference = templateLines.find( const removeOtherLanguagesReference = templateLines.find(
(line) => line.indexOf("./src/UI/RemoveOtherLanguages.js") >= 0 (line) => line.indexOf("./src/UI/RemoveOtherLanguages.js") >= 0,
) )
let output = template let output = template
.replace("Loading MapComplete, hang on...", asLangSpan(loadingText, "h1")) .replace("Loading MapComplete, hang on...", asLangSpan(loadingText, "h1"))
.replace( .replace(
"Made with OpenStreetMap", "Made with OpenStreetMap",
Translations.t.general.poweredByOsm.textFor(targetLanguage) Translations.t.general.poweredByOsm.textFor(targetLanguage),
) )
.replace(/<!-- THEME-SPECIFIC -->.*<!-- THEME-SPECIFIC-END-->/s, themeSpecific) .replace(/<!-- THEME-SPECIFIC -->.*<!-- THEME-SPECIFIC-END-->/s, themeSpecific)
.replace( .replace(
/<!-- CSP -->/, /<!-- CSP -->/,
generateCsp(layout, { await generateCsp(layout, {
scriptSrcs: [`'sha256-${removeOtherLanguagesHash}'`], scriptSrcs: [`'sha256-${removeOtherLanguagesHash}'`],
}) }),
) )
.replace(removeOtherLanguagesReference, "<script>" + removeOtherLanguages + "</script>") .replace(removeOtherLanguagesReference, "<script>" + removeOtherLanguages + "</script>")
.replace( .replace(
/<!-- DESCRIPTION START -->.*<!-- DESCRIPTION END -->/s, /<!-- DESCRIPTION START -->.*<!-- DESCRIPTION END -->/s,
asLangSpan(layout.shortDescription) asLangSpan(layout.shortDescription),
) )
.replace( .replace(
/<!-- IMAGE-START -->.*<!-- IMAGE-END -->/s, /<!-- IMAGE-START -->.*<!-- IMAGE-END -->/s,
"<img class='p-8 h-32 w-32 self-start' src='" + icon + "' />" "<img class='p-8 h-32 w-32 self-start' src='" + icon + "' />",
) )
.replace( .replace(
/.*\/src\/index\.ts.*/, /.*\/src\/index\.ts.*/,
`<script type="module" src="./index_${layout.id}.ts"></script>` `<script type="module" src="./index_${layout.id}.ts"></script>`,
) )
return output return output
@ -504,13 +517,14 @@ async function main(): Promise<void> {
title: { en: "MapComplete" }, title: { en: "MapComplete" },
description: { en: "A thematic map viewer and editor based on OpenStreetMap" }, description: { en: "A thematic map viewer and editor based on OpenStreetMap" },
}), }),
alreadyWritten alreadyWritten,
) )
const manif = JSON.stringify(manifest, undefined, 2) const manif = JSON.stringify(manifest, undefined, 2)
writeFileSync("public/index.webmanifest", manif) writeFileSync("public/index.webmanifest", manif)
} }
ScriptUtils.fixUtils()
main().then(() => { main().then(() => {
console.log("All done!") console.log("All done!")
}) })