Bastian-Greshake-Tzovaras/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Merge develop

Browse source
This commit is contained in:
Pieter Vander Vennet 2023-10-01 13:13:07 +02:00
commit 55c6442cac
388 changed files with 16178 additions and 17860 deletions
Download patch file Download diff file Expand all files Collapse all files

1
scripts/GenerateSeries.ts
View file

@ -273,7 +273,6 @@ class GenerateSeries extends Script {
allFeatures = allFeatures.filter((f) => f.properties.metadata?.theme !== "EMPTY CS")
const centerpoints = allFeatures.map((f) => GeoOperations.centerpoint(f))
console.log("Found", centerpoints.length, " changesets in total")
const path = `${targetDir}/all_centerpoints.geojson`
const perBbox = GeoOperations.spreadIntoBboxes(centerpoints, options.zoomlevel)

5
scripts/fetchLanguages.ts
View file

@ -50,8 +50,11 @@ async function fetchRegularLanguages() {
const result = await Utils.downloadJson(url, { "User-Agent": "MapComplete script" })
const bindings = <LanguageSpecResult[]>result.results.bindings
// Traditional chinese = 繁體中文 or 正體中文
const zh_hant = await fetchSpecial(18130932, "zh_Hant")
const zh_hans = await fetchSpecial(13414913, "zh_Hant")
// Simplified chinese = 簡體中文 or 简体中文(
const zh_hans = await fetchSpecial(13414913, "zh_Hans")
const pt_br = await fetchSpecial(750553, "pt_BR")
const punjabi = await fetchSpecial(58635, "pa_PK")
const Shahmukhi = await Wikidata.LoadWikidataEntryAsync(133800)

2
scripts/generateContributors.ts
View file

@ -26,7 +26,7 @@ function asList(hist: Map<string, number>): ContributorList {
}
function main() {
exec("git log --pretty='%aN %%!%% %s' ", (error, stdout, stderr) => {
exec("git log --pretty='%aN %%!%% %s' ", (_, stdout) => {
const entries = stdout.split("\n").filter((str) => str !== "")
const codeContributors = new Map<string, number>()
const translationContributors = new Map<string, number>()

142
scripts/generateLayerOverview.ts
View file

@ -12,8 +12,6 @@ import {
ValidateThemeAndLayers,
} from "../src/Models/ThemeConfig/Conversion/Validation"
import { Translation } from "../src/UI/i18n/Translation"
import { TagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/TagRenderingConfigJson"
import PointRenderingConfigJson from "../src/Models/ThemeConfig/Json/PointRenderingConfigJson"
import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
import { PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme"
import { DesugaringContext } from "../src/Models/ThemeConfig/Conversion/Conversion"
@ -21,6 +19,8 @@ import { Utils } from "../src/Utils"
import Script from "./Script"
import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
import { parse as parse_html } from "node-html-parser"
import { ExtraFunctions } from "../src/Logic/ExtraFunctions"
import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
// This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files.
// It spits out an overview of those to be used to load them
@ -161,8 +161,8 @@ class LayerOverviewUtils extends Script {
getSharedTagRenderings(
doesImageExist: DoesImageExist,
bootstrapTagRenderings: Map<string, TagRenderingConfigJson> = null
): Map<string, TagRenderingConfigJson> {
bootstrapTagRenderings: Map<string, QuestionableTagRenderingConfigJson> = null
): Map<string, QuestionableTagRenderingConfigJson> {
const prepareLayer = new PrepareLayer({
tagRenderings: bootstrapTagRenderings,
sharedLayers: null,
@ -172,10 +172,10 @@ class LayerOverviewUtils extends Script {
let path = "assets/layers/questions/questions.json"
const sharedQuestions = this.parseLayer(doesImageExist, prepareLayer, path)
const dict = new Map<string, TagRenderingConfigJson>()
const dict = new Map<string, QuestionableTagRenderingConfigJson>()
for (const tr of sharedQuestions.tagRenderings) {
const tagRendering = <TagRenderingConfigJson>tr
const tagRendering = <QuestionableTagRenderingConfigJson>tr
dict.set(tagRendering["id"], tagRendering)
}
@ -276,7 +276,8 @@ class LayerOverviewUtils extends Script {
if (
recompiledThemes.length > 0 &&
!(recompiledThemes.length === 1 && recompiledThemes[0] === "mapcomplete-changes")
!(recompiledThemes.length === 1 && recompiledThemes[0] === "mapcomplete-changes") &&
args.indexOf("--generate-change-map") >= 0
) {
// mapcomplete-changes shows an icon for each corresponding mapcomplete-theme
const iconsPerTheme = Array.from(sharedThemes.values()).map((th) => ({
@ -401,10 +402,133 @@ class LayerOverviewUtils extends Script {
skippedLayers.length +
" layers"
)
// We always need the calculated tags of 'usersettings', so we export them separately
this.extractJavascriptCodeForLayer(
state.sharedLayers.get("usersettings"),
"./src/Logic/State/UserSettingsMetaTagging.ts"
)
return sharedLayers
}
/**
* Given: a fully expanded themeConfigJson
*
* Will extract a dictionary of the special code and write it into a javascript file which can be imported.
* This removes the need for _eval_, allowing for a correct CSP
* @param themeFile
* @private
*/
private extractJavascriptCode(themeFile: LayoutConfigJson) {
const allCode = [
"import {Feature} from 'geojson'",
'import { ExtraFuncType } from "../../../Logic/ExtraFunctions";',
'import { Utils } from "../../../Utils"',
"export class ThemeMetaTagging {",
" public static readonly themeName = " + JSON.stringify(themeFile.id),
"",
]
for (const layer of themeFile.layers) {
const l = <LayerConfigJson>layer
const id = l.id.replace(/[^a-zA-Z0-9_]/g, "_")
const code = l.calculatedTags ?? []
allCode.push(
" public metaTaggging_for_" +
id +
"(feat: Feature, helperFunctions: Record<ExtraFuncType, (feature: Feature) => Function>) {"
)
allCode.push(" const {" + ExtraFunctions.types.join(", ") + "} = helperFunctions")
for (const line of code) {
const firstEq = line.indexOf("=")
let attributeName = line.substring(0, firstEq).trim()
const expression = line.substring(firstEq + 1)
const isStrict = attributeName.endsWith(":")
if (!isStrict) {
allCode.push(
" Utils.AddLazyProperty(feat.properties, '" +
attributeName +
"', () => " +
expression +
" ) "
)
} else {
attributeName = attributeName.substring(0, attributeName.length - 1).trim()
allCode.push(" feat.properties['" + attributeName + "'] = " + expression)
}
}
allCode.push(" }")
}
const targetDir = "./src/assets/generated/metatagging/"
if (!existsSync(targetDir)) {
mkdirSync(targetDir)
}
allCode.push("}")
writeFileSync(targetDir + themeFile.id + ".ts", allCode.join("\n"))
}
private extractJavascriptCodeForLayer(l: LayerConfigJson, targetPath?: string) {
if (!l) {
return // Probably a bootstrapping run
}
let importPath = "../../../"
if (targetPath) {
const l = targetPath.split("/")
if (l.length == 1) {
importPath = "./"
} else {
importPath = ""
for (let i = 0; i < l.length - 3; i++) {
const _ = l[i]
importPath += "../"
}
}
}
const allCode = [
`import { Utils } from "${importPath}Utils"`,
`/** This code is autogenerated - do not edit. Edit ./assets/layers/${l?.id}/${l?.id}.json instead */`,
"export class ThemeMetaTagging {",
" public static readonly themeName = " + JSON.stringify(l.id),
"",
]
const code = l.calculatedTags ?? []
allCode.push(
" public metaTaggging_for_" + l.id + "(feat: {properties: Record<string, string>}) {"
)
for (const line of code) {
const firstEq = line.indexOf("=")
let attributeName = line.substring(0, firstEq).trim()
const expression = line.substring(firstEq + 1)
const isStrict = attributeName.endsWith(":")
if (!isStrict) {
allCode.push(
" Utils.AddLazyProperty(feat.properties, '" +
attributeName +
"', () => " +
expression +
" ) "
)
} else {
attributeName = attributeName.substring(0, attributeName.length - 2).trim()
allCode.push(" feat.properties['" + attributeName + "'] = " + expression)
}
}
allCode.push(" }")
allCode.push("}")
const targetDir = "./src/assets/generated/metatagging/"
if (!targetPath) {
if (!existsSync(targetDir)) {
mkdirSync(targetDir)
}
}
writeFileSync(targetPath ?? targetDir + "layer_" + l.id + ".ts", allCode.join("\n"))
}
private buildThemeIndex(
licensePaths: Set<string>,
sharedLayers: Map<string, LayerConfigJson>,
@ -442,6 +566,7 @@ class LayerOverviewUtils extends Script {
})
const skippedThemes: string[] = []
for (let i = 0; i < themeFiles.length; i++) {
const themeInfo = themeFiles[i]
const themePath = themeInfo.path
@ -449,6 +574,7 @@ class LayerOverviewUtils extends Script {
const targetPath =
LayerOverviewUtils.themePath + "/" + themePath.substring(themePath.lastIndexOf("/"))
const usedLayers = Array.from(
LayerOverviewUtils.extractLayerIdsFrom(themeFile, false)
).map((id) => LayerOverviewUtils.layerPath + id + ".json")
@ -510,6 +636,8 @@ class LayerOverviewUtils extends Script {
this.writeTheme(themeFile)
fixed.set(themeFile.id, themeFile)
this.extractJavascriptCode(themeFile)
} catch (e) {
console.error("ERROR: could not prepare theme " + themePath + " due to " + e)
throw e

28
scripts/generateLayouts.ts
View file

@ -200,6 +200,26 @@ function asLangSpan(t: Translation, tag = "span"): string {
return values.join("\n")
}
let cspCached: string = undefined
function generateCsp(): string {
if (cspCached !== undefined) {
return cspCached
}
const csp = {
"default-src": "'self'",
"script-src": "'self'",
"img-src": "*",
"connect-src": "*",
}
const content = Object.keys(csp)
.map((k) => k + ": " + csp[k])
.join("; ")
cspCached = `<meta http-equiv="Content-Security-Policy" content="${content}">`
return cspCached
}
async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alreadyWritten) {
Locale.language.setData(layout.language[0])
const targetLanguage = layout.language[0]
@ -279,6 +299,7 @@ async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alr
Translations.t.general.poweredByOsm.textFor(targetLanguage)
)
.replace(/<!-- THEME-SPECIFIC -->.*<!-- THEME-SPECIFIC-END-->/s, themeSpecific)
.replace(/<!-- CSP -->/, generateCsp())
.replace(
/<!-- DESCRIPTION START -->.*<!-- DESCRIPTION END -->/s,
asLangSpan(layout.shortDescription)
@ -298,7 +319,12 @@ async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alr
async function createIndexFor(theme: LayoutConfig) {
const filename = "index_" + theme.id + ".ts"
writeFileSync(filename, `import layout from "./src/assets/generated/themes/${theme.id}.json"\n`)
const imports = [
`import layout from "./src/assets/generated/themes/${theme.id}.json"`,
`import { ThemeMetaTagging } from "./src/assets/generated/metatagging/${theme.id}"`,
]
writeFileSync(filename, imports.join("\n") + "\n")
appendFileSync(filename, codeTemplate)
}

2
scripts/generateLicenseInfo.ts
View file

@ -3,7 +3,7 @@ import SmallLicense from "../src/Models/smallLicense"
import ScriptUtils from "./ScriptUtils"
import Script from "./Script"
import { Utils } from "../src/Utils"
const prompt = require("prompt-sync")()
export class GenerateLicenseInfo extends Script {
private static readonly needsLicenseRef = new Set(
ScriptUtils.readDirRecSync("./LICENSES")

2
scripts/generateTaginfoProjectFiles.ts
View file

@ -34,8 +34,6 @@ function generateTagOverview(
return overview
}
function tagrenderingToTaginfoDescription(tr: TagRenderingConfig) {}
function generateLayerUsage(layer: LayerConfig, layout: LayoutConfig): any[] {
if (layer.name === undefined) {
return [] // Probably a duplicate or irrelevant layer

4
scripts/hetzner/config.json Normal file
View file

@ -0,0 +1,4 @@
{
"#":"Some configuration tweaks specifically for hetzner",
"country_coder_host": "https://countrycoder.mapcomplete.org/"
}

21
scripts/hetzner/config/Caddyfile Normal file
View file

@ -0,0 +1,21 @@
hosted.mapcomplete.org {
root * public/
file_server
header {
+Permissions-Policy "interest-cohort=()"
+Report-To `\{"group":"csp-endpoint", "max_age": 86400,"endpoints": [\{"url": "https://report.mapcomplete.org/csp"}], "include_subdomains": true}`
+Content-Security-Policy-Report-Only "default-src 'self'; script-src 'self' https://gc.zgo.at ; img-src * ; report-uri https://report.mapcomplete.org/csp ; report-to csp-endpoint ;"
}
}
countrycoder.mapcomplete.org {
root * tiles/
file_server
}
report.mapcomplete.org {
reverse_proxy {
to http://127.0.0.1:2600
}
}

7
scripts/hetzner/config/csp-logger-config.json Normal file
View file

@ -0,0 +1,7 @@
{
"store": "console",
"allowedOrigin": null,
"port": 2600,
"domainWhitelist": ["localhost:10179", "localhost:2600","hosted.mapcomplete.org", "dev.mapcomplete.org", "mapcomplete.org","*"],
"sourceBlacklist": ["chrome-extension://gighmmpiobklfepjocnamgkkbiglidom"]
}

24
scripts/hetzner/deployHetzner.sh Executable file
View file

@ -0,0 +1,24 @@
#! /bin/bash
### To be run from the root of the repository
# Some pointers to get started:
# apt install npm
# apt install unzip
# npm i -g csp-logger
# wget https://github.com/pietervdvn/latlon2country/raw/main/tiles.zip
# unzip tiles.zip
MAPCOMPLETE_CONFIGURATION="config_hetzner"
npm run reset:layeroverview
npm run test
cp config.json config.json.bu &&
cp ./scripts/hetzner/config.json . &&
npm run prepare-deploy &&
mv config.json.bu config.json &&
zip dist.zip -r dist/* &&
scp -r dist.zip hetzner:/root/ &&
scp ./scripts/hetzner/config/* hetzner:/root/
ssh hetzner -t "unzip dist.zip && rm dist.zip && rm -rf public/ && mv dist public && caddy stop && caddy start"
rm dist.zip
npm run clean

8
scripts/thieves/readIdPresets.ts
View file

@ -3,10 +3,10 @@
*/
import ScriptUtils from "../ScriptUtils"
import { existsSync, readFileSync, writeFileSync } from "fs"
import known_languages from "../../assets/language_native.json"
import { LayerConfigJson } from "../../Models/ThemeConfig/Json/LayerConfigJson"
import { MappingConfigJson } from "../../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import SmallLicense from "../../Models/smallLicense"
import known_languages from "../../src/assets/language_native.json"
import { LayerConfigJson } from "../../src/Models/ThemeConfig/Json/LayerConfigJson"
import { MappingConfigJson } from "../../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import SmallLicense from "../../src/Models/smallLicense"
interface IconThief {
steal(iconName: string): boolean