midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

LayerServer: improve script, add unique and valid id check to layers

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-01-23 22:03:22 +01:00
parent 1d6c9ec1ef
commit ee38cdb9d7
5 changed files with 313 additions and 199 deletions
Download patch file Download diff file Expand all files Collapse all files

23
Docs/SettingUpPSQL.md
View file

@ -10,26 +10,37 @@ Then activate following extensions for this database (right click > Create > Ext
- Postgis activeren (rechtsklikken > Create > extension) - Postgis activeren (rechtsklikken > Create > extension)
- HStore activeren - HStore activeren
Install osm2pgsql (hint: compile from source is painless) Increase the max number of connections. osm2pgsql needs connection one per table (and a few more), and since we are making one table per layer in MapComplete, this amounts to a lot.
pg_tileserv kan hier gedownload worden: https://github.com/CrunchyData/pg_tileserv - Open PGAdmin, open the PGSQL-tool (CLI-button at the top)
- Run `max_connections = 2000;` and `show config_file;` to get the config file location (in docker). This is probably `/var/lib/postgresql/data/postgresql.conf`
DATABASE_URL=postgresql://user:none@localhost:5444/osm-poi ./pg_tileserv - In a terminal, run `sudo docker exec -i <docker-container-id> bash` (run `sudo docker ps` to get the container id)
- `sed -i "s/max_connections = 100/max_connections = 5000/" /var/lib/postgresql/data/postgresql.conf`
- Validate with `cat /var/lib/postgresql/data/postgresql.conf | grep "max_connections"`
- `sudo docker restart <ID>`
## Create export scripts for every layer ## Create export scripts for every layer
Use scripts/osm2pgsl Use `vite-node ./scripts/osm2pgsql/generateBuildDbScript.ts`
## Importing data ## Importing data
Install osm2pgsql (hint: compile from source is painless)
To seed the database: To seed the database:
```` ````
osm2pgsql -O flex -E 4326 -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi andorra-latest.osm.pbf osm2pgsql -O flex -E 4326 -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi <file>.osm.pbf
```` ````
Storing properties to table '"public"."osm2pgsql_properties" takes about 25 minutes with planet.osm
Belgium (~555mb) takes 15m
World (80GB) should take 15m*160 = 2400m = 40hr
## Deploying a tile server ## Deploying a tile server
pg_tileserv kan hier gedownload worden: https://github.com/CrunchyData/pg_tileserv
```` ````
export DATABASE_URL=postgresql://user:password@localhost:5444/osm-poi export DATABASE_URL=postgresql://user:password@localhost:5444/osm-poi
./pg_tileserv ./pg_tileserv

9
scripts/Script.ts
View file

@ -13,8 +13,15 @@ export default abstract class Script {
ScriptUtils.fixUtils() ScriptUtils.fixUtils()
const args = [...process.argv] const args = [...process.argv]
args.splice(0, 2) args.splice(0, 2)
const start = new Date()
this.main(args) this.main(args)
.then((_) => console.log("All done")) .then((_) =>{
const end = new Date()
const millisNeeded = end.getTime() - start.getTime()
const green = (s) => "\x1b[92m" + s + "\x1b[0m"
console.log(green("All done! (" + millisNeeded + " ms)"))
})
.catch((e) => console.log("ERROR:", e)) .catch((e) => console.log("ERROR:", e))
} }

69
scripts/generateLayerOverview.ts
View file

@ -9,7 +9,7 @@ import {
DoesImageExist, DoesImageExist,
PrevalidateTheme, PrevalidateTheme,
ValidateLayer, ValidateLayer,
ValidateThemeAndLayers, ValidateThemeAndLayers, ValidateThemeEnsemble,
} from "../src/Models/ThemeConfig/Conversion/Validation" } from "../src/Models/ThemeConfig/Conversion/Validation"
import { Translation } from "../src/UI/i18n/Translation" import { Translation } from "../src/UI/i18n/Translation"
import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer" import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
@ -25,6 +25,8 @@ import LayerConfig from "../src/Models/ThemeConfig/LayerConfig"
import PointRenderingConfig from "../src/Models/ThemeConfig/PointRenderingConfig" import PointRenderingConfig from "../src/Models/ThemeConfig/PointRenderingConfig"
import { ConversionContext } from "../src/Models/ThemeConfig/Conversion/ConversionContext" import { ConversionContext } from "../src/Models/ThemeConfig/Conversion/ConversionContext"
import { GenerateFavouritesLayer } from "./generateFavouritesLayer" import { GenerateFavouritesLayer } from "./generateFavouritesLayer"
import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig"
import { TagsFilter } from "../src/Logic/Tags/TagsFilter"
// This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files. // This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files.
// It spits out an overview of those to be used to load them // It spits out an overview of those to be used to load them
@ -123,6 +125,7 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye
} }
} }
class LayerOverviewUtils extends Script { class LayerOverviewUtils extends Script {
public static readonly layerPath = "./src/assets/generated/layers/" public static readonly layerPath = "./src/assets/generated/layers/"
public static readonly themePath = "./src/assets/generated/themes/" public static readonly themePath = "./src/assets/generated/themes/"
@ -355,7 +358,6 @@ class LayerOverviewUtils extends Script {
const layerWhitelist = new Set(args.find(a => a.startsWith("--layers=")) const layerWhitelist = new Set(args.find(a => a.startsWith("--layers="))
?.substring("--layers=".length)?.split(",") ?? []) ?.substring("--layers=".length)?.split(",") ?? [])
const start = new Date()
const forceReload = args.some((a) => a == "--force") const forceReload = args.some((a) => a == "--force")
const licensePaths = new Set<string>() const licensePaths = new Set<string>()
@ -382,17 +384,21 @@ class LayerOverviewUtils extends Script {
sharedLayers, sharedLayers,
recompiledThemes, recompiledThemes,
forceReload, forceReload,
themeWhitelist themeWhitelist,
) )
if (recompiledThemes.length > 0){ new ValidateThemeEnsemble().convertStrict(
Array.from(sharedThemes.values()).map(th => new LayoutConfig(th, true)))
if (recompiledThemes.length > 0) {
writeFileSync( writeFileSync(
"./src/assets/generated/known_layers.json", "./src/assets/generated/known_layers.json",
JSON.stringify({ JSON.stringify({
layers: Array.from(sharedLayers.values()).filter((l) => l.id !== "favourite"), layers: Array.from(sharedLayers.values()).filter((l) => l.id !== "favourite"),
}), }),
) )
} }
const mcChangesPath = "./assets/themes/mapcomplete-changes/mapcomplete-changes.json" const mcChangesPath = "./assets/themes/mapcomplete-changes/mapcomplete-changes.json"
if ( if (
@ -437,7 +443,7 @@ class LayerOverviewUtils extends Script {
) )
} }
if(recompiledThemes.length > 0) { if (recompiledThemes.length > 0) {
writeFileSync( writeFileSync(
"./src/assets/generated/known_themes.json", "./src/assets/generated/known_themes.json",
JSON.stringify({ JSON.stringify({
@ -446,17 +452,10 @@ class LayerOverviewUtils extends Script {
) )
} }
const end = new Date()
const millisNeeded = end.getTime() - start.getTime()
if (AllSharedLayers.getSharedLayersConfigs().size == 0) { if (AllSharedLayers.getSharedLayersConfigs().size == 0) {
console.error( console.error(
"This was a bootstrapping-run. Run generate layeroverview again!(" + "This was a bootstrapping-run. Run generate layeroverview again!"
millisNeeded +
" ms)",
) )
} else {
const green = (s) => "\x1b[92m" + s + "\x1b[0m"
console.log(green("All done! (" + millisNeeded + " ms)"))
} }
} }
@ -482,7 +481,7 @@ class LayerOverviewUtils extends Script {
private buildLayerIndex( private buildLayerIndex(
doesImageExist: DoesImageExist, doesImageExist: DoesImageExist,
forceReload: boolean, forceReload: boolean,
whitelist: Set<string> whitelist: Set<string>,
): Map<string, LayerConfigJson> { ): Map<string, LayerConfigJson> {
// First, we expand and validate all builtin layers. These are written to src/assets/generated/layers // First, we expand and validate all builtin layers. These are written to src/assets/generated/layers
// At the same time, an index of available layers is built. // At the same time, an index of available layers is built.
@ -500,9 +499,9 @@ class LayerOverviewUtils extends Script {
const recompiledLayers: string[] = [] const recompiledLayers: string[] = []
let warningCount = 0 let warningCount = 0
for (const sharedLayerPath of ScriptUtils.getLayerPaths()) { for (const sharedLayerPath of ScriptUtils.getLayerPaths()) {
if(whitelist.size > 0){ if (whitelist.size > 0) {
const idByPath = sharedLayerPath.split("/").at(-1).split(".")[0] const idByPath = sharedLayerPath.split("/").at(-1).split(".")[0]
if(Constants.priviliged_layers.indexOf(<any> idByPath) < 0 && !whitelist.has(idByPath)){ if (Constants.priviliged_layers.indexOf(<any>idByPath) < 0 && !whitelist.has(idByPath)) {
continue continue
} }
} }
@ -672,7 +671,7 @@ class LayerOverviewUtils extends Script {
sharedLayers: Map<string, LayerConfigJson>, sharedLayers: Map<string, LayerConfigJson>,
recompiledThemes: string[], recompiledThemes: string[],
forceReload: boolean, forceReload: boolean,
whitelist: Set<string> whitelist: Set<string>,
): Map<string, LayoutConfigJson> { ): Map<string, LayoutConfigJson> {
console.log(" ---------- VALIDATING BUILTIN THEMES ---------") console.log(" ---------- VALIDATING BUILTIN THEMES ---------")
const themeFiles = ScriptUtils.getThemeFiles() const themeFiles = ScriptUtils.getThemeFiles()
@ -710,7 +709,7 @@ class LayerOverviewUtils extends Script {
const themeInfo = themeFiles[i] const themeInfo = themeFiles[i]
const themePath = themeInfo.path const themePath = themeInfo.path
let themeFile = themeInfo.parsed let themeFile = themeInfo.parsed
if(whitelist.size > 0 && !whitelist.has(themeFile.id)){ if (whitelist.size > 0 && !whitelist.has(themeFile.id)) {
continue continue
} }
@ -795,21 +794,21 @@ class LayerOverviewUtils extends Script {
} }
} }
if(whitelist.size == 0){ if (whitelist.size == 0) {
this.writeSmallOverview( this.writeSmallOverview(
Array.from(fixed.values()).map((t) => { Array.from(fixed.values()).map((t) => {
return { return {
...t, ...t,
hideFromOverview: t.hideFromOverview ?? false, hideFromOverview: t.hideFromOverview ?? false,
shortDescription: shortDescription:
t.shortDescription ?? t.shortDescription ??
new Translation(t.description) new Translation(t.description)
.FirstSentence() .FirstSentence()
.OnEveryLanguage((s) => parse_html(s).textContent).translations, .OnEveryLanguage((s) => parse_html(s).textContent).translations,
mustHaveLanguage: t.mustHaveLanguage?.length > 0, mustHaveLanguage: t.mustHaveLanguage?.length > 0,
} }
}), }),
) )
} }
console.log( console.log(

52
scripts/osm2pgsql/generateBuildDbScript.ts
View file

@ -1,13 +1,13 @@
import LayerConfig from "../../src/Models/ThemeConfig/LayerConfig"
import { TagsFilter } from "../../src/Logic/Tags/TagsFilter" import { TagsFilter } from "../../src/Logic/Tags/TagsFilter"
import { Tag } from "../../src/Logic/Tags/Tag" import { Tag } from "../../src/Logic/Tags/Tag"
import { And } from "../../src/Logic/Tags/And" import { And } from "../../src/Logic/Tags/And"
import Script from "../Script" import Script from "../Script"
import { AllSharedLayers } from "../../src/Customizations/AllSharedLayers"
import fs from "fs" import fs from "fs"
import { Or } from "../../src/Logic/Tags/Or" import { Or } from "../../src/Logic/Tags/Or"
import { RegexTag } from "../../src/Logic/Tags/RegexTag" import { RegexTag } from "../../src/Logic/Tags/RegexTag"
import { Utils } from "../../src/Utils" import { Utils } from "../../src/Utils"
import { ValidateThemeEnsemble } from "../../src/Models/ThemeConfig/Conversion/Validation"
import { AllKnownLayouts } from "../../src/Customizations/AllKnownLayouts"
class LuaSnippets { class LuaSnippets {
/** /**
@ -35,28 +35,31 @@ class LuaSnippets {
} }
class GenerateLayerLua { class GenerateLayerLua {
private readonly _layer: LayerConfig private readonly _id: string
private readonly _tags: TagsFilter
private readonly _foundInThemes: string[]
constructor(layer: LayerConfig) { constructor(id: string, tags: TagsFilter, foundInThemes: string[] = []) {
this._layer = layer this._tags = tags
this._id = id
this._foundInThemes = foundInThemes
} }
public functionName() { public functionName() {
const l = this._layer if (!this._tags) {
if (!l.source?.osmTags) {
return undefined return undefined
} }
return `process_poi_${l.id}` return `process_poi_${this._id}`
} }
public generateFunction(): string { public generateFunction(): string {
const l = this._layer if (!this._tags) {
if (!l.source?.osmTags) {
return undefined return undefined
} }
return [ return [
`local pois_${l.id} = osm2pgsql.define_table({`, `local pois_${this._id} = osm2pgsql.define_table({`,
` name = '${l.id}',`, this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "",
` name = '${this._id}',`,
" ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },",
" columns = {", " columns = {",
" { column = 'tags', type = 'jsonb' },", " { column = 'tags', type = 'jsonb' },",
@ -66,7 +69,7 @@ class GenerateLayerLua {
"", "",
"", "",
`function ${this.functionName()}(object, geom)`, `function ${this.functionName()}(object, geom)`,
" local matches_filter = " + this.toLuaFilter(l.source.osmTags), " local matches_filter = " + this.toLuaFilter(this._tags),
" if( not matches_filter) then", " if( not matches_filter) then",
" return", " return",
" end", " end",
@ -75,7 +78,7 @@ class GenerateLayerLua {
" tags = object.tags", " tags = object.tags",
" }", " }",
" ", " ",
` pois_${l.id}:insert(a)`, ` pois_${this._id}:insert(a)`,
"end", "end",
"", "",
].join("\n") ].join("\n")
@ -86,6 +89,8 @@ class GenerateLayerLua {
return `object.tags["${tag.key}"] ~= "${tag.value}"` return `object.tags["${tag.key}"] ~= "${tag.value}"`
} }
const v = (<RegExp> tag.value).source.replace(/\\\//g, "/")
if ("" + tag.value === "/.+/is" && !tag.invert) { if ("" + tag.value === "/.+/is" && !tag.invert) {
return `object.tags["${tag.key}"] ~= nil` return `object.tags["${tag.key}"] ~= nil`
} }
@ -104,10 +109,10 @@ class GenerateLayerLua {
} }
if (tag.invert) { if (tag.invert) {
return `object.tags["${tag.key}"] == nil or not string.find(object.tags["${tag.key}"], "${tag.value}")` return `object.tags["${tag.key}"] == nil or not string.find(object.tags["${tag.key}"], "${v}")`
} }
return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${tag.value}"))` return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${v}"))`
} }
private toLuaFilter(tag: TagsFilter, useParens: boolean = false): string { private toLuaFilter(tag: TagsFilter, useParens: boolean = false): string {
@ -141,15 +146,21 @@ class GenerateLayerLua {
} }
} }
class GenerateLayerFile extends Script { class GenerateBuildDbScript extends Script {
constructor() { constructor() {
super("Generates a .lua-file to use with osm2pgsql") super("Generates a .lua-file to use with osm2pgsql")
} }
async main(args: string[]) { async main(args: string[]) {
const layers = Array.from(AllSharedLayers.sharedLayers.values()) const allNeededLayers = new ValidateThemeEnsemble().convertStrict(
AllKnownLayouts.allKnownLayouts.values(),
)
const generators = layers.filter(l => l.source.geojsonSource === undefined).map(l => new GenerateLayerLua(l)) const generators: GenerateLayerLua[] = []
allNeededLayers.forEach(({ tags, foundInTheme }, layerId) => {
generators.push(new GenerateLayerLua(layerId, tags, foundInTheme))
})
const script = [ const script = [
...generators.map(g => g.generateFunction()), ...generators.map(g => g.generateFunction()),
@ -159,7 +170,8 @@ class GenerateLayerFile extends Script {
const path = "build_db.lua" const path = "build_db.lua"
fs.writeFileSync(path, script, "utf-8") fs.writeFileSync(path, script, "utf-8")
console.log("Written", path) console.log("Written", path)
console.log(allNeededLayers.size+" layers will be created. Make sure to set 'max_connections' to at least "+(10 + allNeededLayers.size) )
} }
} }
new GenerateLayerFile().run() new GenerateBuildDbScript().run()

359
src/Models/ThemeConfig/Conversion/Validation.ts
View file

@ -21,9 +21,7 @@ import PresetConfig from "../PresetConfig"
import { TagsFilter } from "../../../Logic/Tags/TagsFilter" import { TagsFilter } from "../../../Logic/Tags/TagsFilter"
import { Translatable } from "../Json/Translatable" import { Translatable } from "../Json/Translatable"
import { ConversionContext } from "./ConversionContext" import { ConversionContext } from "./ConversionContext"
import * as eli from "../../../assets/editor-layer-index.json"
import { AvailableRasterLayers } from "../../RasterLayers" import { AvailableRasterLayers } from "../../RasterLayers"
import Back from "../../../assets/svg/Back.svelte"
import PointRenderingConfigJson from "../Json/PointRenderingConfigJson" import PointRenderingConfigJson from "../Json/PointRenderingConfigJson"
class ValidateLanguageCompleteness extends DesugaringStep<LayoutConfig> { class ValidateLanguageCompleteness extends DesugaringStep<LayoutConfig> {
@ -33,7 +31,7 @@ class ValidateLanguageCompleteness extends DesugaringStep<LayoutConfig> {
super( super(
"Checks that the given object is fully translated in the specified languages", "Checks that the given object is fully translated in the specified languages",
[], [],
"ValidateLanguageCompleteness" "ValidateLanguageCompleteness",
) )
this._languages = languages ?? ["en"] this._languages = languages ?? ["en"]
} }
@ -47,18 +45,18 @@ class ValidateLanguageCompleteness extends DesugaringStep<LayoutConfig> {
.filter( .filter(
(t) => (t) =>
t.tr.translations[neededLanguage] === undefined && t.tr.translations[neededLanguage] === undefined &&
t.tr.translations["*"] === undefined t.tr.translations["*"] === undefined,
) )
.forEach((missing) => { .forEach((missing) => {
context context
.enter(missing.context.split(".")) .enter(missing.context.split("."))
.err( .err(
`The theme ${obj.id} should be translation-complete for ` + `The theme ${obj.id} should be translation-complete for ` +
neededLanguage + neededLanguage +
", but it lacks a translation for " + ", but it lacks a translation for " +
missing.context + missing.context +
".\n\tThe known translation is " + ".\n\tThe known translation is " +
missing.tr.textFor("en") missing.tr.textFor("en"),
) )
}) })
} }
@ -75,7 +73,7 @@ export class DoesImageExist extends DesugaringStep<string> {
constructor( constructor(
knownImagePaths: Set<string>, knownImagePaths: Set<string>,
checkExistsSync: (path: string) => boolean = undefined, checkExistsSync: (path: string) => boolean = undefined,
ignore?: Set<string> ignore?: Set<string>,
) { ) {
super("Checks if an image exists", [], "DoesImageExist") super("Checks if an image exists", [], "DoesImageExist")
this._ignore = ignore this._ignore = ignore
@ -111,15 +109,15 @@ export class DoesImageExist extends DesugaringStep<string> {
if (!this._knownImagePaths.has(image)) { if (!this._knownImagePaths.has(image)) {
if (this.doesPathExist === undefined) { if (this.doesPathExist === undefined) {
context.err( context.err(
`Image with path ${image} not found or not attributed; it is used in ${context}` `Image with path ${image} not found or not attributed; it is used in ${context}`,
) )
} else if (!this.doesPathExist(image)) { } else if (!this.doesPathExist(image)) {
context.err( context.err(
`Image with path ${image} does not exist.\n Check for typo's and missing directories in the path.` `Image with path ${image} does not exist.\n Check for typo's and missing directories in the path.`,
) )
} else { } else {
context.err( context.err(
`Image with path ${image} is not attributed (but it exists); execute 'npm run query:licenses' to add the license information and/or run 'npm run generate:licenses' to compile all the license info` `Image with path ${image} is not attributed (but it exists); execute 'npm run query:licenses' to add the license information and/or run 'npm run generate:licenses' to compile all the license info`,
) )
} }
} }
@ -143,7 +141,7 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
doesImageExist: DoesImageExist, doesImageExist: DoesImageExist,
path: string, path: string,
isBuiltin: boolean, isBuiltin: boolean,
sharedTagRenderings?: Set<string> sharedTagRenderings?: Set<string>,
) { ) {
super("Doesn't change anything, but emits warnings and errors", [], "ValidateTheme") super("Doesn't change anything, but emits warnings and errors", [], "ValidateTheme")
this._validateImage = doesImageExist this._validateImage = doesImageExist
@ -162,15 +160,15 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
if (json["units"] !== undefined) { if (json["units"] !== undefined) {
context.err( context.err(
"The theme " + "The theme " +
json.id + json.id +
" has units defined - these should be defined on the layer instead. (Hint: use overrideAll: { '+units': ... }) " " has units defined - these should be defined on the layer instead. (Hint: use overrideAll: { '+units': ... }) ",
) )
} }
if (json["roamingRenderings"] !== undefined) { if (json["roamingRenderings"] !== undefined) {
context.err( context.err(
"Theme " + "Theme " +
json.id + json.id +
" contains an old 'roamingRenderings'. Use an 'overrideAll' instead" " contains an old 'roamingRenderings'. Use an 'overrideAll' instead",
) )
} }
} }
@ -178,7 +176,7 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
if (!json.title) { if (!json.title) {
context.enter("title").err(`The theme ${json.id} does not have a title defined.`) context.enter("title").err(`The theme ${json.id} does not have a title defined.`)
} }
if(!json.icon){ if (!json.icon) {
context.enter("icon").err("A theme should have an icon") context.enter("icon").err("A theme should have an icon")
} }
if (this._isBuiltin && this._extractImages !== undefined) { if (this._isBuiltin && this._extractImages !== undefined) {
@ -188,10 +186,10 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
for (const remoteImage of remoteImages) { for (const remoteImage of remoteImages) {
context.err( context.err(
"Found a remote image: " + "Found a remote image: " +
remoteImage.path + remoteImage.path +
" in theme " + " in theme " +
json.id + json.id +
", please download it." ", please download it.",
) )
} }
for (const image of images) { for (const image of images) {
@ -207,17 +205,17 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
const filename = this._path.substring( const filename = this._path.substring(
this._path.lastIndexOf("/") + 1, this._path.lastIndexOf("/") + 1,
this._path.length - 5 this._path.length - 5,
) )
if (theme.id !== filename) { if (theme.id !== filename) {
context.err( context.err(
"Theme ids should be the same as the name.json, but we got id: " + "Theme ids should be the same as the name.json, but we got id: " +
theme.id + theme.id +
" and filename " + " and filename " +
filename + filename +
" (" + " (" +
this._path + this._path +
")" ")",
) )
} }
this._validateImage.convert(theme.icon, context.enter("icon")) this._validateImage.convert(theme.icon, context.enter("icon"))
@ -225,13 +223,13 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
const dups = Utils.Duplicates(json.layers.map((layer) => layer["id"])) const dups = Utils.Duplicates(json.layers.map((layer) => layer["id"]))
if (dups.length > 0) { if (dups.length > 0) {
context.err( context.err(
`The theme ${json.id} defines multiple layers with id ${dups.join(", ")}` `The theme ${json.id} defines multiple layers with id ${dups.join(", ")}`,
) )
} }
if (json["mustHaveLanguage"] !== undefined) { if (json["mustHaveLanguage"] !== undefined) {
new ValidateLanguageCompleteness(...json["mustHaveLanguage"]).convert( new ValidateLanguageCompleteness(...json["mustHaveLanguage"]).convert(
theme, theme,
context context,
) )
} }
if (!json.hideFromOverview && theme.id !== "personal" && this._isBuiltin) { if (!json.hideFromOverview && theme.id !== "personal" && this._isBuiltin) {
@ -239,7 +237,7 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
const targetLanguage = theme.title.SupportedLanguages()[0] const targetLanguage = theme.title.SupportedLanguages()[0]
if (targetLanguage !== "en") { if (targetLanguage !== "en") {
context.err( context.err(
`TargetLanguage is not 'en' for public theme ${theme.id}, it is ${targetLanguage}. Move 'en' up in the title of the theme and set it as the first key` `TargetLanguage is not 'en' for public theme ${theme.id}, it is ${targetLanguage}. Move 'en' up in the title of the theme and set it as the first key`,
) )
} }
@ -282,6 +280,13 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
} }
} }
for (let i = 0; i < theme.layers.length; i++) {
const layer = theme.layers[i]
if (!layer.id.match("[a-z][a-z0-9_]*")) {
context.enters("layers", i, "id").err("Invalid ID:" + layer.id + "should match [a-z][a-z0-9_]*")
}
}
return json return json
} }
} }
@ -291,7 +296,7 @@ export class ValidateThemeAndLayers extends Fuse<LayoutConfigJson> {
doesImageExist: DoesImageExist, doesImageExist: DoesImageExist,
path: string, path: string,
isBuiltin: boolean, isBuiltin: boolean,
sharedTagRenderings?: Set<string> sharedTagRenderings?: Set<string>,
) { ) {
super( super(
"Validates a theme and the contained layers", "Validates a theme and the contained layers",
@ -301,10 +306,10 @@ export class ValidateThemeAndLayers extends Fuse<LayoutConfigJson> {
new Each( new Each(
new Bypass( new Bypass(
(layer) => Constants.added_by_default.indexOf(<any>layer.id) < 0, (layer) => Constants.added_by_default.indexOf(<any>layer.id) < 0,
new ValidateLayerConfig(undefined, isBuiltin, doesImageExist, false, true) new ValidateLayerConfig(undefined, isBuiltin, doesImageExist, false, true),
) ),
) ),
) ),
) )
} }
} }
@ -314,7 +319,7 @@ class OverrideShadowingCheck extends DesugaringStep<LayoutConfigJson> {
super( super(
"Checks that an 'overrideAll' does not override a single override", "Checks that an 'overrideAll' does not override a single override",
[], [],
"OverrideShadowingCheck" "OverrideShadowingCheck",
) )
} }
@ -363,6 +368,22 @@ class MiscThemeChecks extends DesugaringStep<LayoutConfigJson> {
if (json.socialImage === "") { if (json.socialImage === "") {
context.warn("Social image for theme " + json.id + " is the emtpy string") context.warn("Social image for theme " + json.id + " is the emtpy string")
} }
{
for (let i = 0; i < json.layers.length; i++) {
const l = json.layers[i]
if (l["override"]?.["source"] === undefined) {
continue
}
if (l["override"]?.["source"]?.["geoJson"]) {
continue // We don't care about external data as we won't cache it anyway
}
if (l["override"]["id"] !== undefined) {
continue
}
context.enters("layers", i).err("A layer which changes the source-tags must also change the ID")
}
}
return json return json
} }
} }
@ -372,7 +393,7 @@ export class PrevalidateTheme extends Fuse<LayoutConfigJson> {
super( super(
"Various consistency checks on the raw JSON", "Various consistency checks on the raw JSON",
new MiscThemeChecks(), new MiscThemeChecks(),
new OverrideShadowingCheck() new OverrideShadowingCheck(),
) )
} }
} }
@ -382,7 +403,7 @@ export class DetectConflictingAddExtraTags extends DesugaringStep<TagRenderingCo
super( super(
"The `if`-part in a mapping might set some keys. Those keys are not allowed to be set in the `addExtraTags`, as this might result in conflicting values", "The `if`-part in a mapping might set some keys. Those keys are not allowed to be set in the `addExtraTags`, as this might result in conflicting values",
[], [],
"DetectConflictingAddExtraTags" "DetectConflictingAddExtraTags",
) )
} }
@ -409,7 +430,7 @@ export class DetectConflictingAddExtraTags extends DesugaringStep<TagRenderingCo
.enters("mappings", i) .enters("mappings", i)
.err( .err(
"AddExtraTags overrides a key that is set in the `if`-clause of this mapping. Selecting this answer might thus first set one value (needed to match as answer) and then override it with a different value, resulting in an unsaveable question. The offending `addExtraTags` is " + "AddExtraTags overrides a key that is set in the `if`-clause of this mapping. Selecting this answer might thus first set one value (needed to match as answer) and then override it with a different value, resulting in an unsaveable question. The offending `addExtraTags` is " +
duplicateKeys.join(", ") duplicateKeys.join(", "),
) )
} }
} }
@ -427,13 +448,13 @@ export class DetectNonErasedKeysInMappings extends DesugaringStep<QuestionableTa
super( super(
"A tagRendering might set a freeform key (e.g. `name` and have an option that _should_ erase this name, e.g. `noname=yes`). Under normal circumstances, every mapping/freeform should affect all touched keys", "A tagRendering might set a freeform key (e.g. `name` and have an option that _should_ erase this name, e.g. `noname=yes`). Under normal circumstances, every mapping/freeform should affect all touched keys",
[], [],
"DetectNonErasedKeysInMappings" "DetectNonErasedKeysInMappings",
) )
} }
convert( convert(
json: QuestionableTagRenderingConfigJson, json: QuestionableTagRenderingConfigJson,
context: ConversionContext context: ConversionContext,
): QuestionableTagRenderingConfigJson { ): QuestionableTagRenderingConfigJson {
if (json.multiAnswer) { if (json.multiAnswer) {
// No need to check this here, this has its own validation // No need to check this here, this has its own validation
@ -487,8 +508,8 @@ export class DetectNonErasedKeysInMappings extends DesugaringStep<QuestionableTa
.enters("freeform") .enters("freeform")
.warn( .warn(
"The freeform block does not modify the key `" + "The freeform block does not modify the key `" +
neededKey + neededKey +
"` which is set in a mapping. Use `addExtraTags` to overwrite it" "` which is set in a mapping. Use `addExtraTags` to overwrite it",
) )
} }
} }
@ -506,8 +527,8 @@ export class DetectNonErasedKeysInMappings extends DesugaringStep<QuestionableTa
.enters("mappings", i) .enters("mappings", i)
.warn( .warn(
"This mapping does not modify the key `" + "This mapping does not modify the key `" +
neededKey + neededKey +
"` which is set in a mapping or by the freeform block. Use `addExtraTags` to overwrite it" "` which is set in a mapping or by the freeform block. Use `addExtraTags` to overwrite it",
) )
} }
} }
@ -531,7 +552,7 @@ export class DetectShadowedMappings extends DesugaringStep<TagRenderingConfigJso
* DetectShadowedMappings.extractCalculatedTagNames({calculatedTags: ["_abc=js()"]}) // => ["_abc"] * DetectShadowedMappings.extractCalculatedTagNames({calculatedTags: ["_abc=js()"]}) // => ["_abc"]
*/ */
private static extractCalculatedTagNames( private static extractCalculatedTagNames(
layerConfig?: LayerConfigJson | { calculatedTags: string[] } layerConfig?: LayerConfigJson | { calculatedTags: string[] },
) { ) {
return ( return (
layerConfig?.calculatedTags?.map((ct) => { layerConfig?.calculatedTags?.map((ct) => {
@ -617,16 +638,16 @@ export class DetectShadowedMappings extends DesugaringStep<TagRenderingConfigJso
json.mappings[i]["hideInAnswer"] !== true json.mappings[i]["hideInAnswer"] !== true
) { ) {
context.warn( context.warn(
`Mapping ${i} is shadowed by mapping ${j}. However, mapping ${j} has 'hideInAnswer' set, which will result in a different rendering in question-mode.` `Mapping ${i} is shadowed by mapping ${j}. However, mapping ${j} has 'hideInAnswer' set, which will result in a different rendering in question-mode.`,
) )
} else if (doesMatch) { } else if (doesMatch) {
// The current mapping is shadowed! // The current mapping is shadowed!
context.err(`Mapping ${i} is shadowed by mapping ${j} and will thus never be shown: context.err(`Mapping ${i} is shadowed by mapping ${j} and will thus never be shown:
The mapping ${parsedConditions[i].asHumanString( The mapping ${parsedConditions[i].asHumanString(
false, false,
false, false,
{} {},
)} is fully matched by a previous mapping (namely ${j}), which matches: )} is fully matched by a previous mapping (namely ${j}), which matches:
${parsedConditions[j].asHumanString(false, false, {})}. ${parsedConditions[j].asHumanString(false, false, {})}.
To fix this problem, you can try to: To fix this problem, you can try to:
@ -653,7 +674,7 @@ export class DetectMappingsWithImages extends DesugaringStep<TagRenderingConfigJ
super( super(
"Checks that 'then'clauses in mappings don't have images, but use 'icon' instead", "Checks that 'then'clauses in mappings don't have images, but use 'icon' instead",
[], [],
"DetectMappingsWithImages" "DetectMappingsWithImages",
) )
this._doesImageExist = doesImageExist this._doesImageExist = doesImageExist
} }
@ -693,14 +714,14 @@ export class DetectMappingsWithImages extends DesugaringStep<TagRenderingConfigJ
if (!ignore) { if (!ignore) {
ctx.err( ctx.err(
`A mapping has an image in the 'then'-clause. Remove the image there and use \`"icon": <your-image>\` instead. The images found are ${images.join( `A mapping has an image in the 'then'-clause. Remove the image there and use \`"icon": <your-image>\` instead. The images found are ${images.join(
", " ", ",
)}. (This check can be turned of by adding "#": "${ignoreToken}" in the mapping, but this is discouraged` )}. (This check can be turned of by adding "#": "${ignoreToken}" in the mapping, but this is discouraged`,
) )
} else { } else {
ctx.info( ctx.info(
`Ignored image ${images.join( `Ignored image ${images.join(
", " ", ",
)} in 'then'-clause of a mapping as this check has been disabled` )} in 'then'-clause of a mapping as this check has been disabled`,
) )
for (const image of images) { for (const image of images) {
@ -721,7 +742,7 @@ class ValidatePossibleLinks extends DesugaringStep<string | Record<string, strin
super( super(
"Given a possible set of translations, validates that <a href=... target='_blank'> does have `rel='noopener'` set", "Given a possible set of translations, validates that <a href=... target='_blank'> does have `rel='noopener'` set",
[], [],
"ValidatePossibleLinks" "ValidatePossibleLinks",
) )
} }
@ -751,21 +772,21 @@ class ValidatePossibleLinks extends DesugaringStep<string | Record<string, strin
convert( convert(
json: string | Record<string, string>, json: string | Record<string, string>,
context: ConversionContext context: ConversionContext,
): string | Record<string, string> { ): string | Record<string, string> {
if (typeof json === "string") { if (typeof json === "string") {
if (this.isTabnabbingProne(json)) { if (this.isTabnabbingProne(json)) {
context.err( context.err(
"The string " + "The string " +
json + json +
" has a link targeting `_blank`, but it doesn't have `rel='noopener'` set. This gives rise to reverse tabnapping" " has a link targeting `_blank`, but it doesn't have `rel='noopener'` set. This gives rise to reverse tabnapping",
) )
} }
} else { } else {
for (const k in json) { for (const k in json) {
if (this.isTabnabbingProne(json[k])) { if (this.isTabnabbingProne(json[k])) {
context.err( context.err(
`The translation for ${k} '${json[k]}' has a link targeting \`_blank\`, but it doesn't have \`rel='noopener'\` set. This gives rise to reverse tabnapping` `The translation for ${k} '${json[k]}' has a link targeting \`_blank\`, but it doesn't have \`rel='noopener'\` set. This gives rise to reverse tabnapping`,
) )
} }
} }
@ -783,7 +804,7 @@ class CheckTranslation extends DesugaringStep<Translatable> {
super( super(
"Checks that a translation is valid and internally consistent", "Checks that a translation is valid and internally consistent",
["*"], ["*"],
"CheckTranslation" "CheckTranslation",
) )
this._allowUndefined = allowUndefined this._allowUndefined = allowUndefined
} }
@ -829,17 +850,17 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
convert( convert(
json: TagRenderingConfigJson | QuestionableTagRenderingConfigJson, json: TagRenderingConfigJson | QuestionableTagRenderingConfigJson,
context: ConversionContext context: ConversionContext,
): TagRenderingConfigJson { ): TagRenderingConfigJson {
if (json["special"] !== undefined) { if (json["special"] !== undefined) {
context.err( context.err(
'Detected `special` on the top level. Did you mean `{"render":{ "special": ... }}`' "Detected `special` on the top level. Did you mean `{\"render\":{ \"special\": ... }}`",
) )
} }
if (Object.keys(json).length === 1 && typeof json["render"] === "string") { if (Object.keys(json).length === 1 && typeof json["render"] === "string") {
context.warn( context.warn(
`use the content directly instead of {render: ${JSON.stringify(json["render"])}}` `use the content directly instead of {render: ${JSON.stringify(json["render"])}}`,
) )
} }
@ -851,7 +872,7 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
const mapping = json.mappings[i] const mapping = json.mappings[i]
CheckTranslation.noUndefined.convert( CheckTranslation.noUndefined.convert(
mapping.then, mapping.then,
context.enters("mappings", i, "then") context.enters("mappings", i, "then"),
) )
if (!mapping.if) { if (!mapping.if) {
context.enters("mappings", i).err("No `if` is defined") context.enters("mappings", i).err("No `if` is defined")
@ -862,18 +883,18 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
context context
.enters("mappings", i, "then") .enters("mappings", i, "then")
.warn( .warn(
"A mapping should not start with 'yes' or 'no'. If the attribute is known, it will only show 'yes' or 'no' <i>without</i> the question, resulting in a weird phrasing in the information box" "A mapping should not start with 'yes' or 'no'. If the attribute is known, it will only show 'yes' or 'no' <i>without</i> the question, resulting in a weird phrasing in the information box",
) )
} }
} }
} }
if (json["group"]) { if (json["group"]) {
context.err('Groups are deprecated, use `"label": ["' + json["group"] + '"]` instead') context.err("Groups are deprecated, use `\"label\": [\"" + json["group"] + "\"]` instead")
} }
if (json["question"] && json.freeform?.key === undefined && json.mappings === undefined) { if (json["question"] && json.freeform?.key === undefined && json.mappings === undefined) {
context.err( context.err(
"A question is defined, but no mappings nor freeform (key) are. Add at least one of them" "A question is defined, but no mappings nor freeform (key) are. Add at least one of them",
) )
} }
if (json["question"] && !json.freeform && (json.mappings?.length ?? 0) == 1) { if (json["question"] && !json.freeform && (json.mappings?.length ?? 0) == 1) {
@ -883,7 +904,7 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
context context
.enter("questionHint") .enter("questionHint")
.err( .err(
"A questionHint is defined, but no question is given. As such, the questionHint will never be shown" "A questionHint is defined, but no question is given. As such, the questionHint will never be shown",
) )
} }
@ -893,10 +914,10 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
.enter("render") .enter("render")
.err( .err(
"This tagRendering allows to set a value to key " + "This tagRendering allows to set a value to key " +
json.freeform.key + json.freeform.key +
", but does not define a `render`. Please, add a value here which contains `{" + ", but does not define a `render`. Please, add a value here which contains `{" +
json.freeform.key + json.freeform.key +
"}`" "}`",
) )
} else { } else {
const render = new Translation(<any>json.render) const render = new Translation(<any>json.render)
@ -927,7 +948,7 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
const keyFirstArg = ["canonical", "fediverse_link", "translated"] const keyFirstArg = ["canonical", "fediverse_link", "translated"]
if ( if (
keyFirstArg.some( keyFirstArg.some(
(funcName) => txt.indexOf(`{${funcName}(${json.freeform.key}`) >= 0 (funcName) => txt.indexOf(`{${funcName}(${json.freeform.key}`) >= 0,
) )
) { ) {
continue continue
@ -950,7 +971,7 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
context context
.enter("render") .enter("render")
.err( .err(
`The rendering for language ${ln} does not contain \`{${json.freeform.key}}\`. This is a bug, as this rendering should show exactly this freeform key!` `The rendering for language ${ln} does not contain \`{${json.freeform.key}}\`. This is a bug, as this rendering should show exactly this freeform key!`,
) )
} }
} }
@ -958,8 +979,8 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
if (json.render && json["question"] && json.freeform === undefined) { if (json.render && json["question"] && json.freeform === undefined) {
context.err( context.err(
`Detected a tagrendering which takes input without freeform key in ${context}; the question is ${new Translation( `Detected a tagrendering which takes input without freeform key in ${context}; the question is ${new Translation(
json["question"] json["question"],
).textFor("en")}` ).textFor("en")}`,
) )
} }
@ -970,9 +991,9 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
.enters("freeform", "type") .enters("freeform", "type")
.err( .err(
"Unknown type: " + "Unknown type: " +
freeformType + freeformType +
"; try one of " + "; try one of " +
Validators.availableTypes.join(", ") Validators.availableTypes.join(", "),
) )
} }
} }
@ -1004,7 +1025,7 @@ export class ValidateTagRenderings extends Fuse<TagRenderingConfigJson> {
new On("question", new ValidatePossibleLinks()), new On("question", new ValidatePossibleLinks()),
new On("questionHint", new ValidatePossibleLinks()), new On("questionHint", new ValidatePossibleLinks()),
new On("mappings", new Each(new On("then", new ValidatePossibleLinks()))), new On("mappings", new Each(new On("then", new ValidatePossibleLinks()))),
new MiscTagRenderingChecks() new MiscTagRenderingChecks(),
) )
} }
} }
@ -1034,8 +1055,9 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
if (json.id?.toLowerCase() !== json.id) { if (json.id?.toLowerCase() !== json.id) {
context.enter("id").err(`The id of a layer should be lowercase: ${json.id}`) context.enter("id").err(`The id of a layer should be lowercase: ${json.id}`)
} }
if (json.id?.match(/[a-z0-9-_]/) == null) { const layerRegex = /[a-zA-Z][a-zA-Z_0-9]+/
context.enter("id").err(`The id of a layer should match [a-z0-9-_]*: ${json.id}`) if (json.id.match(layerRegex) === null) {
context.enter("id").err("Invalid ID. A layer ID should match " + layerRegex.source)
} }
} }
@ -1043,7 +1065,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
context context
.enter("source") .enter("source")
.err( .err(
"No source section is defined; please define one as data is not loaded otherwise" "No source section is defined; please define one as data is not loaded otherwise",
) )
} else { } else {
if (json.source === "special" || json.source === "special:library") { if (json.source === "special" || json.source === "special:library") {
@ -1051,7 +1073,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
context context
.enters("source", "osmTags") .enters("source", "osmTags")
.err( .err(
"No osmTags defined in the source section - these should always be present, even for geojson layer" "No osmTags defined in the source section - these should always be present, even for geojson layer",
) )
} else { } else {
const osmTags = TagUtils.Tag(json.source["osmTags"], context + "source.osmTags") const osmTags = TagUtils.Tag(json.source["osmTags"], context + "source.osmTags")
@ -1060,7 +1082,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
.enters("source", "osmTags") .enters("source", "osmTags")
.err( .err(
"The source states tags which give a very wide selection: it only uses negative expressions, which will result in too much and unexpected data. Add at least one required tag. The tags are:\n\t" + "The source states tags which give a very wide selection: it only uses negative expressions, which will result in too much and unexpected data. Add at least one required tag. The tags are:\n\t" +
osmTags.asHumanString(false, false, {}) osmTags.asHumanString(false, false, {}),
) )
} }
} }
@ -1086,10 +1108,10 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
.enter("syncSelection") .enter("syncSelection")
.err( .err(
"Invalid sync-selection: must be one of " + "Invalid sync-selection: must be one of " +
LayerConfig.syncSelectionAllowed.map((v) => `'${v}'`).join(", ") + LayerConfig.syncSelectionAllowed.map((v) => `'${v}'`).join(", ") +
" but got '" + " but got '" +
json.syncSelection + json.syncSelection +
"'" "'",
) )
} }
if (json["pointRenderings"]?.length > 0) { if (json["pointRenderings"]?.length > 0) {
@ -1107,7 +1129,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
context.enter("pointRendering").err("There are no pointRenderings at all...") context.enter("pointRendering").err("There are no pointRenderings at all...")
} }
json.pointRendering?.forEach((pr,i) => this._validatePointRendering.convert(pr, context.enters("pointeRendering", i))) json.pointRendering?.forEach((pr, i) => this._validatePointRendering.convert(pr, context.enters("pointeRendering", i)))
if (json["mapRendering"]) { if (json["mapRendering"]) {
context.enter("mapRendering").err("This layer has a legacy 'mapRendering'") context.enter("mapRendering").err("This layer has a legacy 'mapRendering'")
@ -1123,8 +1145,8 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
if (!Constants.priviliged_layers.find((x) => x == json.id)) { if (!Constants.priviliged_layers.find((x) => x == json.id)) {
context.err( context.err(
"Layer " + "Layer " +
json.id + json.id +
" uses 'special' as source.osmTags. However, this layer is not a priviliged layer" " uses 'special' as source.osmTags. However, this layer is not a priviliged layer",
) )
} }
} }
@ -1139,19 +1161,19 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
context context
.enter("title") .enter("title")
.err( .err(
"This layer does not have a title defined but it does have tagRenderings. Not having a title will disable the popups, resulting in an unclickable element. Please add a title. If not having a popup is intended and the tagrenderings need to be kept (e.g. in a library layer), set `title: null` to disable this error." "This layer does not have a title defined but it does have tagRenderings. Not having a title will disable the popups, resulting in an unclickable element. Please add a title. If not having a popup is intended and the tagrenderings need to be kept (e.g. in a library layer), set `title: null` to disable this error.",
) )
} }
if (json.title === null) { if (json.title === null) {
context.info( context.info(
"Title is `null`. This results in an element that cannot be clicked - even though tagRenderings is set." "Title is `null`. This results in an element that cannot be clicked - even though tagRenderings is set.",
) )
} }
{ {
// Check for multiple, identical builtin questions - usability for studio users // Check for multiple, identical builtin questions - usability for studio users
const duplicates = Utils.Duplicates( const duplicates = Utils.Duplicates(
<string[]>json.tagRenderings.filter((tr) => typeof tr === "string") <string[]>json.tagRenderings.filter((tr) => typeof tr === "string"),
) )
for (let i = 0; i < json.tagRenderings.length; i++) { for (let i = 0; i < json.tagRenderings.length; i++) {
const tagRendering = json.tagRenderings[i] const tagRendering = json.tagRenderings[i]
@ -1181,7 +1203,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
{ {
// duplicate ids in tagrenderings check // duplicate ids in tagrenderings check
const duplicates = Utils.NoNull( const duplicates = Utils.NoNull(
Utils.Duplicates(Utils.NoNull((json.tagRenderings ?? []).map((tr) => tr["id"]))) Utils.Duplicates(Utils.NoNull((json.tagRenderings ?? []).map((tr) => tr["id"]))),
) )
if (duplicates.length > 0) { if (duplicates.length > 0) {
// It is tempting to add an index to this warning; however, due to labels the indices here might be different from the index in the tagRendering list // It is tempting to add an index to this warning; however, due to labels the indices here might be different from the index in the tagRendering list
@ -1219,8 +1241,8 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
if (json["overpassTags"] !== undefined) { if (json["overpassTags"] !== undefined) {
context.err( context.err(
"Layer " + "Layer " +
json.id + json.id +
'still uses the old \'overpassTags\'-format. Please use "source": {"osmTags": <tags>}\' instead of "overpassTags": <tags> (note: this isn\'t your fault, the custom theme generator still spits out the old format)' "still uses the old 'overpassTags'-format. Please use \"source\": {\"osmTags\": <tags>}' instead of \"overpassTags\": <tags> (note: this isn't your fault, the custom theme generator still spits out the old format)",
) )
} }
const forbiddenTopLevel = [ const forbiddenTopLevel = [
@ -1240,7 +1262,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
} }
if (json["hideUnderlayingFeaturesMinPercentage"] !== undefined) { if (json["hideUnderlayingFeaturesMinPercentage"] !== undefined) {
context.err( context.err(
"Layer " + json.id + " contains an old 'hideUnderlayingFeaturesMinPercentage'" "Layer " + json.id + " contains an old 'hideUnderlayingFeaturesMinPercentage'",
) )
} }
@ -1257,9 +1279,9 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
if (this._path != undefined && this._path.indexOf(expected) < 0) { if (this._path != undefined && this._path.indexOf(expected) < 0) {
context.err( context.err(
"Layer is in an incorrect place. The path is " + "Layer is in an incorrect place. The path is " +
this._path + this._path +
", but expected " + ", but expected " +
expected expected,
) )
} }
} }
@ -1277,13 +1299,13 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
.enter(["tagRenderings", ...emptyIndexes]) .enter(["tagRenderings", ...emptyIndexes])
.err( .err(
`Some tagrendering-ids are empty or have an emtpy string; this is not allowed (at ${emptyIndexes.join( `Some tagrendering-ids are empty or have an emtpy string; this is not allowed (at ${emptyIndexes.join(
"," ",",
)}])` )}])`,
) )
} }
const duplicateIds = Utils.Duplicates( const duplicateIds = Utils.Duplicates(
(json.tagRenderings ?? [])?.map((f) => f["id"]).filter((id) => id !== "questions") (json.tagRenderings ?? [])?.map((f) => f["id"]).filter((id) => id !== "questions"),
) )
if (duplicateIds.length > 0 && !Utils.runningFromConsole) { if (duplicateIds.length > 0 && !Utils.runningFromConsole) {
context context
@ -1307,7 +1329,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
if (json.tagRenderings !== undefined) { if (json.tagRenderings !== undefined) {
new On( new On(
"tagRenderings", "tagRenderings",
new Each(new ValidateTagRenderings(json, this._doesImageExist)) new Each(new ValidateTagRenderings(json, this._doesImageExist)),
).convert(json, context) ).convert(json, context)
} }
@ -1334,7 +1356,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
context context
.enters("pointRendering", i, "marker", indexM, "icon", "condition") .enters("pointRendering", i, "marker", indexM, "icon", "condition")
.err( .err(
"Don't set a condition in a marker as this will result in an invisible but clickable element. Use extra filters in the source instead." "Don't set a condition in a marker as this will result in an invisible but clickable element. Use extra filters in the source instead.",
) )
} }
} }
@ -1372,9 +1394,9 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
.enters("presets", i, "tags") .enters("presets", i, "tags")
.err( .err(
"This preset does not match the required tags of this layer. This implies that a newly added point will not show up.\n A newly created point will have properties: " + "This preset does not match the required tags of this layer. This implies that a newly added point will not show up.\n A newly created point will have properties: " +
tags.asHumanString(false, false, {}) + tags.asHumanString(false, false, {}) +
"\n The required tags are: " + "\n The required tags are: " +
baseTags.asHumanString(false, false, {}) baseTags.asHumanString(false, false, {}),
) )
} }
} }
@ -1391,7 +1413,7 @@ export class ValidateLayerConfig extends DesugaringStep<LayerConfigJson> {
isBuiltin: boolean, isBuiltin: boolean,
doesImageExist: DoesImageExist, doesImageExist: DoesImageExist,
studioValidations: boolean = false, studioValidations: boolean = false,
skipDefaultLayers: boolean = false skipDefaultLayers: boolean = false,
) { ) {
super("Thin wrapper around 'ValidateLayer", [], "ValidateLayerConfig") super("Thin wrapper around 'ValidateLayer", [], "ValidateLayerConfig")
this.validator = new ValidateLayer( this.validator = new ValidateLayer(
@ -1399,7 +1421,7 @@ export class ValidateLayerConfig extends DesugaringStep<LayerConfigJson> {
isBuiltin, isBuiltin,
doesImageExist, doesImageExist,
studioValidations, studioValidations,
skipDefaultLayers skipDefaultLayers,
) )
} }
@ -1428,12 +1450,12 @@ class ValidatePointRendering extends DesugaringStep<PointRenderingConfigJson> {
} }
if (json.marker && !Array.isArray(json.marker)) { if (json.marker && !Array.isArray(json.marker)) {
context.enter("marker").err( context.enter("marker").err(
"The marker in a pointRendering should be an array" "The marker in a pointRendering should be an array",
) )
} }
if (json.location.length == 0) { if (json.location.length == 0) {
context.enter("location").err ( context.enter("location").err(
"A pointRendering should have at least one 'location' to defined where it should be rendered. " "A pointRendering should have at least one 'location' to defined where it should be rendered. ",
) )
} }
return json return json
@ -1441,41 +1463,44 @@ class ValidatePointRendering extends DesugaringStep<PointRenderingConfigJson> {
} }
} }
export class ValidateLayer extends Conversion< export class ValidateLayer extends Conversion<
LayerConfigJson, LayerConfigJson,
{ parsed: LayerConfig; raw: LayerConfigJson } { parsed: LayerConfig; raw: LayerConfigJson }
> { > {
private readonly _skipDefaultLayers: boolean private readonly _skipDefaultLayers: boolean
private readonly _prevalidation: PrevalidateLayer private readonly _prevalidation: PrevalidateLayer
constructor( constructor(
path: string, path: string,
isBuiltin: boolean, isBuiltin: boolean,
doesImageExist: DoesImageExist, doesImageExist: DoesImageExist,
studioValidations: boolean = false, studioValidations: boolean = false,
skipDefaultLayers: boolean = false skipDefaultLayers: boolean = false,
) { ) {
super("Doesn't change anything, but emits warnings and errors", [], "ValidateLayer") super("Doesn't change anything, but emits warnings and errors", [], "ValidateLayer")
this._prevalidation = new PrevalidateLayer( this._prevalidation = new PrevalidateLayer(
path, path,
isBuiltin, isBuiltin,
doesImageExist, doesImageExist,
studioValidations studioValidations,
) )
this._skipDefaultLayers = skipDefaultLayers this._skipDefaultLayers = skipDefaultLayers
} }
convert( convert(
json: LayerConfigJson, json: LayerConfigJson,
context: ConversionContext context: ConversionContext,
): { parsed: LayerConfig; raw: LayerConfigJson } { ): { parsed: LayerConfig; raw: LayerConfigJson } {
context = context.inOperation(this.name) context = context.inOperation(this.name)
if (typeof json === "string") { if (typeof json === "string") {
context.err( context.err(
`Not a valid layer: the layerConfig is a string. 'npm run generate:layeroverview' might be needed` `Not a valid layer: the layerConfig is a string. 'npm run generate:layeroverview' might be needed`,
) )
return undefined return undefined
} }
if (this._skipDefaultLayers && Constants.added_by_default.indexOf(<any>json.id) >= 0) { if (this._skipDefaultLayers && Constants.added_by_default.indexOf(<any>json.id) >= 0) {
return { parsed: undefined, raw: json } return { parsed: undefined, raw: json }
} }
@ -1502,7 +1527,7 @@ export class ValidateLayer extends Conversion<
context context
.enters("calculatedTags", i) .enters("calculatedTags", i)
.err( .err(
`Invalid function definition: the custom javascript is invalid:${e}. The offending javascript code is:\n ${code}` `Invalid function definition: the custom javascript is invalid:${e}. The offending javascript code is:\n ${code}`,
) )
} }
} }
@ -1553,8 +1578,8 @@ export class ValidateFilter extends DesugaringStep<FilterConfigJson> {
.enters("fields", i) .enters("fields", i)
.err( .err(
`Invalid filter: ${type} is not a valid textfield type.\n\tTry one of ${Array.from( `Invalid filter: ${type} is not a valid textfield type.\n\tTry one of ${Array.from(
Validators.availableTypes Validators.availableTypes,
).join(",")}` ).join(",")}`,
) )
} }
} }
@ -1571,13 +1596,13 @@ export class DetectDuplicateFilters extends DesugaringStep<{
super( super(
"Tries to detect layers where a shared filter can be used (or where similar filters occur)", "Tries to detect layers where a shared filter can be used (or where similar filters occur)",
[], [],
"DetectDuplicateFilters" "DetectDuplicateFilters",
) )
} }
convert( convert(
json: { layers: LayerConfigJson[]; themes: LayoutConfigJson[] }, json: { layers: LayerConfigJson[]; themes: LayoutConfigJson[] },
context: ConversionContext context: ConversionContext,
): { layers: LayerConfigJson[]; themes: LayoutConfigJson[] } { ): { layers: LayerConfigJson[]; themes: LayoutConfigJson[] } {
const { layers, themes } = json const { layers, themes } = json
const perOsmTag = new Map< const perOsmTag = new Map<
@ -1641,7 +1666,7 @@ export class DetectDuplicateFilters extends DesugaringStep<{
filter: FilterConfigJson filter: FilterConfigJson
}[] }[]
>, >,
layout?: LayoutConfigJson | undefined layout?: LayoutConfigJson | undefined,
): void { ): void {
if (layer.filter === undefined || layer.filter === null) { if (layer.filter === undefined || layer.filter === null) {
return return
@ -1681,7 +1706,7 @@ export class DetectDuplicatePresets extends DesugaringStep<LayoutConfig> {
super( super(
"Detects mappings which have identical (english) names or identical mappings.", "Detects mappings which have identical (english) names or identical mappings.",
["presets"], ["presets"],
"DetectDuplicatePresets" "DetectDuplicatePresets",
) )
} }
@ -1692,13 +1717,13 @@ export class DetectDuplicatePresets extends DesugaringStep<LayoutConfig> {
if (new Set(enNames).size != enNames.length) { if (new Set(enNames).size != enNames.length) {
const dups = Utils.Duplicates(enNames) const dups = Utils.Duplicates(enNames)
const layersWithDup = json.layers.filter((l) => const layersWithDup = json.layers.filter((l) =>
l.presets.some((p) => dups.indexOf(p.title.textFor("en")) >= 0) l.presets.some((p) => dups.indexOf(p.title.textFor("en")) >= 0),
) )
const layerIds = layersWithDup.map((l) => l.id) const layerIds = layersWithDup.map((l) => l.id)
context.err( context.err(
`This themes has multiple presets which are named:${dups}, namely layers ${layerIds.join( `This themes has multiple presets which are named:${dups}, namely layers ${layerIds.join(
", " ", ",
)} this is confusing for contributors and is probably the result of reusing the same layer multiple times. Use \`{"override": {"=presets": []}}\` to remove some presets` )} this is confusing for contributors and is probably the result of reusing the same layer multiple times. Use \`{"override": {"=presets": []}}\` to remove some presets`,
) )
} }
@ -1713,17 +1738,17 @@ export class DetectDuplicatePresets extends DesugaringStep<LayoutConfig> {
Utils.SameObject(presetATags, presetBTags) && Utils.SameObject(presetATags, presetBTags) &&
Utils.sameList( Utils.sameList(
presetA.preciseInput.snapToLayers, presetA.preciseInput.snapToLayers,
presetB.preciseInput.snapToLayers presetB.preciseInput.snapToLayers,
) )
) { ) {
context.err( context.err(
`This themes has multiple presets with the same tags: ${presetATags.asHumanString( `This themes has multiple presets with the same tags: ${presetATags.asHumanString(
false, false,
false, false,
{} {},
)}, namely the preset '${presets[i].title.textFor("en")}' and '${presets[ )}, namely the preset '${presets[i].title.textFor("en")}' and '${presets[
j j
].title.textFor("en")}'` ].title.textFor("en")}'`,
) )
} }
} }
@ -1732,3 +1757,63 @@ export class DetectDuplicatePresets extends DesugaringStep<LayoutConfig> {
return json return json
} }
} }
export class ValidateThemeEnsemble extends Conversion<LayoutConfig[], Map<string, {
tags: TagsFilter,
foundInTheme: string[]
}>> {
constructor() {
super("Validates that all themes together are logical, i.e. no duplicate ids exists within (overriden) themes", [], "ValidateThemeEnsemble")
}
convert(json: LayoutConfig[], context: ConversionContext): Map<string, {
tags: TagsFilter,
foundInTheme: string[]
}> {
const idToSource = new Map<string, { tags: TagsFilter, foundInTheme: string[] }>()
for (const theme of json) {
for (const layer of theme.layers) {
if (typeof layer.source === "string") {
continue
}
if (Constants.priviliged_layers.indexOf(<any>layer.id) >= 0) {
continue
}
if (!layer.source) {
console.log(theme, layer, layer.source)
context.enters(theme.id, "layers", "source", layer.id).err("No source defined")
continue
}
if (layer.source.geojsonSource) {
continue
}
const id = layer.id
const tags = layer.source.osmTags
if (!idToSource.has(id)) {
idToSource.set(id, { tags, foundInTheme: [theme.id] })
continue
}
const oldTags = idToSource.get(id).tags
const oldTheme = idToSource.get(id).foundInTheme
if (oldTags.shadows(tags) && tags.shadows(oldTags)) {
// All is good, all is well
oldTheme.push(theme.id)
continue
}
context.err(["The layer with id '" + id + "' is found in multiple themes with different tag definitions:",
"\t In theme " + oldTheme + ":\t" + oldTags.asHumanString(false, false, {}),
"\tIn theme " + theme.id + ":\t" + tags.asHumanString(false, false, {}),
].join("\n"))
}
}
return idToSource
}
}