midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

LayerServer: improve script, add unique and valid id check to layers

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-01-23 22:03:22 +01:00
parent 1d6c9ec1ef
commit ee38cdb9d7
5 changed files with 313 additions and 199 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
Docs/SettingUpPSQL.md
View file

@ -10,26 +10,37 @@ Then activate following extensions for this database (right click > Create > Ext
- Postgis activeren (rechtsklikken > Create > extension)
- HStore activeren
Install osm2pgsql (hint: compile from source is painless)
Increase the max number of connections. osm2pgsql needs connection one per table (and a few more), and since we are making one table per layer in MapComplete, this amounts to a lot.
pg_tileserv kan hier gedownload worden: https://github.com/CrunchyData/pg_tileserv
DATABASE_URL=postgresql://user:none@localhost:5444/osm-poi ./pg_tileserv
- Open PGAdmin, open the PGSQL-tool (CLI-button at the top)
- Run `max_connections = 2000;` and `show config_file;` to get the config file location (in docker). This is probably `/var/lib/postgresql/data/postgresql.conf`
- In a terminal, run `sudo docker exec -i <docker-container-id> bash` (run `sudo docker ps` to get the container id)
- `sed -i "s/max_connections = 100/max_connections = 5000/" /var/lib/postgresql/data/postgresql.conf`
- Validate with `cat /var/lib/postgresql/data/postgresql.conf | grep "max_connections"`
- `sudo docker restart <ID>`
## Create export scripts for every layer
Use scripts/osm2pgsl
Use `vite-node ./scripts/osm2pgsql/generateBuildDbScript.ts`
## Importing data
Install osm2pgsql (hint: compile from source is painless)
To seed the database:
````
osm2pgsql -O flex -E 4326 -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi andorra-latest.osm.pbf
osm2pgsql -O flex -E 4326 -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi <file>.osm.pbf
````
Storing properties to table '"public"."osm2pgsql_properties" takes about 25 minutes with planet.osm
Belgium (~555mb) takes 15m
World (80GB) should take 15m*160 = 2400m = 40hr
## Deploying a tile server
pg_tileserv kan hier gedownload worden: https://github.com/CrunchyData/pg_tileserv
````
export DATABASE_URL=postgresql://user:password@localhost:5444/osm-poi
./pg_tileserv

9
scripts/Script.ts
View file

@ -13,8 +13,15 @@ export default abstract class Script {
ScriptUtils.fixUtils()
const args = [...process.argv]
args.splice(0, 2)
const start = new Date()
this.main(args)
.then((_) => console.log("All done"))
.then((_) =>{
const end = new Date()
const millisNeeded = end.getTime() - start.getTime()
const green = (s) => "\x1b[92m" + s + "\x1b[0m"
console.log(green("All done! (" + millisNeeded + " ms)"))
})
.catch((e) => console.log("ERROR:", e))
}

25
scripts/generateLayerOverview.ts
View file

@ -9,7 +9,7 @@ import {
DoesImageExist,
PrevalidateTheme,
ValidateLayer,
ValidateThemeAndLayers,
ValidateThemeAndLayers, ValidateThemeEnsemble,
} from "../src/Models/ThemeConfig/Conversion/Validation"
import { Translation } from "../src/UI/i18n/Translation"
import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
@ -25,6 +25,8 @@ import LayerConfig from "../src/Models/ThemeConfig/LayerConfig"
import PointRenderingConfig from "../src/Models/ThemeConfig/PointRenderingConfig"
import { ConversionContext } from "../src/Models/ThemeConfig/Conversion/ConversionContext"
import { GenerateFavouritesLayer } from "./generateFavouritesLayer"
import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig"
import { TagsFilter } from "../src/Logic/Tags/TagsFilter"
// This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files.
// It spits out an overview of those to be used to load them
@ -123,6 +125,7 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye
}
}
class LayerOverviewUtils extends Script {
public static readonly layerPath = "./src/assets/generated/layers/"
public static readonly themePath = "./src/assets/generated/themes/"
@ -355,7 +358,6 @@ class LayerOverviewUtils extends Script {
const layerWhitelist = new Set(args.find(a => a.startsWith("--layers="))
?.substring("--layers=".length)?.split(",") ?? [])
const start = new Date()
const forceReload = args.some((a) => a == "--force")
const licensePaths = new Set<string>()
@ -382,9 +384,13 @@ class LayerOverviewUtils extends Script {
sharedLayers,
recompiledThemes,
forceReload,
themeWhitelist
themeWhitelist,
)
new ValidateThemeEnsemble().convertStrict(
Array.from(sharedThemes.values()).map(th => new LayoutConfig(th, true)))
if (recompiledThemes.length > 0) {
writeFileSync(
"./src/assets/generated/known_layers.json",
@ -446,17 +452,10 @@ class LayerOverviewUtils extends Script {
)
}
const end = new Date()
const millisNeeded = end.getTime() - start.getTime()
if (AllSharedLayers.getSharedLayersConfigs().size == 0) {
console.error(
"This was a bootstrapping-run. Run generate layeroverview again!(" +
millisNeeded +
" ms)",
"This was a bootstrapping-run. Run generate layeroverview again!"
)
} else {
const green = (s) => "\x1b[92m" + s + "\x1b[0m"
console.log(green("All done! (" + millisNeeded + " ms)"))
}
}
@ -482,7 +481,7 @@ class LayerOverviewUtils extends Script {
private buildLayerIndex(
doesImageExist: DoesImageExist,
forceReload: boolean,
whitelist: Set<string>
whitelist: Set<string>,
): Map<string, LayerConfigJson> {
// First, we expand and validate all builtin layers. These are written to src/assets/generated/layers
// At the same time, an index of available layers is built.
@ -672,7 +671,7 @@ class LayerOverviewUtils extends Script {
sharedLayers: Map<string, LayerConfigJson>,
recompiledThemes: string[],
forceReload: boolean,
whitelist: Set<string>
whitelist: Set<string>,
): Map<string, LayoutConfigJson> {
console.log(" ---------- VALIDATING BUILTIN THEMES ---------")
const themeFiles = ScriptUtils.getThemeFiles()

52
scripts/osm2pgsql/generateBuildDbScript.ts
View file

@ -1,13 +1,13 @@
import LayerConfig from "../../src/Models/ThemeConfig/LayerConfig"
import { TagsFilter } from "../../src/Logic/Tags/TagsFilter"
import { Tag } from "../../src/Logic/Tags/Tag"
import { And } from "../../src/Logic/Tags/And"
import Script from "../Script"
import { AllSharedLayers } from "../../src/Customizations/AllSharedLayers"
import fs from "fs"
import { Or } from "../../src/Logic/Tags/Or"
import { RegexTag } from "../../src/Logic/Tags/RegexTag"
import { Utils } from "../../src/Utils"
import { ValidateThemeEnsemble } from "../../src/Models/ThemeConfig/Conversion/Validation"
import { AllKnownLayouts } from "../../src/Customizations/AllKnownLayouts"
class LuaSnippets {
/**
@ -35,28 +35,31 @@ class LuaSnippets {
}
class GenerateLayerLua {
private readonly _layer: LayerConfig
private readonly _id: string
private readonly _tags: TagsFilter
private readonly _foundInThemes: string[]
constructor(layer: LayerConfig) {
this._layer = layer
constructor(id: string, tags: TagsFilter, foundInThemes: string[] = []) {
this._tags = tags
this._id = id
this._foundInThemes = foundInThemes
}
public functionName() {
const l = this._layer
if (!l.source?.osmTags) {
if (!this._tags) {
return undefined
}
return `process_poi_${l.id}`
return `process_poi_${this._id}`
}
public generateFunction(): string {
const l = this._layer
if (!l.source?.osmTags) {
if (!this._tags) {
return undefined
}
return [
`local pois_${l.id} = osm2pgsql.define_table({`,
` name = '${l.id}',`,
`local pois_${this._id} = osm2pgsql.define_table({`,
this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "",
` name = '${this._id}',`,
" ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },",
" columns = {",
" { column = 'tags', type = 'jsonb' },",
@ -66,7 +69,7 @@ class GenerateLayerLua {
"",
"",
`function ${this.functionName()}(object, geom)`,
" local matches_filter = " + this.toLuaFilter(l.source.osmTags),
" local matches_filter = " + this.toLuaFilter(this._tags),
" if( not matches_filter) then",
" return",
" end",
@ -75,7 +78,7 @@ class GenerateLayerLua {
" tags = object.tags",
" }",
" ",
` pois_${l.id}:insert(a)`,
` pois_${this._id}:insert(a)`,
"end",
"",
].join("\n")
@ -86,6 +89,8 @@ class GenerateLayerLua {
return `object.tags["${tag.key}"] ~= "${tag.value}"`
}
const v = (<RegExp> tag.value).source.replace(/\\\//g, "/")
if ("" + tag.value === "/.+/is" && !tag.invert) {
return `object.tags["${tag.key}"] ~= nil`
}
@ -104,10 +109,10 @@ class GenerateLayerLua {
}
if (tag.invert) {
return `object.tags["${tag.key}"] == nil or not string.find(object.tags["${tag.key}"], "${tag.value}")`
return `object.tags["${tag.key}"] == nil or not string.find(object.tags["${tag.key}"], "${v}")`
}
return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${tag.value}"))`
return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${v}"))`
}
private toLuaFilter(tag: TagsFilter, useParens: boolean = false): string {
@ -141,15 +146,21 @@ class GenerateLayerLua {
}
}
class GenerateLayerFile extends Script {
class GenerateBuildDbScript extends Script {
constructor() {
super("Generates a .lua-file to use with osm2pgsql")
}
async main(args: string[]) {
const layers = Array.from(AllSharedLayers.sharedLayers.values())
const allNeededLayers = new ValidateThemeEnsemble().convertStrict(
AllKnownLayouts.allKnownLayouts.values(),
)
const generators = layers.filter(l => l.source.geojsonSource === undefined).map(l => new GenerateLayerLua(l))
const generators: GenerateLayerLua[] = []
allNeededLayers.forEach(({ tags, foundInTheme }, layerId) => {
generators.push(new GenerateLayerLua(layerId, tags, foundInTheme))
})
const script = [
...generators.map(g => g.generateFunction()),
@ -159,7 +170,8 @@ class GenerateLayerFile extends Script {
const path = "build_db.lua"
fs.writeFileSync(path, script, "utf-8")
console.log("Written", path)
console.log(allNeededLayers.size+" layers will be created. Make sure to set 'max_connections' to at least "+(10 + allNeededLayers.size) )
}
}
new GenerateLayerFile().run()
new GenerateBuildDbScript().run()

285
src/Models/ThemeConfig/Conversion/Validation.ts
View file

@ -21,9 +21,7 @@ import PresetConfig from "../PresetConfig"
import { TagsFilter } from "../../../Logic/Tags/TagsFilter"
import { Translatable } from "../Json/Translatable"
import { ConversionContext } from "./ConversionContext"
import * as eli from "../../../assets/editor-layer-index.json"
import { AvailableRasterLayers } from "../../RasterLayers"
import Back from "../../../assets/svg/Back.svelte"
import PointRenderingConfigJson from "../Json/PointRenderingConfigJson"
class ValidateLanguageCompleteness extends DesugaringStep<LayoutConfig> {
@ -33,7 +31,7 @@ class ValidateLanguageCompleteness extends DesugaringStep<LayoutConfig> {
super(
"Checks that the given object is fully translated in the specified languages",
[],
"ValidateLanguageCompleteness"
"ValidateLanguageCompleteness",
)
this._languages = languages ?? ["en"]
}
@ -47,7 +45,7 @@ class ValidateLanguageCompleteness extends DesugaringStep<LayoutConfig> {
.filter(
(t) =>
t.tr.translations[neededLanguage] === undefined &&
t.tr.translations["*"] === undefined
t.tr.translations["*"] === undefined,
)
.forEach((missing) => {
context
@ -58,7 +56,7 @@ class ValidateLanguageCompleteness extends DesugaringStep<LayoutConfig> {
", but it lacks a translation for " +
missing.context +
".\n\tThe known translation is " +
missing.tr.textFor("en")
missing.tr.textFor("en"),
)
})
}
@ -75,7 +73,7 @@ export class DoesImageExist extends DesugaringStep<string> {
constructor(
knownImagePaths: Set<string>,
checkExistsSync: (path: string) => boolean = undefined,
ignore?: Set<string>
ignore?: Set<string>,
) {
super("Checks if an image exists", [], "DoesImageExist")
this._ignore = ignore
@ -111,15 +109,15 @@ export class DoesImageExist extends DesugaringStep<string> {
if (!this._knownImagePaths.has(image)) {
if (this.doesPathExist === undefined) {
context.err(
`Image with path ${image} not found or not attributed; it is used in ${context}`
`Image with path ${image} not found or not attributed; it is used in ${context}`,
)
} else if (!this.doesPathExist(image)) {
context.err(
`Image with path ${image} does not exist.\n Check for typo's and missing directories in the path.`
`Image with path ${image} does not exist.\n Check for typo's and missing directories in the path.`,
)
} else {
context.err(
`Image with path ${image} is not attributed (but it exists); execute 'npm run query:licenses' to add the license information and/or run 'npm run generate:licenses' to compile all the license info`
`Image with path ${image} is not attributed (but it exists); execute 'npm run query:licenses' to add the license information and/or run 'npm run generate:licenses' to compile all the license info`,
)
}
}
@ -143,7 +141,7 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
doesImageExist: DoesImageExist,
path: string,
isBuiltin: boolean,
sharedTagRenderings?: Set<string>
sharedTagRenderings?: Set<string>,
) {
super("Doesn't change anything, but emits warnings and errors", [], "ValidateTheme")
this._validateImage = doesImageExist
@ -163,14 +161,14 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
context.err(
"The theme " +
json.id +
" has units defined - these should be defined on the layer instead. (Hint: use overrideAll: { '+units': ... }) "
" has units defined - these should be defined on the layer instead. (Hint: use overrideAll: { '+units': ... }) ",
)
}
if (json["roamingRenderings"] !== undefined) {
context.err(
"Theme " +
json.id +
" contains an old 'roamingRenderings'. Use an 'overrideAll' instead"
" contains an old 'roamingRenderings'. Use an 'overrideAll' instead",
)
}
}
@ -191,7 +189,7 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
remoteImage.path +
" in theme " +
json.id +
", please download it."
", please download it.",
)
}
for (const image of images) {
@ -207,7 +205,7 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
const filename = this._path.substring(
this._path.lastIndexOf("/") + 1,
this._path.length - 5
this._path.length - 5,
)
if (theme.id !== filename) {
context.err(
@ -217,7 +215,7 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
filename +
" (" +
this._path +
")"
")",
)
}
this._validateImage.convert(theme.icon, context.enter("icon"))
@ -225,13 +223,13 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
const dups = Utils.Duplicates(json.layers.map((layer) => layer["id"]))
if (dups.length > 0) {
context.err(
`The theme ${json.id} defines multiple layers with id ${dups.join(", ")}`
`The theme ${json.id} defines multiple layers with id ${dups.join(", ")}`,
)
}
if (json["mustHaveLanguage"] !== undefined) {
new ValidateLanguageCompleteness(...json["mustHaveLanguage"]).convert(
theme,
context
context,
)
}
if (!json.hideFromOverview && theme.id !== "personal" && this._isBuiltin) {
@ -239,7 +237,7 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
const targetLanguage = theme.title.SupportedLanguages()[0]
if (targetLanguage !== "en") {
context.err(
`TargetLanguage is not 'en' for public theme ${theme.id}, it is ${targetLanguage}. Move 'en' up in the title of the theme and set it as the first key`
`TargetLanguage is not 'en' for public theme ${theme.id}, it is ${targetLanguage}. Move 'en' up in the title of the theme and set it as the first key`,
)
}
@ -282,6 +280,13 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
}
}
for (let i = 0; i < theme.layers.length; i++) {
const layer = theme.layers[i]
if (!layer.id.match("[a-z][a-z0-9_]*")) {
context.enters("layers", i, "id").err("Invalid ID:" + layer.id + "should match [a-z][a-z0-9_]*")
}
}
return json
}
}
@ -291,7 +296,7 @@ export class ValidateThemeAndLayers extends Fuse<LayoutConfigJson> {
doesImageExist: DoesImageExist,
path: string,
isBuiltin: boolean,
sharedTagRenderings?: Set<string>
sharedTagRenderings?: Set<string>,
) {
super(
"Validates a theme and the contained layers",
@ -301,10 +306,10 @@ export class ValidateThemeAndLayers extends Fuse<LayoutConfigJson> {
new Each(
new Bypass(
(layer) => Constants.added_by_default.indexOf(<any>layer.id) < 0,
new ValidateLayerConfig(undefined, isBuiltin, doesImageExist, false, true)
)
)
)
new ValidateLayerConfig(undefined, isBuiltin, doesImageExist, false, true),
),
),
),
)
}
}
@ -314,7 +319,7 @@ class OverrideShadowingCheck extends DesugaringStep<LayoutConfigJson> {
super(
"Checks that an 'overrideAll' does not override a single override",
[],
"OverrideShadowingCheck"
"OverrideShadowingCheck",
)
}
@ -363,6 +368,22 @@ class MiscThemeChecks extends DesugaringStep<LayoutConfigJson> {
if (json.socialImage === "") {
context.warn("Social image for theme " + json.id + " is the emtpy string")
}
{
for (let i = 0; i < json.layers.length; i++) {
const l = json.layers[i]
if (l["override"]?.["source"] === undefined) {
continue
}
if (l["override"]?.["source"]?.["geoJson"]) {
continue // We don't care about external data as we won't cache it anyway
}
if (l["override"]["id"] !== undefined) {
continue
}
context.enters("layers", i).err("A layer which changes the source-tags must also change the ID")
}
}
return json
}
}
@ -372,7 +393,7 @@ export class PrevalidateTheme extends Fuse<LayoutConfigJson> {
super(
"Various consistency checks on the raw JSON",
new MiscThemeChecks(),
new OverrideShadowingCheck()
new OverrideShadowingCheck(),
)
}
}
@ -382,7 +403,7 @@ export class DetectConflictingAddExtraTags extends DesugaringStep<TagRenderingCo
super(
"The `if`-part in a mapping might set some keys. Those keys are not allowed to be set in the `addExtraTags`, as this might result in conflicting values",
[],
"DetectConflictingAddExtraTags"
"DetectConflictingAddExtraTags",
)
}
@ -409,7 +430,7 @@ export class DetectConflictingAddExtraTags extends DesugaringStep<TagRenderingCo
.enters("mappings", i)
.err(
"AddExtraTags overrides a key that is set in the `if`-clause of this mapping. Selecting this answer might thus first set one value (needed to match as answer) and then override it with a different value, resulting in an unsaveable question. The offending `addExtraTags` is " +
duplicateKeys.join(", ")
duplicateKeys.join(", "),
)
}
}
@ -427,13 +448,13 @@ export class DetectNonErasedKeysInMappings extends DesugaringStep<QuestionableTa
super(
"A tagRendering might set a freeform key (e.g. `name` and have an option that _should_ erase this name, e.g. `noname=yes`). Under normal circumstances, every mapping/freeform should affect all touched keys",
[],
"DetectNonErasedKeysInMappings"
"DetectNonErasedKeysInMappings",
)
}
convert(
json: QuestionableTagRenderingConfigJson,
context: ConversionContext
context: ConversionContext,
): QuestionableTagRenderingConfigJson {
if (json.multiAnswer) {
// No need to check this here, this has its own validation
@ -488,7 +509,7 @@ export class DetectNonErasedKeysInMappings extends DesugaringStep<QuestionableTa
.warn(
"The freeform block does not modify the key `" +
neededKey +
"` which is set in a mapping. Use `addExtraTags` to overwrite it"
"` which is set in a mapping. Use `addExtraTags` to overwrite it",
)
}
}
@ -507,7 +528,7 @@ export class DetectNonErasedKeysInMappings extends DesugaringStep<QuestionableTa
.warn(
"This mapping does not modify the key `" +
neededKey +
"` which is set in a mapping or by the freeform block. Use `addExtraTags` to overwrite it"
"` which is set in a mapping or by the freeform block. Use `addExtraTags` to overwrite it",
)
}
}
@ -531,7 +552,7 @@ export class DetectShadowedMappings extends DesugaringStep<TagRenderingConfigJso
* DetectShadowedMappings.extractCalculatedTagNames({calculatedTags: ["_abc=js()"]}) // => ["_abc"]
*/
private static extractCalculatedTagNames(
layerConfig?: LayerConfigJson | { calculatedTags: string[] }
layerConfig?: LayerConfigJson | { calculatedTags: string[] },
) {
return (
layerConfig?.calculatedTags?.map((ct) => {
@ -617,7 +638,7 @@ export class DetectShadowedMappings extends DesugaringStep<TagRenderingConfigJso
json.mappings[i]["hideInAnswer"] !== true
) {
context.warn(
`Mapping ${i} is shadowed by mapping ${j}. However, mapping ${j} has 'hideInAnswer' set, which will result in a different rendering in question-mode.`
`Mapping ${i} is shadowed by mapping ${j}. However, mapping ${j} has 'hideInAnswer' set, which will result in a different rendering in question-mode.`,
)
} else if (doesMatch) {
// The current mapping is shadowed!
@ -625,7 +646,7 @@ export class DetectShadowedMappings extends DesugaringStep<TagRenderingConfigJso
The mapping ${parsedConditions[i].asHumanString(
false,
false,
{}
{},
)} is fully matched by a previous mapping (namely ${j}), which matches:
${parsedConditions[j].asHumanString(false, false, {})}.
@ -653,7 +674,7 @@ export class DetectMappingsWithImages extends DesugaringStep<TagRenderingConfigJ
super(
"Checks that 'then'clauses in mappings don't have images, but use 'icon' instead",
[],
"DetectMappingsWithImages"
"DetectMappingsWithImages",
)
this._doesImageExist = doesImageExist
}
@ -693,14 +714,14 @@ export class DetectMappingsWithImages extends DesugaringStep<TagRenderingConfigJ
if (!ignore) {
ctx.err(
`A mapping has an image in the 'then'-clause. Remove the image there and use \`"icon": <your-image>\` instead. The images found are ${images.join(
", "
)}. (This check can be turned of by adding "#": "${ignoreToken}" in the mapping, but this is discouraged`
", ",
)}. (This check can be turned of by adding "#": "${ignoreToken}" in the mapping, but this is discouraged`,
)
} else {
ctx.info(
`Ignored image ${images.join(
", "
)} in 'then'-clause of a mapping as this check has been disabled`
", ",
)} in 'then'-clause of a mapping as this check has been disabled`,
)
for (const image of images) {
@ -721,7 +742,7 @@ class ValidatePossibleLinks extends DesugaringStep<string | Record<string, strin
super(
"Given a possible set of translations, validates that <a href=... target='_blank'> does have `rel='noopener'` set",
[],
"ValidatePossibleLinks"
"ValidatePossibleLinks",
)
}
@ -751,21 +772,21 @@ class ValidatePossibleLinks extends DesugaringStep<string | Record<string, strin
convert(
json: string | Record<string, string>,
context: ConversionContext
context: ConversionContext,
): string | Record<string, string> {
if (typeof json === "string") {
if (this.isTabnabbingProne(json)) {
context.err(
"The string " +
json +
" has a link targeting `_blank`, but it doesn't have `rel='noopener'` set. This gives rise to reverse tabnapping"
" has a link targeting `_blank`, but it doesn't have `rel='noopener'` set. This gives rise to reverse tabnapping",
)
}
} else {
for (const k in json) {
if (this.isTabnabbingProne(json[k])) {
context.err(
`The translation for ${k} '${json[k]}' has a link targeting \`_blank\`, but it doesn't have \`rel='noopener'\` set. This gives rise to reverse tabnapping`
`The translation for ${k} '${json[k]}' has a link targeting \`_blank\`, but it doesn't have \`rel='noopener'\` set. This gives rise to reverse tabnapping`,
)
}
}
@ -783,7 +804,7 @@ class CheckTranslation extends DesugaringStep<Translatable> {
super(
"Checks that a translation is valid and internally consistent",
["*"],
"CheckTranslation"
"CheckTranslation",
)
this._allowUndefined = allowUndefined
}
@ -829,17 +850,17 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
convert(
json: TagRenderingConfigJson | QuestionableTagRenderingConfigJson,
context: ConversionContext
context: ConversionContext,
): TagRenderingConfigJson {
if (json["special"] !== undefined) {
context.err(
'Detected `special` on the top level. Did you mean `{"render":{ "special": ... }}`'
"Detected `special` on the top level. Did you mean `{\"render\":{ \"special\": ... }}`",
)
}
if (Object.keys(json).length === 1 && typeof json["render"] === "string") {
context.warn(
`use the content directly instead of {render: ${JSON.stringify(json["render"])}}`
`use the content directly instead of {render: ${JSON.stringify(json["render"])}}`,
)
}
@ -851,7 +872,7 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
const mapping = json.mappings[i]
CheckTranslation.noUndefined.convert(
mapping.then,
context.enters("mappings", i, "then")
context.enters("mappings", i, "then"),
)
if (!mapping.if) {
context.enters("mappings", i).err("No `if` is defined")
@ -862,18 +883,18 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
context
.enters("mappings", i, "then")
.warn(
"A mapping should not start with 'yes' or 'no'. If the attribute is known, it will only show 'yes' or 'no' <i>without</i> the question, resulting in a weird phrasing in the information box"
"A mapping should not start with 'yes' or 'no'. If the attribute is known, it will only show 'yes' or 'no' <i>without</i> the question, resulting in a weird phrasing in the information box",
)
}
}
}
if (json["group"]) {
context.err('Groups are deprecated, use `"label": ["' + json["group"] + '"]` instead')
context.err("Groups are deprecated, use `\"label\": [\"" + json["group"] + "\"]` instead")
}
if (json["question"] && json.freeform?.key === undefined && json.mappings === undefined) {
context.err(
"A question is defined, but no mappings nor freeform (key) are. Add at least one of them"
"A question is defined, but no mappings nor freeform (key) are. Add at least one of them",
)
}
if (json["question"] && !json.freeform && (json.mappings?.length ?? 0) == 1) {
@ -883,7 +904,7 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
context
.enter("questionHint")
.err(
"A questionHint is defined, but no question is given. As such, the questionHint will never be shown"
"A questionHint is defined, but no question is given. As such, the questionHint will never be shown",
)
}
@ -896,7 +917,7 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
json.freeform.key +
", but does not define a `render`. Please, add a value here which contains `{" +
json.freeform.key +
"}`"
"}`",
)
} else {
const render = new Translation(<any>json.render)
@ -927,7 +948,7 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
const keyFirstArg = ["canonical", "fediverse_link", "translated"]
if (
keyFirstArg.some(
(funcName) => txt.indexOf(`{${funcName}(${json.freeform.key}`) >= 0
(funcName) => txt.indexOf(`{${funcName}(${json.freeform.key}`) >= 0,
)
) {
continue
@ -950,7 +971,7 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
context
.enter("render")
.err(
`The rendering for language ${ln} does not contain \`{${json.freeform.key}}\`. This is a bug, as this rendering should show exactly this freeform key!`
`The rendering for language ${ln} does not contain \`{${json.freeform.key}}\`. This is a bug, as this rendering should show exactly this freeform key!`,
)
}
}
@ -958,8 +979,8 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
if (json.render && json["question"] && json.freeform === undefined) {
context.err(
`Detected a tagrendering which takes input without freeform key in ${context}; the question is ${new Translation(
json["question"]
).textFor("en")}`
json["question"],
).textFor("en")}`,
)
}
@ -972,7 +993,7 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
"Unknown type: " +
freeformType +
"; try one of " +
Validators.availableTypes.join(", ")
Validators.availableTypes.join(", "),
)
}
}
@ -1004,7 +1025,7 @@ export class ValidateTagRenderings extends Fuse<TagRenderingConfigJson> {
new On("question", new ValidatePossibleLinks()),
new On("questionHint", new ValidatePossibleLinks()),
new On("mappings", new Each(new On("then", new ValidatePossibleLinks()))),
new MiscTagRenderingChecks()
new MiscTagRenderingChecks(),
)
}
}
@ -1034,8 +1055,9 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
if (json.id?.toLowerCase() !== json.id) {
context.enter("id").err(`The id of a layer should be lowercase: ${json.id}`)
}
if (json.id?.match(/[a-z0-9-_]/) == null) {
context.enter("id").err(`The id of a layer should match [a-z0-9-_]*: ${json.id}`)
const layerRegex = /[a-zA-Z][a-zA-Z_0-9]+/
if (json.id.match(layerRegex) === null) {
context.enter("id").err("Invalid ID. A layer ID should match " + layerRegex.source)
}
}
@ -1043,7 +1065,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
context
.enter("source")
.err(
"No source section is defined; please define one as data is not loaded otherwise"
"No source section is defined; please define one as data is not loaded otherwise",
)
} else {
if (json.source === "special" || json.source === "special:library") {
@ -1051,7 +1073,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
context
.enters("source", "osmTags")
.err(
"No osmTags defined in the source section - these should always be present, even for geojson layer"
"No osmTags defined in the source section - these should always be present, even for geojson layer",
)
} else {
const osmTags = TagUtils.Tag(json.source["osmTags"], context + "source.osmTags")
@ -1060,7 +1082,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
.enters("source", "osmTags")
.err(
"The source states tags which give a very wide selection: it only uses negative expressions, which will result in too much and unexpected data. Add at least one required tag. The tags are:\n\t" +
osmTags.asHumanString(false, false, {})
osmTags.asHumanString(false, false, {}),
)
}
}
@ -1089,7 +1111,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
LayerConfig.syncSelectionAllowed.map((v) => `'${v}'`).join(", ") +
" but got '" +
json.syncSelection +
"'"
"'",
)
}
if (json["pointRenderings"]?.length > 0) {
@ -1124,7 +1146,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
context.err(
"Layer " +
json.id +
" uses 'special' as source.osmTags. However, this layer is not a priviliged layer"
" uses 'special' as source.osmTags. However, this layer is not a priviliged layer",
)
}
}
@ -1139,19 +1161,19 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
context
.enter("title")
.err(
"This layer does not have a title defined but it does have tagRenderings. Not having a title will disable the popups, resulting in an unclickable element. Please add a title. If not having a popup is intended and the tagrenderings need to be kept (e.g. in a library layer), set `title: null` to disable this error."
"This layer does not have a title defined but it does have tagRenderings. Not having a title will disable the popups, resulting in an unclickable element. Please add a title. If not having a popup is intended and the tagrenderings need to be kept (e.g. in a library layer), set `title: null` to disable this error.",
)
}
if (json.title === null) {
context.info(
"Title is `null`. This results in an element that cannot be clicked - even though tagRenderings is set."
"Title is `null`. This results in an element that cannot be clicked - even though tagRenderings is set.",
)
}
{
// Check for multiple, identical builtin questions - usability for studio users
const duplicates = Utils.Duplicates(
<string[]>json.tagRenderings.filter((tr) => typeof tr === "string")
<string[]>json.tagRenderings.filter((tr) => typeof tr === "string"),
)
for (let i = 0; i < json.tagRenderings.length; i++) {
const tagRendering = json.tagRenderings[i]
@ -1181,7 +1203,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
{
// duplicate ids in tagrenderings check
const duplicates = Utils.NoNull(
Utils.Duplicates(Utils.NoNull((json.tagRenderings ?? []).map((tr) => tr["id"])))
Utils.Duplicates(Utils.NoNull((json.tagRenderings ?? []).map((tr) => tr["id"]))),
)
if (duplicates.length > 0) {
// It is tempting to add an index to this warning; however, due to labels the indices here might be different from the index in the tagRendering list
@ -1220,7 +1242,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
context.err(
"Layer " +
json.id +
'still uses the old \'overpassTags\'-format. Please use "source": {"osmTags": <tags>}\' instead of "overpassTags": <tags> (note: this isn\'t your fault, the custom theme generator still spits out the old format)'
"still uses the old 'overpassTags'-format. Please use \"source\": {\"osmTags\": <tags>}' instead of \"overpassTags\": <tags> (note: this isn't your fault, the custom theme generator still spits out the old format)",
)
}
const forbiddenTopLevel = [
@ -1240,7 +1262,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
}
if (json["hideUnderlayingFeaturesMinPercentage"] !== undefined) {
context.err(
"Layer " + json.id + " contains an old 'hideUnderlayingFeaturesMinPercentage'"
"Layer " + json.id + " contains an old 'hideUnderlayingFeaturesMinPercentage'",
)
}
@ -1259,7 +1281,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
"Layer is in an incorrect place. The path is " +
this._path +
", but expected " +
expected
expected,
)
}
}
@ -1277,13 +1299,13 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
.enter(["tagRenderings", ...emptyIndexes])
.err(
`Some tagrendering-ids are empty or have an emtpy string; this is not allowed (at ${emptyIndexes.join(
","
)}])`
",",
)}])`,
)
}
const duplicateIds = Utils.Duplicates(
(json.tagRenderings ?? [])?.map((f) => f["id"]).filter((id) => id !== "questions")
(json.tagRenderings ?? [])?.map((f) => f["id"]).filter((id) => id !== "questions"),
)
if (duplicateIds.length > 0 && !Utils.runningFromConsole) {
context
@ -1307,7 +1329,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
if (json.tagRenderings !== undefined) {
new On(
"tagRenderings",
new Each(new ValidateTagRenderings(json, this._doesImageExist))
new Each(new ValidateTagRenderings(json, this._doesImageExist)),
).convert(json, context)
}
@ -1334,7 +1356,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
context
.enters("pointRendering", i, "marker", indexM, "icon", "condition")
.err(
"Don't set a condition in a marker as this will result in an invisible but clickable element. Use extra filters in the source instead."
"Don't set a condition in a marker as this will result in an invisible but clickable element. Use extra filters in the source instead.",
)
}
}
@ -1374,7 +1396,7 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
"This preset does not match the required tags of this layer. This implies that a newly added point will not show up.\n A newly created point will have properties: " +
tags.asHumanString(false, false, {}) +
"\n The required tags are: " +
baseTags.asHumanString(false, false, {})
baseTags.asHumanString(false, false, {}),
)
}
}
@ -1391,7 +1413,7 @@ export class ValidateLayerConfig extends DesugaringStep<LayerConfigJson> {
isBuiltin: boolean,
doesImageExist: DoesImageExist,
studioValidations: boolean = false,
skipDefaultLayers: boolean = false
skipDefaultLayers: boolean = false,
) {
super("Thin wrapper around 'ValidateLayer", [], "ValidateLayerConfig")
this.validator = new ValidateLayer(
@ -1399,7 +1421,7 @@ export class ValidateLayerConfig extends DesugaringStep<LayerConfigJson> {
isBuiltin,
doesImageExist,
studioValidations,
skipDefaultLayers
skipDefaultLayers,
)
}
@ -1428,12 +1450,12 @@ class ValidatePointRendering extends DesugaringStep<PointRenderingConfigJson> {
}
if (json.marker && !Array.isArray(json.marker)) {
context.enter("marker").err(
"The marker in a pointRendering should be an array"
"The marker in a pointRendering should be an array",
)
}
if (json.location.length == 0) {
context.enter("location").err(
"A pointRendering should have at least one 'location' to defined where it should be rendered. "
"A pointRendering should have at least one 'location' to defined where it should be rendered. ",
)
}
return json
@ -1441,41 +1463,44 @@ class ValidatePointRendering extends DesugaringStep<PointRenderingConfigJson> {
}
}
export class ValidateLayer extends Conversion<
LayerConfigJson,
{ parsed: LayerConfig; raw: LayerConfigJson }
> {
private readonly _skipDefaultLayers: boolean
private readonly _prevalidation: PrevalidateLayer
constructor(
path: string,
isBuiltin: boolean,
doesImageExist: DoesImageExist,
studioValidations: boolean = false,
skipDefaultLayers: boolean = false
skipDefaultLayers: boolean = false,
) {
super("Doesn't change anything, but emits warnings and errors", [], "ValidateLayer")
this._prevalidation = new PrevalidateLayer(
path,
isBuiltin,
doesImageExist,
studioValidations
studioValidations,
)
this._skipDefaultLayers = skipDefaultLayers
}
convert(
json: LayerConfigJson,
context: ConversionContext
context: ConversionContext,
): { parsed: LayerConfig; raw: LayerConfigJson } {
context = context.inOperation(this.name)
if (typeof json === "string") {
context.err(
`Not a valid layer: the layerConfig is a string. 'npm run generate:layeroverview' might be needed`
`Not a valid layer: the layerConfig is a string. 'npm run generate:layeroverview' might be needed`,
)
return undefined
}
if (this._skipDefaultLayers && Constants.added_by_default.indexOf(<any>json.id) >= 0) {
return { parsed: undefined, raw: json }
}
@ -1502,7 +1527,7 @@ export class ValidateLayer extends Conversion<
context
.enters("calculatedTags", i)
.err(
`Invalid function definition: the custom javascript is invalid:${e}. The offending javascript code is:\n ${code}`
`Invalid function definition: the custom javascript is invalid:${e}. The offending javascript code is:\n ${code}`,
)
}
}
@ -1553,8 +1578,8 @@ export class ValidateFilter extends DesugaringStep<FilterConfigJson> {
.enters("fields", i)
.err(
`Invalid filter: ${type} is not a valid textfield type.\n\tTry one of ${Array.from(
Validators.availableTypes
).join(",")}`
Validators.availableTypes,
).join(",")}`,
)
}
}
@ -1571,13 +1596,13 @@ export class DetectDuplicateFilters extends DesugaringStep<{
super(
"Tries to detect layers where a shared filter can be used (or where similar filters occur)",
[],
"DetectDuplicateFilters"
"DetectDuplicateFilters",
)
}
convert(
json: { layers: LayerConfigJson[]; themes: LayoutConfigJson[] },
context: ConversionContext
context: ConversionContext,
): { layers: LayerConfigJson[]; themes: LayoutConfigJson[] } {
const { layers, themes } = json
const perOsmTag = new Map<
@ -1641,7 +1666,7 @@ export class DetectDuplicateFilters extends DesugaringStep<{
filter: FilterConfigJson
}[]
>,
layout?: LayoutConfigJson | undefined
layout?: LayoutConfigJson | undefined,
): void {
if (layer.filter === undefined || layer.filter === null) {
return
@ -1681,7 +1706,7 @@ export class DetectDuplicatePresets extends DesugaringStep<LayoutConfig> {
super(
"Detects mappings which have identical (english) names or identical mappings.",
["presets"],
"DetectDuplicatePresets"
"DetectDuplicatePresets",
)
}
@ -1692,13 +1717,13 @@ export class DetectDuplicatePresets extends DesugaringStep<LayoutConfig> {
if (new Set(enNames).size != enNames.length) {
const dups = Utils.Duplicates(enNames)
const layersWithDup = json.layers.filter((l) =>
l.presets.some((p) => dups.indexOf(p.title.textFor("en")) >= 0)
l.presets.some((p) => dups.indexOf(p.title.textFor("en")) >= 0),
)
const layerIds = layersWithDup.map((l) => l.id)
context.err(
`This themes has multiple presets which are named:${dups}, namely layers ${layerIds.join(
", "
)} this is confusing for contributors and is probably the result of reusing the same layer multiple times. Use \`{"override": {"=presets": []}}\` to remove some presets`
", ",
)} this is confusing for contributors and is probably the result of reusing the same layer multiple times. Use \`{"override": {"=presets": []}}\` to remove some presets`,
)
}
@ -1713,17 +1738,17 @@ export class DetectDuplicatePresets extends DesugaringStep<LayoutConfig> {
Utils.SameObject(presetATags, presetBTags) &&
Utils.sameList(
presetA.preciseInput.snapToLayers,
presetB.preciseInput.snapToLayers
presetB.preciseInput.snapToLayers,
)
) {
context.err(
`This themes has multiple presets with the same tags: ${presetATags.asHumanString(
false,
false,
{}
{},
)}, namely the preset '${presets[i].title.textFor("en")}' and '${presets[
j
].title.textFor("en")}'`
].title.textFor("en")}'`,
)
}
}
@ -1732,3 +1757,63 @@ export class DetectDuplicatePresets extends DesugaringStep<LayoutConfig> {
return json
}
}
export class ValidateThemeEnsemble extends Conversion<LayoutConfig[], Map<string, {
tags: TagsFilter,
foundInTheme: string[]
}>> {
constructor() {
super("Validates that all themes together are logical, i.e. no duplicate ids exists within (overriden) themes", [], "ValidateThemeEnsemble")
}
convert(json: LayoutConfig[], context: ConversionContext): Map<string, {
tags: TagsFilter,
foundInTheme: string[]
}> {
const idToSource = new Map<string, { tags: TagsFilter, foundInTheme: string[] }>()
for (const theme of json) {
for (const layer of theme.layers) {
if (typeof layer.source === "string") {
continue
}
if (Constants.priviliged_layers.indexOf(<any>layer.id) >= 0) {
continue
}
if (!layer.source) {
console.log(theme, layer, layer.source)
context.enters(theme.id, "layers", "source", layer.id).err("No source defined")
continue
}
if (layer.source.geojsonSource) {
continue
}
const id = layer.id
const tags = layer.source.osmTags
if (!idToSource.has(id)) {
idToSource.set(id, { tags, foundInTheme: [theme.id] })
continue
}
const oldTags = idToSource.get(id).tags
const oldTheme = idToSource.get(id).foundInTheme
if (oldTags.shadows(tags) && tags.shadows(oldTags)) {
// All is good, all is well
oldTheme.push(theme.id)
continue
}
context.err(["The layer with id '" + id + "' is found in multiple themes with different tag definitions:",
"\t In theme " + oldTheme + ":\t" + oldTags.asHumanString(false, false, {}),
"\tIn theme " + theme.id + ":\t" + tags.asHumanString(false, false, {}),
].join("\n"))
}
}
return idToSource
}
}