diff --git a/Docs/SettingUpPSQL.md b/Docs/SettingUpPSQL.md index c83ddb662..dfa9989c6 100644 --- a/Docs/SettingUpPSQL.md +++ b/Docs/SettingUpPSQL.md @@ -10,26 +10,37 @@ Then activate following extensions for this database (right click > Create > Ext - Postgis activeren (rechtsklikken > Create > extension) - HStore activeren -Install osm2pgsql (hint: compile from source is painless) +Increase the max number of connections. osm2pgsql needs connection one per table (and a few more), and since we are making one table per layer in MapComplete, this amounts to a lot. -pg_tileserv kan hier gedownload worden: https://github.com/CrunchyData/pg_tileserv - -DATABASE_URL=postgresql://user:none@localhost:5444/osm-poi ./pg_tileserv +- Open PGAdmin, open the PGSQL-tool (CLI-button at the top) +- Run `max_connections = 2000;` and `show config_file;` to get the config file location (in docker). This is probably `/var/lib/postgresql/data/postgresql.conf` +- In a terminal, run `sudo docker exec -i bash` (run `sudo docker ps` to get the container id) +- `sed -i "s/max_connections = 100/max_connections = 5000/" /var/lib/postgresql/data/postgresql.conf` +- Validate with `cat /var/lib/postgresql/data/postgresql.conf | grep "max_connections"` +- `sudo docker restart ` ## Create export scripts for every layer -Use scripts/osm2pgsl +Use `vite-node ./scripts/osm2pgsql/generateBuildDbScript.ts` ## Importing data +Install osm2pgsql (hint: compile from source is painless) To seed the database: ```` -osm2pgsql -O flex -E 4326 -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi andorra-latest.osm.pbf +osm2pgsql -O flex -E 4326 -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi .osm.pbf ```` +Storing properties to table '"public"."osm2pgsql_properties" takes about 25 minutes with planet.osm + +Belgium (~555mb) takes 15m +World (80GB) should take 15m*160 = 2400m = 40hr + ## Deploying a tile server +pg_tileserv kan hier gedownload worden: https://github.com/CrunchyData/pg_tileserv + ```` export DATABASE_URL=postgresql://user:password@localhost:5444/osm-poi ./pg_tileserv diff --git a/scripts/Script.ts b/scripts/Script.ts index 48a727b22..25b0b14e7 100644 --- a/scripts/Script.ts +++ b/scripts/Script.ts @@ -13,8 +13,15 @@ export default abstract class Script { ScriptUtils.fixUtils() const args = [...process.argv] args.splice(0, 2) + const start = new Date() this.main(args) - .then((_) => console.log("All done")) + .then((_) =>{ + const end = new Date() + const millisNeeded = end.getTime() - start.getTime() + + const green = (s) => "\x1b[92m" + s + "\x1b[0m" + console.log(green("All done! (" + millisNeeded + " ms)")) + }) .catch((e) => console.log("ERROR:", e)) } diff --git a/scripts/generateLayerOverview.ts b/scripts/generateLayerOverview.ts index 39e8c6424..f60f96d10 100644 --- a/scripts/generateLayerOverview.ts +++ b/scripts/generateLayerOverview.ts @@ -9,7 +9,7 @@ import { DoesImageExist, PrevalidateTheme, ValidateLayer, - ValidateThemeAndLayers, + ValidateThemeAndLayers, ValidateThemeEnsemble, } from "../src/Models/ThemeConfig/Conversion/Validation" import { Translation } from "../src/UI/i18n/Translation" import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer" @@ -25,6 +25,8 @@ import LayerConfig from "../src/Models/ThemeConfig/LayerConfig" import PointRenderingConfig from "../src/Models/ThemeConfig/PointRenderingConfig" import { ConversionContext } from "../src/Models/ThemeConfig/Conversion/ConversionContext" import { GenerateFavouritesLayer } from "./generateFavouritesLayer" +import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig" +import { TagsFilter } from "../src/Logic/Tags/TagsFilter" // This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files. // It spits out an overview of those to be used to load them @@ -123,6 +125,7 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye } } + class LayerOverviewUtils extends Script { public static readonly layerPath = "./src/assets/generated/layers/" public static readonly themePath = "./src/assets/generated/themes/" @@ -355,7 +358,6 @@ class LayerOverviewUtils extends Script { const layerWhitelist = new Set(args.find(a => a.startsWith("--layers=")) ?.substring("--layers=".length)?.split(",") ?? []) - const start = new Date() const forceReload = args.some((a) => a == "--force") const licensePaths = new Set() @@ -382,17 +384,21 @@ class LayerOverviewUtils extends Script { sharedLayers, recompiledThemes, forceReload, - themeWhitelist + themeWhitelist, ) - if (recompiledThemes.length > 0){ + new ValidateThemeEnsemble().convertStrict( + Array.from(sharedThemes.values()).map(th => new LayoutConfig(th, true))) + + + if (recompiledThemes.length > 0) { writeFileSync( "./src/assets/generated/known_layers.json", JSON.stringify({ layers: Array.from(sharedLayers.values()).filter((l) => l.id !== "favourite"), }), ) - } + } const mcChangesPath = "./assets/themes/mapcomplete-changes/mapcomplete-changes.json" if ( @@ -437,7 +443,7 @@ class LayerOverviewUtils extends Script { ) } - if(recompiledThemes.length > 0) { + if (recompiledThemes.length > 0) { writeFileSync( "./src/assets/generated/known_themes.json", JSON.stringify({ @@ -446,17 +452,10 @@ class LayerOverviewUtils extends Script { ) } - const end = new Date() - const millisNeeded = end.getTime() - start.getTime() if (AllSharedLayers.getSharedLayersConfigs().size == 0) { console.error( - "This was a bootstrapping-run. Run generate layeroverview again!(" + - millisNeeded + - " ms)", + "This was a bootstrapping-run. Run generate layeroverview again!" ) - } else { - const green = (s) => "\x1b[92m" + s + "\x1b[0m" - console.log(green("All done! (" + millisNeeded + " ms)")) } } @@ -482,7 +481,7 @@ class LayerOverviewUtils extends Script { private buildLayerIndex( doesImageExist: DoesImageExist, forceReload: boolean, - whitelist: Set + whitelist: Set, ): Map { // First, we expand and validate all builtin layers. These are written to src/assets/generated/layers // At the same time, an index of available layers is built. @@ -500,9 +499,9 @@ class LayerOverviewUtils extends Script { const recompiledLayers: string[] = [] let warningCount = 0 for (const sharedLayerPath of ScriptUtils.getLayerPaths()) { - if(whitelist.size > 0){ - const idByPath = sharedLayerPath.split("/").at(-1).split(".")[0] - if(Constants.priviliged_layers.indexOf( idByPath) < 0 && !whitelist.has(idByPath)){ + if (whitelist.size > 0) { + const idByPath = sharedLayerPath.split("/").at(-1).split(".")[0] + if (Constants.priviliged_layers.indexOf(idByPath) < 0 && !whitelist.has(idByPath)) { continue } } @@ -672,7 +671,7 @@ class LayerOverviewUtils extends Script { sharedLayers: Map, recompiledThemes: string[], forceReload: boolean, - whitelist: Set + whitelist: Set, ): Map { console.log(" ---------- VALIDATING BUILTIN THEMES ---------") const themeFiles = ScriptUtils.getThemeFiles() @@ -710,7 +709,7 @@ class LayerOverviewUtils extends Script { const themeInfo = themeFiles[i] const themePath = themeInfo.path let themeFile = themeInfo.parsed - if(whitelist.size > 0 && !whitelist.has(themeFile.id)){ + if (whitelist.size > 0 && !whitelist.has(themeFile.id)) { continue } @@ -795,21 +794,21 @@ class LayerOverviewUtils extends Script { } } - if(whitelist.size == 0){ - this.writeSmallOverview( - Array.from(fixed.values()).map((t) => { - return { - ...t, - hideFromOverview: t.hideFromOverview ?? false, - shortDescription: - t.shortDescription ?? - new Translation(t.description) - .FirstSentence() - .OnEveryLanguage((s) => parse_html(s).textContent).translations, - mustHaveLanguage: t.mustHaveLanguage?.length > 0, - } - }), - ) + if (whitelist.size == 0) { + this.writeSmallOverview( + Array.from(fixed.values()).map((t) => { + return { + ...t, + hideFromOverview: t.hideFromOverview ?? false, + shortDescription: + t.shortDescription ?? + new Translation(t.description) + .FirstSentence() + .OnEveryLanguage((s) => parse_html(s).textContent).translations, + mustHaveLanguage: t.mustHaveLanguage?.length > 0, + } + }), + ) } console.log( diff --git a/scripts/osm2pgsql/generateBuildDbScript.ts b/scripts/osm2pgsql/generateBuildDbScript.ts index a2c9363f2..0fce50b06 100644 --- a/scripts/osm2pgsql/generateBuildDbScript.ts +++ b/scripts/osm2pgsql/generateBuildDbScript.ts @@ -1,13 +1,13 @@ -import LayerConfig from "../../src/Models/ThemeConfig/LayerConfig" import { TagsFilter } from "../../src/Logic/Tags/TagsFilter" import { Tag } from "../../src/Logic/Tags/Tag" import { And } from "../../src/Logic/Tags/And" import Script from "../Script" -import { AllSharedLayers } from "../../src/Customizations/AllSharedLayers" import fs from "fs" import { Or } from "../../src/Logic/Tags/Or" import { RegexTag } from "../../src/Logic/Tags/RegexTag" import { Utils } from "../../src/Utils" +import { ValidateThemeEnsemble } from "../../src/Models/ThemeConfig/Conversion/Validation" +import { AllKnownLayouts } from "../../src/Customizations/AllKnownLayouts" class LuaSnippets { /** @@ -35,28 +35,31 @@ class LuaSnippets { } class GenerateLayerLua { - private readonly _layer: LayerConfig + private readonly _id: string + private readonly _tags: TagsFilter + private readonly _foundInThemes: string[] - constructor(layer: LayerConfig) { - this._layer = layer + constructor(id: string, tags: TagsFilter, foundInThemes: string[] = []) { + this._tags = tags + this._id = id + this._foundInThemes = foundInThemes } public functionName() { - const l = this._layer - if (!l.source?.osmTags) { + if (!this._tags) { return undefined } - return `process_poi_${l.id}` + return `process_poi_${this._id}` } public generateFunction(): string { - const l = this._layer - if (!l.source?.osmTags) { + if (!this._tags) { return undefined } return [ - `local pois_${l.id} = osm2pgsql.define_table({`, - ` name = '${l.id}',`, + `local pois_${this._id} = osm2pgsql.define_table({`, + this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "", + ` name = '${this._id}',`, " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", " columns = {", " { column = 'tags', type = 'jsonb' },", @@ -66,7 +69,7 @@ class GenerateLayerLua { "", "", `function ${this.functionName()}(object, geom)`, - " local matches_filter = " + this.toLuaFilter(l.source.osmTags), + " local matches_filter = " + this.toLuaFilter(this._tags), " if( not matches_filter) then", " return", " end", @@ -75,7 +78,7 @@ class GenerateLayerLua { " tags = object.tags", " }", " ", - ` pois_${l.id}:insert(a)`, + ` pois_${this._id}:insert(a)`, "end", "", ].join("\n") @@ -86,6 +89,8 @@ class GenerateLayerLua { return `object.tags["${tag.key}"] ~= "${tag.value}"` } + const v = ( tag.value).source.replace(/\\\//g, "/") + if ("" + tag.value === "/.+/is" && !tag.invert) { return `object.tags["${tag.key}"] ~= nil` } @@ -104,10 +109,10 @@ class GenerateLayerLua { } if (tag.invert) { - return `object.tags["${tag.key}"] == nil or not string.find(object.tags["${tag.key}"], "${tag.value}")` + return `object.tags["${tag.key}"] == nil or not string.find(object.tags["${tag.key}"], "${v}")` } - return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${tag.value}"))` + return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${v}"))` } private toLuaFilter(tag: TagsFilter, useParens: boolean = false): string { @@ -141,15 +146,21 @@ class GenerateLayerLua { } } -class GenerateLayerFile extends Script { +class GenerateBuildDbScript extends Script { constructor() { super("Generates a .lua-file to use with osm2pgsql") } async main(args: string[]) { - const layers = Array.from(AllSharedLayers.sharedLayers.values()) + const allNeededLayers = new ValidateThemeEnsemble().convertStrict( + AllKnownLayouts.allKnownLayouts.values(), + ) - const generators = layers.filter(l => l.source.geojsonSource === undefined).map(l => new GenerateLayerLua(l)) + const generators: GenerateLayerLua[] = [] + + allNeededLayers.forEach(({ tags, foundInTheme }, layerId) => { + generators.push(new GenerateLayerLua(layerId, tags, foundInTheme)) + }) const script = [ ...generators.map(g => g.generateFunction()), @@ -159,7 +170,8 @@ class GenerateLayerFile extends Script { const path = "build_db.lua" fs.writeFileSync(path, script, "utf-8") console.log("Written", path) + console.log(allNeededLayers.size+" layers will be created. Make sure to set 'max_connections' to at least "+(10 + allNeededLayers.size) ) } } -new GenerateLayerFile().run() +new GenerateBuildDbScript().run() diff --git a/src/Models/ThemeConfig/Conversion/Validation.ts b/src/Models/ThemeConfig/Conversion/Validation.ts index ed4c79098..6380df2e5 100644 --- a/src/Models/ThemeConfig/Conversion/Validation.ts +++ b/src/Models/ThemeConfig/Conversion/Validation.ts @@ -21,9 +21,7 @@ import PresetConfig from "../PresetConfig" import { TagsFilter } from "../../../Logic/Tags/TagsFilter" import { Translatable } from "../Json/Translatable" import { ConversionContext } from "./ConversionContext" -import * as eli from "../../../assets/editor-layer-index.json" import { AvailableRasterLayers } from "../../RasterLayers" -import Back from "../../../assets/svg/Back.svelte" import PointRenderingConfigJson from "../Json/PointRenderingConfigJson" class ValidateLanguageCompleteness extends DesugaringStep { @@ -33,7 +31,7 @@ class ValidateLanguageCompleteness extends DesugaringStep { super( "Checks that the given object is fully translated in the specified languages", [], - "ValidateLanguageCompleteness" + "ValidateLanguageCompleteness", ) this._languages = languages ?? ["en"] } @@ -47,18 +45,18 @@ class ValidateLanguageCompleteness extends DesugaringStep { .filter( (t) => t.tr.translations[neededLanguage] === undefined && - t.tr.translations["*"] === undefined + t.tr.translations["*"] === undefined, ) .forEach((missing) => { context .enter(missing.context.split(".")) .err( `The theme ${obj.id} should be translation-complete for ` + - neededLanguage + - ", but it lacks a translation for " + - missing.context + - ".\n\tThe known translation is " + - missing.tr.textFor("en") + neededLanguage + + ", but it lacks a translation for " + + missing.context + + ".\n\tThe known translation is " + + missing.tr.textFor("en"), ) }) } @@ -75,7 +73,7 @@ export class DoesImageExist extends DesugaringStep { constructor( knownImagePaths: Set, checkExistsSync: (path: string) => boolean = undefined, - ignore?: Set + ignore?: Set, ) { super("Checks if an image exists", [], "DoesImageExist") this._ignore = ignore @@ -111,15 +109,15 @@ export class DoesImageExist extends DesugaringStep { if (!this._knownImagePaths.has(image)) { if (this.doesPathExist === undefined) { context.err( - `Image with path ${image} not found or not attributed; it is used in ${context}` + `Image with path ${image} not found or not attributed; it is used in ${context}`, ) } else if (!this.doesPathExist(image)) { context.err( - `Image with path ${image} does not exist.\n Check for typo's and missing directories in the path.` + `Image with path ${image} does not exist.\n Check for typo's and missing directories in the path.`, ) } else { context.err( - `Image with path ${image} is not attributed (but it exists); execute 'npm run query:licenses' to add the license information and/or run 'npm run generate:licenses' to compile all the license info` + `Image with path ${image} is not attributed (but it exists); execute 'npm run query:licenses' to add the license information and/or run 'npm run generate:licenses' to compile all the license info`, ) } } @@ -143,7 +141,7 @@ export class ValidateTheme extends DesugaringStep { doesImageExist: DoesImageExist, path: string, isBuiltin: boolean, - sharedTagRenderings?: Set + sharedTagRenderings?: Set, ) { super("Doesn't change anything, but emits warnings and errors", [], "ValidateTheme") this._validateImage = doesImageExist @@ -162,15 +160,15 @@ export class ValidateTheme extends DesugaringStep { if (json["units"] !== undefined) { context.err( "The theme " + - json.id + - " has units defined - these should be defined on the layer instead. (Hint: use overrideAll: { '+units': ... }) " + json.id + + " has units defined - these should be defined on the layer instead. (Hint: use overrideAll: { '+units': ... }) ", ) } if (json["roamingRenderings"] !== undefined) { context.err( "Theme " + - json.id + - " contains an old 'roamingRenderings'. Use an 'overrideAll' instead" + json.id + + " contains an old 'roamingRenderings'. Use an 'overrideAll' instead", ) } } @@ -178,7 +176,7 @@ export class ValidateTheme extends DesugaringStep { if (!json.title) { context.enter("title").err(`The theme ${json.id} does not have a title defined.`) } - if(!json.icon){ + if (!json.icon) { context.enter("icon").err("A theme should have an icon") } if (this._isBuiltin && this._extractImages !== undefined) { @@ -188,10 +186,10 @@ export class ValidateTheme extends DesugaringStep { for (const remoteImage of remoteImages) { context.err( "Found a remote image: " + - remoteImage.path + - " in theme " + - json.id + - ", please download it." + remoteImage.path + + " in theme " + + json.id + + ", please download it.", ) } for (const image of images) { @@ -207,17 +205,17 @@ export class ValidateTheme extends DesugaringStep { const filename = this._path.substring( this._path.lastIndexOf("/") + 1, - this._path.length - 5 + this._path.length - 5, ) if (theme.id !== filename) { context.err( "Theme ids should be the same as the name.json, but we got id: " + - theme.id + - " and filename " + - filename + - " (" + - this._path + - ")" + theme.id + + " and filename " + + filename + + " (" + + this._path + + ")", ) } this._validateImage.convert(theme.icon, context.enter("icon")) @@ -225,13 +223,13 @@ export class ValidateTheme extends DesugaringStep { const dups = Utils.Duplicates(json.layers.map((layer) => layer["id"])) if (dups.length > 0) { context.err( - `The theme ${json.id} defines multiple layers with id ${dups.join(", ")}` + `The theme ${json.id} defines multiple layers with id ${dups.join(", ")}`, ) } if (json["mustHaveLanguage"] !== undefined) { new ValidateLanguageCompleteness(...json["mustHaveLanguage"]).convert( theme, - context + context, ) } if (!json.hideFromOverview && theme.id !== "personal" && this._isBuiltin) { @@ -239,7 +237,7 @@ export class ValidateTheme extends DesugaringStep { const targetLanguage = theme.title.SupportedLanguages()[0] if (targetLanguage !== "en") { context.err( - `TargetLanguage is not 'en' for public theme ${theme.id}, it is ${targetLanguage}. Move 'en' up in the title of the theme and set it as the first key` + `TargetLanguage is not 'en' for public theme ${theme.id}, it is ${targetLanguage}. Move 'en' up in the title of the theme and set it as the first key`, ) } @@ -282,6 +280,13 @@ export class ValidateTheme extends DesugaringStep { } } + for (let i = 0; i < theme.layers.length; i++) { + const layer = theme.layers[i] + if (!layer.id.match("[a-z][a-z0-9_]*")) { + context.enters("layers", i, "id").err("Invalid ID:" + layer.id + "should match [a-z][a-z0-9_]*") + } + } + return json } } @@ -291,7 +296,7 @@ export class ValidateThemeAndLayers extends Fuse { doesImageExist: DoesImageExist, path: string, isBuiltin: boolean, - sharedTagRenderings?: Set + sharedTagRenderings?: Set, ) { super( "Validates a theme and the contained layers", @@ -301,10 +306,10 @@ export class ValidateThemeAndLayers extends Fuse { new Each( new Bypass( (layer) => Constants.added_by_default.indexOf(layer.id) < 0, - new ValidateLayerConfig(undefined, isBuiltin, doesImageExist, false, true) - ) - ) - ) + new ValidateLayerConfig(undefined, isBuiltin, doesImageExist, false, true), + ), + ), + ), ) } } @@ -314,7 +319,7 @@ class OverrideShadowingCheck extends DesugaringStep { super( "Checks that an 'overrideAll' does not override a single override", [], - "OverrideShadowingCheck" + "OverrideShadowingCheck", ) } @@ -363,6 +368,22 @@ class MiscThemeChecks extends DesugaringStep { if (json.socialImage === "") { context.warn("Social image for theme " + json.id + " is the emtpy string") } + { + for (let i = 0; i < json.layers.length; i++) { + const l = json.layers[i] + if (l["override"]?.["source"] === undefined) { + continue + } + if (l["override"]?.["source"]?.["geoJson"]) { + continue // We don't care about external data as we won't cache it anyway + } + if (l["override"]["id"] !== undefined) { + continue + } + context.enters("layers", i).err("A layer which changes the source-tags must also change the ID") + } + } + return json } } @@ -372,7 +393,7 @@ export class PrevalidateTheme extends Fuse { super( "Various consistency checks on the raw JSON", new MiscThemeChecks(), - new OverrideShadowingCheck() + new OverrideShadowingCheck(), ) } } @@ -382,7 +403,7 @@ export class DetectConflictingAddExtraTags extends DesugaringStep ["_abc"] */ private static extractCalculatedTagNames( - layerConfig?: LayerConfigJson | { calculatedTags: string[] } + layerConfig?: LayerConfigJson | { calculatedTags: string[] }, ) { return ( layerConfig?.calculatedTags?.map((ct) => { @@ -617,16 +638,16 @@ export class DetectShadowedMappings extends DesugaringStep\` instead. The images found are ${images.join( - ", " - )}. (This check can be turned of by adding "#": "${ignoreToken}" in the mapping, but this is discouraged` + ", ", + )}. (This check can be turned of by adding "#": "${ignoreToken}" in the mapping, but this is discouraged`, ) } else { ctx.info( `Ignored image ${images.join( - ", " - )} in 'then'-clause of a mapping as this check has been disabled` + ", ", + )} in 'then'-clause of a mapping as this check has been disabled`, ) for (const image of images) { @@ -721,7 +742,7 @@ class ValidatePossibleLinks extends DesugaringStep does have `rel='noopener'` set", [], - "ValidatePossibleLinks" + "ValidatePossibleLinks", ) } @@ -751,21 +772,21 @@ class ValidatePossibleLinks extends DesugaringStep, - context: ConversionContext + context: ConversionContext, ): string | Record { if (typeof json === "string") { if (this.isTabnabbingProne(json)) { context.err( "The string " + - json + - " has a link targeting `_blank`, but it doesn't have `rel='noopener'` set. This gives rise to reverse tabnapping" + json + + " has a link targeting `_blank`, but it doesn't have `rel='noopener'` set. This gives rise to reverse tabnapping", ) } } else { for (const k in json) { if (this.isTabnabbingProne(json[k])) { context.err( - `The translation for ${k} '${json[k]}' has a link targeting \`_blank\`, but it doesn't have \`rel='noopener'\` set. This gives rise to reverse tabnapping` + `The translation for ${k} '${json[k]}' has a link targeting \`_blank\`, but it doesn't have \`rel='noopener'\` set. This gives rise to reverse tabnapping`, ) } } @@ -783,7 +804,7 @@ class CheckTranslation extends DesugaringStep { super( "Checks that a translation is valid and internally consistent", ["*"], - "CheckTranslation" + "CheckTranslation", ) this._allowUndefined = allowUndefined } @@ -829,17 +850,17 @@ class MiscTagRenderingChecks extends DesugaringStep { convert( json: TagRenderingConfigJson | QuestionableTagRenderingConfigJson, - context: ConversionContext + context: ConversionContext, ): TagRenderingConfigJson { if (json["special"] !== undefined) { context.err( - 'Detected `special` on the top level. Did you mean `{"render":{ "special": ... }}`' + "Detected `special` on the top level. Did you mean `{\"render\":{ \"special\": ... }}`", ) } if (Object.keys(json).length === 1 && typeof json["render"] === "string") { context.warn( - `use the content directly instead of {render: ${JSON.stringify(json["render"])}}` + `use the content directly instead of {render: ${JSON.stringify(json["render"])}}`, ) } @@ -851,7 +872,7 @@ class MiscTagRenderingChecks extends DesugaringStep { const mapping = json.mappings[i] CheckTranslation.noUndefined.convert( mapping.then, - context.enters("mappings", i, "then") + context.enters("mappings", i, "then"), ) if (!mapping.if) { context.enters("mappings", i).err("No `if` is defined") @@ -862,18 +883,18 @@ class MiscTagRenderingChecks extends DesugaringStep { context .enters("mappings", i, "then") .warn( - "A mapping should not start with 'yes' or 'no'. If the attribute is known, it will only show 'yes' or 'no' without the question, resulting in a weird phrasing in the information box" + "A mapping should not start with 'yes' or 'no'. If the attribute is known, it will only show 'yes' or 'no' without the question, resulting in a weird phrasing in the information box", ) } } } if (json["group"]) { - context.err('Groups are deprecated, use `"label": ["' + json["group"] + '"]` instead') + context.err("Groups are deprecated, use `\"label\": [\"" + json["group"] + "\"]` instead") } if (json["question"] && json.freeform?.key === undefined && json.mappings === undefined) { context.err( - "A question is defined, but no mappings nor freeform (key) are. Add at least one of them" + "A question is defined, but no mappings nor freeform (key) are. Add at least one of them", ) } if (json["question"] && !json.freeform && (json.mappings?.length ?? 0) == 1) { @@ -883,7 +904,7 @@ class MiscTagRenderingChecks extends DesugaringStep { context .enter("questionHint") .err( - "A questionHint is defined, but no question is given. As such, the questionHint will never be shown" + "A questionHint is defined, but no question is given. As such, the questionHint will never be shown", ) } @@ -893,10 +914,10 @@ class MiscTagRenderingChecks extends DesugaringStep { .enter("render") .err( "This tagRendering allows to set a value to key " + - json.freeform.key + - ", but does not define a `render`. Please, add a value here which contains `{" + - json.freeform.key + - "}`" + json.freeform.key + + ", but does not define a `render`. Please, add a value here which contains `{" + + json.freeform.key + + "}`", ) } else { const render = new Translation(json.render) @@ -927,7 +948,7 @@ class MiscTagRenderingChecks extends DesugaringStep { const keyFirstArg = ["canonical", "fediverse_link", "translated"] if ( keyFirstArg.some( - (funcName) => txt.indexOf(`{${funcName}(${json.freeform.key}`) >= 0 + (funcName) => txt.indexOf(`{${funcName}(${json.freeform.key}`) >= 0, ) ) { continue @@ -950,7 +971,7 @@ class MiscTagRenderingChecks extends DesugaringStep { context .enter("render") .err( - `The rendering for language ${ln} does not contain \`{${json.freeform.key}}\`. This is a bug, as this rendering should show exactly this freeform key!` + `The rendering for language ${ln} does not contain \`{${json.freeform.key}}\`. This is a bug, as this rendering should show exactly this freeform key!`, ) } } @@ -958,8 +979,8 @@ class MiscTagRenderingChecks extends DesugaringStep { if (json.render && json["question"] && json.freeform === undefined) { context.err( `Detected a tagrendering which takes input without freeform key in ${context}; the question is ${new Translation( - json["question"] - ).textFor("en")}` + json["question"], + ).textFor("en")}`, ) } @@ -970,9 +991,9 @@ class MiscTagRenderingChecks extends DesugaringStep { .enters("freeform", "type") .err( "Unknown type: " + - freeformType + - "; try one of " + - Validators.availableTypes.join(", ") + freeformType + + "; try one of " + + Validators.availableTypes.join(", "), ) } } @@ -1004,7 +1025,7 @@ export class ValidateTagRenderings extends Fuse { new On("question", new ValidatePossibleLinks()), new On("questionHint", new ValidatePossibleLinks()), new On("mappings", new Each(new On("then", new ValidatePossibleLinks()))), - new MiscTagRenderingChecks() + new MiscTagRenderingChecks(), ) } } @@ -1034,8 +1055,9 @@ export class PrevalidateLayer extends DesugaringStep { if (json.id?.toLowerCase() !== json.id) { context.enter("id").err(`The id of a layer should be lowercase: ${json.id}`) } - if (json.id?.match(/[a-z0-9-_]/) == null) { - context.enter("id").err(`The id of a layer should match [a-z0-9-_]*: ${json.id}`) + const layerRegex = /[a-zA-Z][a-zA-Z_0-9]+/ + if (json.id.match(layerRegex) === null) { + context.enter("id").err("Invalid ID. A layer ID should match " + layerRegex.source) } } @@ -1043,7 +1065,7 @@ export class PrevalidateLayer extends DesugaringStep { context .enter("source") .err( - "No source section is defined; please define one as data is not loaded otherwise" + "No source section is defined; please define one as data is not loaded otherwise", ) } else { if (json.source === "special" || json.source === "special:library") { @@ -1051,7 +1073,7 @@ export class PrevalidateLayer extends DesugaringStep { context .enters("source", "osmTags") .err( - "No osmTags defined in the source section - these should always be present, even for geojson layer" + "No osmTags defined in the source section - these should always be present, even for geojson layer", ) } else { const osmTags = TagUtils.Tag(json.source["osmTags"], context + "source.osmTags") @@ -1060,7 +1082,7 @@ export class PrevalidateLayer extends DesugaringStep { .enters("source", "osmTags") .err( "The source states tags which give a very wide selection: it only uses negative expressions, which will result in too much and unexpected data. Add at least one required tag. The tags are:\n\t" + - osmTags.asHumanString(false, false, {}) + osmTags.asHumanString(false, false, {}), ) } } @@ -1086,10 +1108,10 @@ export class PrevalidateLayer extends DesugaringStep { .enter("syncSelection") .err( "Invalid sync-selection: must be one of " + - LayerConfig.syncSelectionAllowed.map((v) => `'${v}'`).join(", ") + - " but got '" + - json.syncSelection + - "'" + LayerConfig.syncSelectionAllowed.map((v) => `'${v}'`).join(", ") + + " but got '" + + json.syncSelection + + "'", ) } if (json["pointRenderings"]?.length > 0) { @@ -1107,7 +1129,7 @@ export class PrevalidateLayer extends DesugaringStep { context.enter("pointRendering").err("There are no pointRenderings at all...") } - json.pointRendering?.forEach((pr,i) => this._validatePointRendering.convert(pr, context.enters("pointeRendering", i))) + json.pointRendering?.forEach((pr, i) => this._validatePointRendering.convert(pr, context.enters("pointeRendering", i))) if (json["mapRendering"]) { context.enter("mapRendering").err("This layer has a legacy 'mapRendering'") @@ -1123,8 +1145,8 @@ export class PrevalidateLayer extends DesugaringStep { if (!Constants.priviliged_layers.find((x) => x == json.id)) { context.err( "Layer " + - json.id + - " uses 'special' as source.osmTags. However, this layer is not a priviliged layer" + json.id + + " uses 'special' as source.osmTags. However, this layer is not a priviliged layer", ) } } @@ -1139,19 +1161,19 @@ export class PrevalidateLayer extends DesugaringStep { context .enter("title") .err( - "This layer does not have a title defined but it does have tagRenderings. Not having a title will disable the popups, resulting in an unclickable element. Please add a title. If not having a popup is intended and the tagrenderings need to be kept (e.g. in a library layer), set `title: null` to disable this error." + "This layer does not have a title defined but it does have tagRenderings. Not having a title will disable the popups, resulting in an unclickable element. Please add a title. If not having a popup is intended and the tagrenderings need to be kept (e.g. in a library layer), set `title: null` to disable this error.", ) } if (json.title === null) { context.info( - "Title is `null`. This results in an element that cannot be clicked - even though tagRenderings is set." + "Title is `null`. This results in an element that cannot be clicked - even though tagRenderings is set.", ) } { // Check for multiple, identical builtin questions - usability for studio users const duplicates = Utils.Duplicates( - json.tagRenderings.filter((tr) => typeof tr === "string") + json.tagRenderings.filter((tr) => typeof tr === "string"), ) for (let i = 0; i < json.tagRenderings.length; i++) { const tagRendering = json.tagRenderings[i] @@ -1181,7 +1203,7 @@ export class PrevalidateLayer extends DesugaringStep { { // duplicate ids in tagrenderings check const duplicates = Utils.NoNull( - Utils.Duplicates(Utils.NoNull((json.tagRenderings ?? []).map((tr) => tr["id"]))) + Utils.Duplicates(Utils.NoNull((json.tagRenderings ?? []).map((tr) => tr["id"]))), ) if (duplicates.length > 0) { // It is tempting to add an index to this warning; however, due to labels the indices here might be different from the index in the tagRendering list @@ -1219,8 +1241,8 @@ export class PrevalidateLayer extends DesugaringStep { if (json["overpassTags"] !== undefined) { context.err( "Layer " + - json.id + - 'still uses the old \'overpassTags\'-format. Please use "source": {"osmTags": }\' instead of "overpassTags": (note: this isn\'t your fault, the custom theme generator still spits out the old format)' + json.id + + "still uses the old 'overpassTags'-format. Please use \"source\": {\"osmTags\": }' instead of \"overpassTags\": (note: this isn't your fault, the custom theme generator still spits out the old format)", ) } const forbiddenTopLevel = [ @@ -1240,7 +1262,7 @@ export class PrevalidateLayer extends DesugaringStep { } if (json["hideUnderlayingFeaturesMinPercentage"] !== undefined) { context.err( - "Layer " + json.id + " contains an old 'hideUnderlayingFeaturesMinPercentage'" + "Layer " + json.id + " contains an old 'hideUnderlayingFeaturesMinPercentage'", ) } @@ -1257,9 +1279,9 @@ export class PrevalidateLayer extends DesugaringStep { if (this._path != undefined && this._path.indexOf(expected) < 0) { context.err( "Layer is in an incorrect place. The path is " + - this._path + - ", but expected " + - expected + this._path + + ", but expected " + + expected, ) } } @@ -1277,13 +1299,13 @@ export class PrevalidateLayer extends DesugaringStep { .enter(["tagRenderings", ...emptyIndexes]) .err( `Some tagrendering-ids are empty or have an emtpy string; this is not allowed (at ${emptyIndexes.join( - "," - )}])` + ",", + )}])`, ) } const duplicateIds = Utils.Duplicates( - (json.tagRenderings ?? [])?.map((f) => f["id"]).filter((id) => id !== "questions") + (json.tagRenderings ?? [])?.map((f) => f["id"]).filter((id) => id !== "questions"), ) if (duplicateIds.length > 0 && !Utils.runningFromConsole) { context @@ -1307,7 +1329,7 @@ export class PrevalidateLayer extends DesugaringStep { if (json.tagRenderings !== undefined) { new On( "tagRenderings", - new Each(new ValidateTagRenderings(json, this._doesImageExist)) + new Each(new ValidateTagRenderings(json, this._doesImageExist)), ).convert(json, context) } @@ -1334,7 +1356,7 @@ export class PrevalidateLayer extends DesugaringStep { context .enters("pointRendering", i, "marker", indexM, "icon", "condition") .err( - "Don't set a condition in a marker as this will result in an invisible but clickable element. Use extra filters in the source instead." + "Don't set a condition in a marker as this will result in an invisible but clickable element. Use extra filters in the source instead.", ) } } @@ -1372,9 +1394,9 @@ export class PrevalidateLayer extends DesugaringStep { .enters("presets", i, "tags") .err( "This preset does not match the required tags of this layer. This implies that a newly added point will not show up.\n A newly created point will have properties: " + - tags.asHumanString(false, false, {}) + - "\n The required tags are: " + - baseTags.asHumanString(false, false, {}) + tags.asHumanString(false, false, {}) + + "\n The required tags are: " + + baseTags.asHumanString(false, false, {}), ) } } @@ -1391,7 +1413,7 @@ export class ValidateLayerConfig extends DesugaringStep { isBuiltin: boolean, doesImageExist: DoesImageExist, studioValidations: boolean = false, - skipDefaultLayers: boolean = false + skipDefaultLayers: boolean = false, ) { super("Thin wrapper around 'ValidateLayer", [], "ValidateLayerConfig") this.validator = new ValidateLayer( @@ -1399,7 +1421,7 @@ export class ValidateLayerConfig extends DesugaringStep { isBuiltin, doesImageExist, studioValidations, - skipDefaultLayers + skipDefaultLayers, ) } @@ -1428,12 +1450,12 @@ class ValidatePointRendering extends DesugaringStep { } if (json.marker && !Array.isArray(json.marker)) { context.enter("marker").err( - "The marker in a pointRendering should be an array" + "The marker in a pointRendering should be an array", ) } if (json.location.length == 0) { - context.enter("location").err ( - "A pointRendering should have at least one 'location' to defined where it should be rendered. " + context.enter("location").err( + "A pointRendering should have at least one 'location' to defined where it should be rendered. ", ) } return json @@ -1441,41 +1463,44 @@ class ValidatePointRendering extends DesugaringStep { } } + export class ValidateLayer extends Conversion< LayerConfigJson, { parsed: LayerConfig; raw: LayerConfigJson } > { private readonly _skipDefaultLayers: boolean private readonly _prevalidation: PrevalidateLayer + constructor( path: string, isBuiltin: boolean, doesImageExist: DoesImageExist, studioValidations: boolean = false, - skipDefaultLayers: boolean = false + skipDefaultLayers: boolean = false, ) { super("Doesn't change anything, but emits warnings and errors", [], "ValidateLayer") this._prevalidation = new PrevalidateLayer( path, isBuiltin, doesImageExist, - studioValidations + studioValidations, ) this._skipDefaultLayers = skipDefaultLayers } convert( json: LayerConfigJson, - context: ConversionContext + context: ConversionContext, ): { parsed: LayerConfig; raw: LayerConfigJson } { context = context.inOperation(this.name) if (typeof json === "string") { context.err( - `Not a valid layer: the layerConfig is a string. 'npm run generate:layeroverview' might be needed` + `Not a valid layer: the layerConfig is a string. 'npm run generate:layeroverview' might be needed`, ) return undefined } + if (this._skipDefaultLayers && Constants.added_by_default.indexOf(json.id) >= 0) { return { parsed: undefined, raw: json } } @@ -1502,7 +1527,7 @@ export class ValidateLayer extends Conversion< context .enters("calculatedTags", i) .err( - `Invalid function definition: the custom javascript is invalid:${e}. The offending javascript code is:\n ${code}` + `Invalid function definition: the custom javascript is invalid:${e}. The offending javascript code is:\n ${code}`, ) } } @@ -1553,8 +1578,8 @@ export class ValidateFilter extends DesugaringStep { .enters("fields", i) .err( `Invalid filter: ${type} is not a valid textfield type.\n\tTry one of ${Array.from( - Validators.availableTypes - ).join(",")}` + Validators.availableTypes, + ).join(",")}`, ) } } @@ -1571,13 +1596,13 @@ export class DetectDuplicateFilters extends DesugaringStep<{ super( "Tries to detect layers where a shared filter can be used (or where similar filters occur)", [], - "DetectDuplicateFilters" + "DetectDuplicateFilters", ) } convert( json: { layers: LayerConfigJson[]; themes: LayoutConfigJson[] }, - context: ConversionContext + context: ConversionContext, ): { layers: LayerConfigJson[]; themes: LayoutConfigJson[] } { const { layers, themes } = json const perOsmTag = new Map< @@ -1641,7 +1666,7 @@ export class DetectDuplicateFilters extends DesugaringStep<{ filter: FilterConfigJson }[] >, - layout?: LayoutConfigJson | undefined + layout?: LayoutConfigJson | undefined, ): void { if (layer.filter === undefined || layer.filter === null) { return @@ -1681,7 +1706,7 @@ export class DetectDuplicatePresets extends DesugaringStep { super( "Detects mappings which have identical (english) names or identical mappings.", ["presets"], - "DetectDuplicatePresets" + "DetectDuplicatePresets", ) } @@ -1692,13 +1717,13 @@ export class DetectDuplicatePresets extends DesugaringStep { if (new Set(enNames).size != enNames.length) { const dups = Utils.Duplicates(enNames) const layersWithDup = json.layers.filter((l) => - l.presets.some((p) => dups.indexOf(p.title.textFor("en")) >= 0) + l.presets.some((p) => dups.indexOf(p.title.textFor("en")) >= 0), ) const layerIds = layersWithDup.map((l) => l.id) context.err( `This themes has multiple presets which are named:${dups}, namely layers ${layerIds.join( - ", " - )} this is confusing for contributors and is probably the result of reusing the same layer multiple times. Use \`{"override": {"=presets": []}}\` to remove some presets` + ", ", + )} this is confusing for contributors and is probably the result of reusing the same layer multiple times. Use \`{"override": {"=presets": []}}\` to remove some presets`, ) } @@ -1713,17 +1738,17 @@ export class DetectDuplicatePresets extends DesugaringStep { Utils.SameObject(presetATags, presetBTags) && Utils.sameList( presetA.preciseInput.snapToLayers, - presetB.preciseInput.snapToLayers + presetB.preciseInput.snapToLayers, ) ) { context.err( `This themes has multiple presets with the same tags: ${presetATags.asHumanString( false, false, - {} + {}, )}, namely the preset '${presets[i].title.textFor("en")}' and '${presets[ j - ].title.textFor("en")}'` + ].title.textFor("en")}'`, ) } } @@ -1732,3 +1757,63 @@ export class DetectDuplicatePresets extends DesugaringStep { return json } } + +export class ValidateThemeEnsemble extends Conversion> { + constructor() { + super("Validates that all themes together are logical, i.e. no duplicate ids exists within (overriden) themes", [], "ValidateThemeEnsemble") + } + + convert(json: LayoutConfig[], context: ConversionContext): Map { + + + const idToSource = new Map() + + for (const theme of json) { + for (const layer of theme.layers) { + if (typeof layer.source === "string") { + continue + } + if (Constants.priviliged_layers.indexOf(layer.id) >= 0) { + continue + } + if (!layer.source) { + console.log(theme, layer, layer.source) + context.enters(theme.id, "layers", "source", layer.id).err("No source defined") + continue + } + if (layer.source.geojsonSource) { + continue + } + const id = layer.id + const tags = layer.source.osmTags + if (!idToSource.has(id)) { + idToSource.set(id, { tags, foundInTheme: [theme.id] }) + continue + } + + const oldTags = idToSource.get(id).tags + const oldTheme = idToSource.get(id).foundInTheme + if (oldTags.shadows(tags) && tags.shadows(oldTags)) { + // All is good, all is well + oldTheme.push(theme.id) + continue + } + context.err(["The layer with id '" + id + "' is found in multiple themes with different tag definitions:", + "\t In theme " + oldTheme + ":\t" + oldTags.asHumanString(false, false, {}), + "\tIn theme " + theme.id + ":\t" + tags.asHumanString(false, false, {}), + + + ].join("\n")) + } + } + + + return idToSource + } +}