forked from MapComplete/MapComplete
LayerServer: improve script, add unique and valid id check to layers
This commit is contained in:
parent
1d6c9ec1ef
commit
ee38cdb9d7
5 changed files with 313 additions and 199 deletions
|
|
@ -13,8 +13,15 @@ export default abstract class Script {
|
|||
ScriptUtils.fixUtils()
|
||||
const args = [...process.argv]
|
||||
args.splice(0, 2)
|
||||
const start = new Date()
|
||||
this.main(args)
|
||||
.then((_) => console.log("All done"))
|
||||
.then((_) =>{
|
||||
const end = new Date()
|
||||
const millisNeeded = end.getTime() - start.getTime()
|
||||
|
||||
const green = (s) => "\x1b[92m" + s + "\x1b[0m"
|
||||
console.log(green("All done! (" + millisNeeded + " ms)"))
|
||||
})
|
||||
.catch((e) => console.log("ERROR:", e))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import {
|
|||
DoesImageExist,
|
||||
PrevalidateTheme,
|
||||
ValidateLayer,
|
||||
ValidateThemeAndLayers,
|
||||
ValidateThemeAndLayers, ValidateThemeEnsemble,
|
||||
} from "../src/Models/ThemeConfig/Conversion/Validation"
|
||||
import { Translation } from "../src/UI/i18n/Translation"
|
||||
import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
|
||||
|
|
@ -25,6 +25,8 @@ import LayerConfig from "../src/Models/ThemeConfig/LayerConfig"
|
|||
import PointRenderingConfig from "../src/Models/ThemeConfig/PointRenderingConfig"
|
||||
import { ConversionContext } from "../src/Models/ThemeConfig/Conversion/ConversionContext"
|
||||
import { GenerateFavouritesLayer } from "./generateFavouritesLayer"
|
||||
import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig"
|
||||
import { TagsFilter } from "../src/Logic/Tags/TagsFilter"
|
||||
|
||||
// This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files.
|
||||
// It spits out an overview of those to be used to load them
|
||||
|
|
@ -123,6 +125,7 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
class LayerOverviewUtils extends Script {
|
||||
public static readonly layerPath = "./src/assets/generated/layers/"
|
||||
public static readonly themePath = "./src/assets/generated/themes/"
|
||||
|
|
@ -355,7 +358,6 @@ class LayerOverviewUtils extends Script {
|
|||
const layerWhitelist = new Set(args.find(a => a.startsWith("--layers="))
|
||||
?.substring("--layers=".length)?.split(",") ?? [])
|
||||
|
||||
const start = new Date()
|
||||
const forceReload = args.some((a) => a == "--force")
|
||||
|
||||
const licensePaths = new Set<string>()
|
||||
|
|
@ -382,17 +384,21 @@ class LayerOverviewUtils extends Script {
|
|||
sharedLayers,
|
||||
recompiledThemes,
|
||||
forceReload,
|
||||
themeWhitelist
|
||||
themeWhitelist,
|
||||
)
|
||||
|
||||
if (recompiledThemes.length > 0){
|
||||
new ValidateThemeEnsemble().convertStrict(
|
||||
Array.from(sharedThemes.values()).map(th => new LayoutConfig(th, true)))
|
||||
|
||||
|
||||
if (recompiledThemes.length > 0) {
|
||||
writeFileSync(
|
||||
"./src/assets/generated/known_layers.json",
|
||||
JSON.stringify({
|
||||
layers: Array.from(sharedLayers.values()).filter((l) => l.id !== "favourite"),
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const mcChangesPath = "./assets/themes/mapcomplete-changes/mapcomplete-changes.json"
|
||||
if (
|
||||
|
|
@ -437,7 +443,7 @@ class LayerOverviewUtils extends Script {
|
|||
)
|
||||
}
|
||||
|
||||
if(recompiledThemes.length > 0) {
|
||||
if (recompiledThemes.length > 0) {
|
||||
writeFileSync(
|
||||
"./src/assets/generated/known_themes.json",
|
||||
JSON.stringify({
|
||||
|
|
@ -446,17 +452,10 @@ class LayerOverviewUtils extends Script {
|
|||
)
|
||||
}
|
||||
|
||||
const end = new Date()
|
||||
const millisNeeded = end.getTime() - start.getTime()
|
||||
if (AllSharedLayers.getSharedLayersConfigs().size == 0) {
|
||||
console.error(
|
||||
"This was a bootstrapping-run. Run generate layeroverview again!(" +
|
||||
millisNeeded +
|
||||
" ms)",
|
||||
"This was a bootstrapping-run. Run generate layeroverview again!"
|
||||
)
|
||||
} else {
|
||||
const green = (s) => "\x1b[92m" + s + "\x1b[0m"
|
||||
console.log(green("All done! (" + millisNeeded + " ms)"))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -482,7 +481,7 @@ class LayerOverviewUtils extends Script {
|
|||
private buildLayerIndex(
|
||||
doesImageExist: DoesImageExist,
|
||||
forceReload: boolean,
|
||||
whitelist: Set<string>
|
||||
whitelist: Set<string>,
|
||||
): Map<string, LayerConfigJson> {
|
||||
// First, we expand and validate all builtin layers. These are written to src/assets/generated/layers
|
||||
// At the same time, an index of available layers is built.
|
||||
|
|
@ -500,9 +499,9 @@ class LayerOverviewUtils extends Script {
|
|||
const recompiledLayers: string[] = []
|
||||
let warningCount = 0
|
||||
for (const sharedLayerPath of ScriptUtils.getLayerPaths()) {
|
||||
if(whitelist.size > 0){
|
||||
const idByPath = sharedLayerPath.split("/").at(-1).split(".")[0]
|
||||
if(Constants.priviliged_layers.indexOf(<any> idByPath) < 0 && !whitelist.has(idByPath)){
|
||||
if (whitelist.size > 0) {
|
||||
const idByPath = sharedLayerPath.split("/").at(-1).split(".")[0]
|
||||
if (Constants.priviliged_layers.indexOf(<any>idByPath) < 0 && !whitelist.has(idByPath)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
|
@ -672,7 +671,7 @@ class LayerOverviewUtils extends Script {
|
|||
sharedLayers: Map<string, LayerConfigJson>,
|
||||
recompiledThemes: string[],
|
||||
forceReload: boolean,
|
||||
whitelist: Set<string>
|
||||
whitelist: Set<string>,
|
||||
): Map<string, LayoutConfigJson> {
|
||||
console.log(" ---------- VALIDATING BUILTIN THEMES ---------")
|
||||
const themeFiles = ScriptUtils.getThemeFiles()
|
||||
|
|
@ -710,7 +709,7 @@ class LayerOverviewUtils extends Script {
|
|||
const themeInfo = themeFiles[i]
|
||||
const themePath = themeInfo.path
|
||||
let themeFile = themeInfo.parsed
|
||||
if(whitelist.size > 0 && !whitelist.has(themeFile.id)){
|
||||
if (whitelist.size > 0 && !whitelist.has(themeFile.id)) {
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
@ -795,21 +794,21 @@ class LayerOverviewUtils extends Script {
|
|||
}
|
||||
}
|
||||
|
||||
if(whitelist.size == 0){
|
||||
this.writeSmallOverview(
|
||||
Array.from(fixed.values()).map((t) => {
|
||||
return {
|
||||
...t,
|
||||
hideFromOverview: t.hideFromOverview ?? false,
|
||||
shortDescription:
|
||||
t.shortDescription ??
|
||||
new Translation(t.description)
|
||||
.FirstSentence()
|
||||
.OnEveryLanguage((s) => parse_html(s).textContent).translations,
|
||||
mustHaveLanguage: t.mustHaveLanguage?.length > 0,
|
||||
}
|
||||
}),
|
||||
)
|
||||
if (whitelist.size == 0) {
|
||||
this.writeSmallOverview(
|
||||
Array.from(fixed.values()).map((t) => {
|
||||
return {
|
||||
...t,
|
||||
hideFromOverview: t.hideFromOverview ?? false,
|
||||
shortDescription:
|
||||
t.shortDescription ??
|
||||
new Translation(t.description)
|
||||
.FirstSentence()
|
||||
.OnEveryLanguage((s) => parse_html(s).textContent).translations,
|
||||
mustHaveLanguage: t.mustHaveLanguage?.length > 0,
|
||||
}
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
console.log(
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
import LayerConfig from "../../src/Models/ThemeConfig/LayerConfig"
|
||||
import { TagsFilter } from "../../src/Logic/Tags/TagsFilter"
|
||||
import { Tag } from "../../src/Logic/Tags/Tag"
|
||||
import { And } from "../../src/Logic/Tags/And"
|
||||
import Script from "../Script"
|
||||
import { AllSharedLayers } from "../../src/Customizations/AllSharedLayers"
|
||||
import fs from "fs"
|
||||
import { Or } from "../../src/Logic/Tags/Or"
|
||||
import { RegexTag } from "../../src/Logic/Tags/RegexTag"
|
||||
import { Utils } from "../../src/Utils"
|
||||
import { ValidateThemeEnsemble } from "../../src/Models/ThemeConfig/Conversion/Validation"
|
||||
import { AllKnownLayouts } from "../../src/Customizations/AllKnownLayouts"
|
||||
|
||||
class LuaSnippets {
|
||||
/**
|
||||
|
|
@ -35,28 +35,31 @@ class LuaSnippets {
|
|||
}
|
||||
|
||||
class GenerateLayerLua {
|
||||
private readonly _layer: LayerConfig
|
||||
private readonly _id: string
|
||||
private readonly _tags: TagsFilter
|
||||
private readonly _foundInThemes: string[]
|
||||
|
||||
constructor(layer: LayerConfig) {
|
||||
this._layer = layer
|
||||
constructor(id: string, tags: TagsFilter, foundInThemes: string[] = []) {
|
||||
this._tags = tags
|
||||
this._id = id
|
||||
this._foundInThemes = foundInThemes
|
||||
}
|
||||
|
||||
public functionName() {
|
||||
const l = this._layer
|
||||
if (!l.source?.osmTags) {
|
||||
if (!this._tags) {
|
||||
return undefined
|
||||
}
|
||||
return `process_poi_${l.id}`
|
||||
return `process_poi_${this._id}`
|
||||
}
|
||||
|
||||
public generateFunction(): string {
|
||||
const l = this._layer
|
||||
if (!l.source?.osmTags) {
|
||||
if (!this._tags) {
|
||||
return undefined
|
||||
}
|
||||
return [
|
||||
`local pois_${l.id} = osm2pgsql.define_table({`,
|
||||
` name = '${l.id}',`,
|
||||
`local pois_${this._id} = osm2pgsql.define_table({`,
|
||||
this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "",
|
||||
` name = '${this._id}',`,
|
||||
" ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },",
|
||||
" columns = {",
|
||||
" { column = 'tags', type = 'jsonb' },",
|
||||
|
|
@ -66,7 +69,7 @@ class GenerateLayerLua {
|
|||
"",
|
||||
"",
|
||||
`function ${this.functionName()}(object, geom)`,
|
||||
" local matches_filter = " + this.toLuaFilter(l.source.osmTags),
|
||||
" local matches_filter = " + this.toLuaFilter(this._tags),
|
||||
" if( not matches_filter) then",
|
||||
" return",
|
||||
" end",
|
||||
|
|
@ -75,7 +78,7 @@ class GenerateLayerLua {
|
|||
" tags = object.tags",
|
||||
" }",
|
||||
" ",
|
||||
` pois_${l.id}:insert(a)`,
|
||||
` pois_${this._id}:insert(a)`,
|
||||
"end",
|
||||
"",
|
||||
].join("\n")
|
||||
|
|
@ -86,6 +89,8 @@ class GenerateLayerLua {
|
|||
return `object.tags["${tag.key}"] ~= "${tag.value}"`
|
||||
}
|
||||
|
||||
const v = (<RegExp> tag.value).source.replace(/\\\//g, "/")
|
||||
|
||||
if ("" + tag.value === "/.+/is" && !tag.invert) {
|
||||
return `object.tags["${tag.key}"] ~= nil`
|
||||
}
|
||||
|
|
@ -104,10 +109,10 @@ class GenerateLayerLua {
|
|||
}
|
||||
|
||||
if (tag.invert) {
|
||||
return `object.tags["${tag.key}"] == nil or not string.find(object.tags["${tag.key}"], "${tag.value}")`
|
||||
return `object.tags["${tag.key}"] == nil or not string.find(object.tags["${tag.key}"], "${v}")`
|
||||
}
|
||||
|
||||
return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${tag.value}"))`
|
||||
return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${v}"))`
|
||||
}
|
||||
|
||||
private toLuaFilter(tag: TagsFilter, useParens: boolean = false): string {
|
||||
|
|
@ -141,15 +146,21 @@ class GenerateLayerLua {
|
|||
}
|
||||
}
|
||||
|
||||
class GenerateLayerFile extends Script {
|
||||
class GenerateBuildDbScript extends Script {
|
||||
constructor() {
|
||||
super("Generates a .lua-file to use with osm2pgsql")
|
||||
}
|
||||
|
||||
async main(args: string[]) {
|
||||
const layers = Array.from(AllSharedLayers.sharedLayers.values())
|
||||
const allNeededLayers = new ValidateThemeEnsemble().convertStrict(
|
||||
AllKnownLayouts.allKnownLayouts.values(),
|
||||
)
|
||||
|
||||
const generators = layers.filter(l => l.source.geojsonSource === undefined).map(l => new GenerateLayerLua(l))
|
||||
const generators: GenerateLayerLua[] = []
|
||||
|
||||
allNeededLayers.forEach(({ tags, foundInTheme }, layerId) => {
|
||||
generators.push(new GenerateLayerLua(layerId, tags, foundInTheme))
|
||||
})
|
||||
|
||||
const script = [
|
||||
...generators.map(g => g.generateFunction()),
|
||||
|
|
@ -159,7 +170,8 @@ class GenerateLayerFile extends Script {
|
|||
const path = "build_db.lua"
|
||||
fs.writeFileSync(path, script, "utf-8")
|
||||
console.log("Written", path)
|
||||
console.log(allNeededLayers.size+" layers will be created. Make sure to set 'max_connections' to at least "+(10 + allNeededLayers.size) )
|
||||
}
|
||||
}
|
||||
|
||||
new GenerateLayerFile().run()
|
||||
new GenerateBuildDbScript().run()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue