From fb088059a5bddde88d0e7b6212720b4a0419e7fe Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Wed, 17 Jan 2024 02:30:13 +0100 Subject: [PATCH] Layerserver: improve docs, add stub of script that generates configuration file --- Docs/SettingUpPSQL.md | 43 +++++++- scripts/osm2pgsql/generateLayerFile.ts | 136 +++++++++++++++++++++++++ src/UI/Test.svelte | 20 ++++ 3 files changed, 194 insertions(+), 5 deletions(-) create mode 100644 scripts/osm2pgsql/generateLayerFile.ts diff --git a/Docs/SettingUpPSQL.md b/Docs/SettingUpPSQL.md index 67c2ddeb2f..9c9e5e6f39 100644 --- a/Docs/SettingUpPSQL.md +++ b/Docs/SettingUpPSQL.md @@ -1,11 +1,44 @@ -sudo docker run --name some-postgis -e POSTGRES_PASSWORD=none -e POSTGRES_USER=user -d -p 5444:5432 -v /home/pietervdvn/data/pgsql/:/var/lib/postgresql/data postgis/postgis +# Setting up a synced OSM-server for quick layer access --> Via PGAdmin een database maken en: -1) Postgis activeren (rechtsklikken > Create > extension) -2) HStore activeren +## Setting up the SQL-server: -Installeer osm2pgsql (hint: compile from source is painless) +`sudo docker run --name some-postgis -e POSTGRES_PASSWORD=password -e POSTGRES_USER=user -d -p 5444:5432 -v /home/pietervdvn/data/pgsql/:/var/lib/postgresql/data postgis/postgis` + +Then, connect to this databank with PGAdmin, create a database within it. +Then activate following extensions for this database (right click > Create > Extension): + +- Postgis activeren (rechtsklikken > Create > extension) +- HStore activeren + +Install osm2pgsql (hint: compile from source is painless) pg_tileserv kan hier gedownload worden: https://github.com/CrunchyData/pg_tileserv DATABASE_URL=postgresql://user:none@localhost:5444/osm-poi ./pg_tileserv + +## Create export scripts for every layer + +Use scripts/osm2pgsl + +## Importing data + +To seed the database: + +```` +osm2pgsql -O flex -S drinking_water.lua -s --flat-nodes=import-help-file -d postgresql://user:none@localhost:5444/osm-poi andorra-latest.osm.pbf +```` + +## Deploying a tile server + +```` +export DATABASE_URL=postgresql://user:none@localhost:5444/osm-poi +./pg_tileserv +```` + +Tiles are available at: +```` +map.addSource("drinking_water", { +"type": "vector", +"tiles": ["http://127.0.0.2:7800/public.drinking_water/{z}/{x}/{y}.pbf"] // http://127.0.0.2:7800/public.drinking_water.json", +}) +```` diff --git a/scripts/osm2pgsql/generateLayerFile.ts b/scripts/osm2pgsql/generateLayerFile.ts new file mode 100644 index 0000000000..670918a897 --- /dev/null +++ b/scripts/osm2pgsql/generateLayerFile.ts @@ -0,0 +1,136 @@ +import LayerConfig from "../../src/Models/ThemeConfig/LayerConfig" +import { TagsFilter } from "../../src/Logic/Tags/TagsFilter" +import { Tag } from "../../src/Logic/Tags/Tag" +import { And } from "../../src/Logic/Tags/And" +import Script from "../Script" +import { AllSharedLayers } from "../../src/Customizations/AllSharedLayers" +import fs from "fs" +import { Or } from "../../src/Logic/Tags/Or" +import { RegexTag } from "../../src/Logic/Tags/RegexTag" + +class LuaSnippets{ + /** + * The main piece of code that calls `process_poi` + */ + static tail = [ + "function osm2pgsql.process_node(object)", + " process_poi(object, object:as_point())", + "end", + "", + "function osm2pgsql.process_way(object)", + " if object.is_closed then", + " process_poi(object, object:as_polygon():centroid())", + " end", + "end", + ""].join("\n") + + public static combine(calls: string[]): string{ + return [ + `function process_poi(object, geom)`, + ...calls.map(c => " "+c+"(object, geom)"), + `end`, + ].join("\n") + } +} +class GenerateLayerLua { + private readonly _layer: LayerConfig + + constructor(layer: LayerConfig) { + this._layer = layer + } + public functionName(){ + const l = this._layer + return `process_poi_${l.id}` + } + + public generateFunction(): string { + const l = this._layer + return [ + `local pois_${l.id} = osm2pgsql.define_table({`, + ` name = '${l.id}',`, + " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", + " columns = {", + " { column = 'tags', type = 'jsonb' },", + " { column = 'geom', type = 'point', not_null = true },", + " }" + + "})", + "", + "", + `function ${this.functionName()}(object, geom)`, + " local matches_filter = " + this.toLuaFilter(l.source.osmTags), + " if( not matches_filter) then", + " return", + " end", + " local a = {", + " geom = geom,", + " tags = object.tags", + " }", + " ", + ` pois_${l.id}:insert(a)`, + "end", + "" + ].join("\n") + } + + + private toLuaFilter(tag: TagsFilter, useParens: boolean = false): string { + if (tag instanceof Tag) { + return `object.tags["${tag.key}"] == "${tag.value}"` + } + if (tag instanceof And) { + const expr = tag.and.map(t => this.toLuaFilter(t, true)).join(" and ") + if (useParens) { + return "(" + expr + ")" + } + return expr + } + if (tag instanceof Or) { + const expr = tag.or.map(t => this.toLuaFilter(t, true)).join(" or ") + if (useParens) { + return "(" + expr + ")" + } + return expr + } + if (tag instanceof RegexTag) { + if(typeof tag.value === "string" && tag.invert){ + return `object.tags["${tag.key}"] ~= "${tag.value}"` + } + + let expr = `not string.find(object.tags["${tag.key}"], "${tag.value}")` + if (!tag.invert) { + expr = "not " + expr + } + if (useParens) { + expr = "(" + expr + ")" + } + return expr + } + let msg = "Could not handle" + tag.asHumanString(false, false, {}) + console.error(msg) + throw msg + } +} + +class GenerateLayerFile extends Script { + constructor() { + super("Generates a .lua-file to use with osm2pgsql") + } + + async main(args: string[]) { + let dw = AllSharedLayers.sharedLayers.get("drinking_water") + let t = AllSharedLayers.sharedLayers.get("toilet") + + const generators = [dw, t].map(l => new GenerateLayerLua(l)) + + const script = [ + ...generators.map(g => g.generateFunction()), + LuaSnippets.combine(generators.map(g => g.functionName())), + LuaSnippets.tail + ].join("\n\n\n") + const path = "build_db.lua" + fs.writeFileSync(path,script, "utf-8") + console.log("Written", path) + } +} + +new GenerateLayerFile().run() diff --git a/src/UI/Test.svelte b/src/UI/Test.svelte index 91b4aa3068..5e13292b14 100644 --- a/src/UI/Test.svelte +++ b/src/UI/Test.svelte @@ -36,6 +36,26 @@ }, ) + map.addSource("toilet", { + "type": "vector", + "tiles": ["http://127.0.0.2:7800/public.toilet/{z}/{x}/{y}.pbf"] // http://127.0.0.2:7800/public.drinking_water.json", + }) + + map.addLayer( + { + "id": "toilet_layer", + "type": "circle", + "source": "toilet", + "source-layer": "public.toilet", + "paint": { + "circle-radius": 5, + "circle-color": "#0000ff", + "circle-stroke-width": 2, + "circle-stroke-color": "#000000", + }, + }, + ) + map.on('click', 'drinking_water_layer', (e) => { // Copy coordinates array. console.log(e)