diff --git a/.forgejo/workflows/update_database.yml b/.forgejo/workflows/update_database.yml index 360c0d7f3..39ed86092 100644 --- a/.forgejo/workflows/update_database.yml +++ b/.forgejo/workflows/update_database.yml @@ -5,7 +5,7 @@ on: jobs: - daily_data_maintenance: + update_database: runs-on: [ osm-cache ] steps: - uses: https://source.mapcomplete.org/actions/checkout@v4 @@ -20,7 +20,7 @@ jobs: run: npm run build:dbscript # output: build_db.lua - - name: Print planet file date + - name: Create and fill database shell: bash run: | # LAIN has a weekly updated planet file in /data/planet-latest.osm.pbf diff --git a/Docs/SettingUpPSQL.md b/Docs/SettingUpPSQL.md index e30bfead2..7aaafb89f 100644 --- a/Docs/SettingUpPSQL.md +++ b/Docs/SettingUpPSQL.md @@ -1,13 +1,19 @@ # Setting up a synced OSM-server for quick layer access +## Requirements + +Debian machine (e.g. Ubuntu) + +`apt install osmium-tool docker.io osm2pgsql` + ## Setting up the SQL-server (only once): `sudo docker run --name some-postgis -e POSTGRES_PASSWORD=password -e POSTGRES_USER=user -d -p 5444:5432 -v ~/data/pgsql/:/var/lib/postgresql/data postgis/postgis` Increase the max number of connections. osm2pgsql needs connection one per table (and a few more), and since we are making one table per layer in MapComplete, this amounts to a lot. -- Open PGAdmin, open the PGSQL-tool (CLI-button at the top); alternativly: psql --host=172.17.0.1 --port=5444 - --user=user +- Open PGAdmin, open the PGSQL-tool (CLI-button at the top); alternativly: + `psql --host=172.17.0.1 --port=5444 --user=user` - Run `show config_file;` to get the config file location (in docker). This is probably `/var/lib/postgresql/data/postgresql.conf` - In a terminal, run `sudo docker exec -i bash` (run `sudo docker ps` to get the container id) diff --git a/scripts/generateLayouts.ts b/scripts/generateLayouts.ts index 522f86f79..cb3446241 100644 --- a/scripts/generateLayouts.ts +++ b/scripts/generateLayouts.ts @@ -40,7 +40,7 @@ class GenerateLayouts extends Script { private eliUrlsCached: string[] private date = new Date().toISOString() private branchName: string = undefined - + private alreadyWritten: Set = new Set() constructor() { super("Generates an '.html' and 'index_.ts' for every theme") } @@ -70,7 +70,7 @@ class GenerateLayouts extends Script { }) } - async createIcon(iconPath: string, size: number, alreadyWritten: string[]) { + async createIcon(iconPath: string, size: number) { let name = iconPath.split(".").slice(0, -1).join(".") // drop svg suffix if (name.startsWith("./")) { name = name.substring(2) @@ -78,10 +78,10 @@ class GenerateLayouts extends Script { const newname = `assets/generated/images/${name.replace(/\//g, "_")}${size}.png` const targetpath = `public/${newname}` - if (alreadyWritten.indexOf(newname) >= 0) { + if (this.alreadyWritten.has(newname)) { return newname } - alreadyWritten.push(newname) + this.alreadyWritten.add(newname) if (existsSync(targetpath)) { return newname } @@ -163,8 +163,7 @@ class GenerateLayouts extends Script { } async createManifest( - layout: ThemeConfig, - alreadyWritten: string[] + layout: ThemeConfig ): Promise<{ manifest: any whiteIcons: string[] @@ -199,8 +198,8 @@ class GenerateLayouts extends Script { const sizes = [72, 96, 120, 128, 144, 152, 180, 192, 384, 512] for (const size of sizes) { - const name = await this.createIcon(path, size, alreadyWritten) - const whiteIcon = await this.createIcon(whiteBackgroundPath, size, alreadyWritten) + const name = await this.createIcon(path, size) + const whiteIcon = await this.createIcon(whiteBackgroundPath, size) whiteIcons.push(whiteIcon) icons.push({ src: name, @@ -447,8 +446,7 @@ class GenerateLayouts extends Script { async createLandingPage( layout: ThemeConfig, layoutJson: ThemeConfigJson, - whiteIcons, - alreadyWritten + whiteIcons ) { Locale.language.setData(layout.language[0]) const targetLanguage = layout.language[0] @@ -471,7 +469,7 @@ class GenerateLayouts extends Script { if (twitterImage.endsWith(".svg")) { try { // svgs are badly supported as social image, we use a generated svg instead - twitterImage = await this.createIcon(twitterImage, 512, alreadyWritten) + twitterImage = await this.createIcon(twitterImage, 512) } catch (e) { console.error("Could not generate image:", e) } @@ -479,7 +477,7 @@ class GenerateLayouts extends Script { if (ogImage.endsWith(".svg")) { try { - ogImage = await this.createIcon(ogImage, 512, alreadyWritten) + ogImage = await this.createIcon(ogImage, 512) } catch (e) { console.error("Could not generate image:", e) } @@ -611,64 +609,62 @@ class GenerateLayouts extends Script { } } + private async createThemeFor(layoutConfigJson: ThemeConfigJson) { + const layout = new ThemeConfig(layoutConfigJson, true) + const layoutName = layout.id + + const err = (err) => { + if (err !== null) { + console.log("Could not write manifest for ", layoutName, " because ", err) + } + } + const { manifest, whiteIcons } = await this.createManifest(layout) + const manif = JSON.stringify(manifest, undefined, 2) + const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest" + writeFile("public/" + manifestLocation, manif, err) + + // Create a landing page for the given theme + const landing = await this.createLandingPage( + layout, + layoutConfigJson, + whiteIcons + ) + + writeFile(this.enc(layout.id) + ".html", landing, err) + await this.createIndexFor(layout) + } + async main(): Promise { - const alreadyWritten = [] this.createDir("./public/assets/") this.createDir("./public/assets/generated") this.createDir("./public/assets/generated/images") - const blacklist = [ + const blacklist = new Set([ "", "test", ".", "..", "manifest", "index", + "inspector", "land", "preferences", "account", "openstreetmap", "custom", "theme", - ] + ]) const args = process.argv const theme = args[2] if (theme !== undefined) { console.warn("Only generating layout " + theme) } const paths = ScriptUtils.readDirRecSync("./public/assets/generated/themes/", 1) - for (const i in paths) { - const layoutConfigJson = JSON.parse(readFileSync(paths[i], "utf8")) - if (theme !== undefined && layoutConfigJson.id !== theme) { - continue - } - const layout = new ThemeConfig(layoutConfigJson, true) - const layoutName = layout.id - if (blacklist.indexOf(layoutName.toLowerCase()) >= 0) { - console.log(`Skipping a layout with name ${layoutName}, it is on the blacklist`) - continue - } - const err = (err) => { - if (err !== null) { - console.log("Could not write manifest for ", layoutName, " because ", err) - } - } - const { manifest, whiteIcons } = await this.createManifest(layout, alreadyWritten) - const manif = JSON.stringify(manifest, undefined, 2) - const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest" - writeFile("public/" + manifestLocation, manif, err) + const configs = paths.map(path => JSON.parse(readFileSync(path, "utf8"))) + const configsFiltered = configs + .filter(config => (theme !== undefined && config.id === theme) || !blacklist.has(config.id)) - // Create a landing page for the given theme - const landing = await this.createLandingPage( - layout, - layoutConfigJson, - whiteIcons, - alreadyWritten - ) - - writeFile(this.enc(layout.id) + ".html", landing, err) - await this.createIndexFor(layout) - } + await Promise.all(configsFiltered.map(config => this.createThemeFor(config))) const { manifest } = await this.createManifest( new ThemeConfig({ @@ -681,8 +677,7 @@ class GenerateLayouts extends Script { startZoom: 0, title: { en: "MapComplete" }, description: { en: "A thematic map viewer and editor based on OpenStreetMap" }, - }), - alreadyWritten + }) ) const manif = JSON.stringify(manifest, undefined, 2)