Scripts: improve documentation and try to speed up things

.forgejo/workflows/update_database.yml

@@ -5,7 +5,7 @@ on:
 
 
 jobs:
-  daily_data_maintenance:
+  update_database:
     runs-on: [ osm-cache ]
     steps:
       - uses: https://source.mapcomplete.org/actions/checkout@v4
@@ -20,7 +20,7 @@ jobs:
         run: npm run build:dbscript # output: build_db.lua
 
 
-      - name: Print planet file date
+      - name: Create and fill database
         shell: bash
         run: |
           # LAIN has a weekly updated planet file in /data/planet-latest.osm.pbf

Docs/SettingUpPSQL.md

@@ -1,13 +1,19 @@
 # Setting up a synced OSM-server for quick layer access
 
+## Requirements
+
+Debian machine (e.g. Ubuntu)
+
+`apt install osmium-tool docker.io osm2pgsql`
+
 ## Setting up the SQL-server (only once):
 
 `sudo docker run --name some-postgis -e POSTGRES_PASSWORD=password -e POSTGRES_USER=user -d -p 5444:5432 -v ~/data/pgsql/:/var/lib/postgresql/data postgis/postgis`
 
 Increase the max number of connections. osm2pgsql needs connection one per table (and a few more), and since we are making one table per layer in MapComplete, this amounts to a lot.
 
-- Open PGAdmin, open the PGSQL-tool (CLI-button at the top); alternativly: psql --host=172.17.0.1 --port=5444
+- Open PGAdmin, open the PGSQL-tool (CLI-button at the top); alternativly:
-  --user=user
+  `psql --host=172.17.0.1 --port=5444 --user=user`
 - Run `show config_file;` to get the config file location (in docker). This is probably
   `/var/lib/postgresql/data/postgresql.conf`
 - In a terminal, run `sudo docker exec -i <docker-container-id> bash` (run `sudo docker ps` to get the container id)

scripts/generateLayouts.ts

@@ -40,7 +40,7 @@ class GenerateLayouts extends Script {
     private eliUrlsCached: string[]
     private date = new Date().toISOString()
     private branchName: string = undefined
-
+    private alreadyWritten: Set<string> = new Set<string>()
     constructor() {
         super("Generates an '<theme>.html' and 'index_<theme>.ts' for every theme")
     }
@@ -70,7 +70,7 @@ class GenerateLayouts extends Script {
         })
     }
 
-    async createIcon(iconPath: string, size: number, alreadyWritten: string[]) {
+    async createIcon(iconPath: string, size: number) {
         let name = iconPath.split(".").slice(0, -1).join(".") // drop svg suffix
         if (name.startsWith("./")) {
             name = name.substring(2)
@@ -78,10 +78,10 @@ class GenerateLayouts extends Script {
 
         const newname = `assets/generated/images/${name.replace(/\//g, "_")}${size}.png`
         const targetpath = `public/${newname}`
-        if (alreadyWritten.indexOf(newname) >= 0) {
+        if (this.alreadyWritten.has(newname)) {
             return newname
         }
-        alreadyWritten.push(newname)
+        this.alreadyWritten.add(newname)
         if (existsSync(targetpath)) {
             return newname
         }
@@ -163,8 +163,7 @@ class GenerateLayouts extends Script {
     }
 
     async createManifest(
-        layout: ThemeConfig,
+        layout: ThemeConfig
-        alreadyWritten: string[]
     ): Promise<{
         manifest: any
         whiteIcons: string[]
@@ -199,8 +198,8 @@ class GenerateLayouts extends Script {
 
             const sizes = [72, 96, 120, 128, 144, 152, 180, 192, 384, 512]
             for (const size of sizes) {
-                const name = await this.createIcon(path, size, alreadyWritten)
+                const name = await this.createIcon(path, size)
-                const whiteIcon = await this.createIcon(whiteBackgroundPath, size, alreadyWritten)
+                const whiteIcon = await this.createIcon(whiteBackgroundPath, size)
                 whiteIcons.push(whiteIcon)
                 icons.push({
                     src: name,
@@ -447,8 +446,7 @@ class GenerateLayouts extends Script {
     async createLandingPage(
         layout: ThemeConfig,
         layoutJson: ThemeConfigJson,
-        whiteIcons,
+        whiteIcons
-        alreadyWritten
     ) {
         Locale.language.setData(layout.language[0])
         const targetLanguage = layout.language[0]
@@ -471,7 +469,7 @@ class GenerateLayouts extends Script {
         if (twitterImage.endsWith(".svg")) {
             try {
                 // svgs are badly supported as social image, we use a generated svg instead
-                twitterImage = await this.createIcon(twitterImage, 512, alreadyWritten)
+                twitterImage = await this.createIcon(twitterImage, 512)
             } catch (e) {
                 console.error("Could not generate image:", e)
             }
@@ -479,7 +477,7 @@ class GenerateLayouts extends Script {
 
         if (ogImage.endsWith(".svg")) {
             try {
-                ogImage = await this.createIcon(ogImage, 512, alreadyWritten)
+                ogImage = await this.createIcon(ogImage, 512)
             } catch (e) {
                 console.error("Could not generate image:", e)
             }
@@ -611,64 +609,62 @@ class GenerateLayouts extends Script {
         }
     }
 
+    private async createThemeFor(layoutConfigJson: ThemeConfigJson) {
+        const layout = new ThemeConfig(layoutConfigJson, true)
+        const layoutName = layout.id
+
+        const err = (err) => {
+            if (err !== null) {
+                console.log("Could not write manifest for ", layoutName, " because ", err)
+            }
+        }
+        const { manifest, whiteIcons } = await this.createManifest(layout)
+        const manif = JSON.stringify(manifest, undefined, 2)
+        const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest"
+        writeFile("public/" + manifestLocation, manif, err)
+
+        // Create a landing page for the given theme
+        const landing = await this.createLandingPage(
+            layout,
+            layoutConfigJson,
+            whiteIcons
+        )
+
+        writeFile(this.enc(layout.id) + ".html", landing, err)
+        await this.createIndexFor(layout)
+    }
+
     async main(): Promise<void> {
-        const alreadyWritten = []
         this.createDir("./public/assets/")
         this.createDir("./public/assets/generated")
         this.createDir("./public/assets/generated/images")
 
-        const blacklist = [
+        const blacklist = new Set([
             "",
             "test",
             ".",
             "..",
             "manifest",
             "index",
+            "inspector",
             "land",
             "preferences",
             "account",
             "openstreetmap",
             "custom",
             "theme",
-        ]
+        ])
         const args = process.argv
         const theme = args[2]
         if (theme !== undefined) {
             console.warn("Only generating layout " + theme)
         }
         const paths = ScriptUtils.readDirRecSync("./public/assets/generated/themes/", 1)
-        for (const i in paths) {
+        const configs = paths.map(path => <ThemeConfigJson>JSON.parse(readFileSync(path, "utf8")))
-            const layoutConfigJson = <ThemeConfigJson>JSON.parse(readFileSync(paths[i], "utf8"))
+        const configsFiltered = configs
-            if (theme !== undefined && layoutConfigJson.id !== theme) {
+            .filter(config => (theme !== undefined && config.id === theme) || !blacklist.has(config.id))
-                continue
-            }
-            const layout = new ThemeConfig(layoutConfigJson, true)
-            const layoutName = layout.id
-            if (blacklist.indexOf(layoutName.toLowerCase()) >= 0) {
-                console.log(`Skipping a layout with name ${layoutName}, it is on the blacklist`)
-                continue
-            }
-            const err = (err) => {
-                if (err !== null) {
-                    console.log("Could not write manifest for ", layoutName, " because ", err)
-                }
-            }
-            const { manifest, whiteIcons } = await this.createManifest(layout, alreadyWritten)
-            const manif = JSON.stringify(manifest, undefined, 2)
-            const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest"
-            writeFile("public/" + manifestLocation, manif, err)
 
-            // Create a landing page for the given theme
+        await Promise.all(configsFiltered.map(config => this.createThemeFor(config)))
-            const landing = await this.createLandingPage(
-                layout,
-                layoutConfigJson,
-                whiteIcons,
-                alreadyWritten
-            )
-
-            writeFile(this.enc(layout.id) + ".html", landing, err)
-            await this.createIndexFor(layout)
-        }
 
         const { manifest } = await this.createManifest(
             new ThemeConfig({
@@ -681,8 +677,7 @@ class GenerateLayouts extends Script {
                 startZoom: 0,
                 title: { en: "MapComplete" },
                 description: { en: "A thematic map viewer and editor based on OpenStreetMap" },
-            }),
+            })
-            alreadyWritten
         )
 
         const manif = JSON.stringify(manifest, undefined, 2)