Scripts: add script to automatically update the database

.forgejo/workflows/update_database.yml

@@ -0,0 +1,42 @@
+on:
+  schedule:
+    - cron: "0 0 1 */2 *"
+
+
+jobs:
+  daily_data_maintenance:
+    runs-on: [ lain ]
+    steps:
+      - uses: https://source.mapcomplete.org/actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: https://source.mapcomplete.org/actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+          cache-dependency-path: package-lock.json
+
+      - name: Init all dependencies and layers
+        shell: bash
+        run: npm run init
+
+      - name: Create export script
+        shell: bash
+        run: npm run build:dbscript # output: build_db.lua
+
+
+      - name: Print planet file date
+        shell: bash
+        run: |
+          # LAIN has a weekly updated planet file in /data/planet-latest.osm.pbf
+          # See https://source.mapcomplete.org/MapComplete/planet-file-updater
+          TIMESTAMP=$(osmium fileinfo /data/planet-latest.osm.pbf -g header.option.timestamp)
+          DATE=$(echo $TIMESTAMP | sed "s/T.*//")
+          echo $DATE
+          # Create a new database in postgres
+          vite-node scripts/osm2pgsql/createNewDatabase.ts -- $DATE
+          echo "Seeding database '$DATE'"
+          osm2pgsql -O flex -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi.${DATE} /data/planet-latest.osm.pbf
+          npm run delete:database:old
+
+

Docs/SettingUpPSQL.md

@@ -13,16 +13,9 @@ Increase the max number of connections. osm2pgsql needs connection one per table
 - Validate with `cat /var/lib/postgresql/data/postgresql.conf | grep "max_connections"`
 - `sudo docker restart <ID>`
 
+# Seeding the data
 
-> The following steps are also automated in `update.sh`
-
-## Create a database in the SQL-server
-
-Run `vite-node scripts/osm2pgsql/createNewDatabase.ts -- YYYY-MM-DD` to create a new, appropriate, database
-
-## Create export scripts for every layer
-
-Use `npm run build:dbscript`
+> The following steps are also automated in .forgejo/workflows/update_database.yml
 
 ## Importing data
 

scripts/osm2pgsql/update.sh

@@ -1,38 +0,0 @@
-#! /bin/bash
-
-# Full database update. Downloads the latest from planet.osm.org, build update script, setup and seed it
-# Meant to be run with 'nohup'
-export NVM_DIR="$HOME/.nvm"
-[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
-nvm use 20.12.2
-npm run init # contains a 'npm run generate, which builds the layers'
-npm run generate:buildDbScript
-mv build_db.lua ~/data/
-
-
-cd ~/data || exit
-rm planet-latest.osm.pbf
-
-wget https://planet.osm.org/pbf/planet-latest.osm.pbf
-
-TIMESTAMP=$(osmium fileinfo ~/data/planet-latest.osm.pbf -g header.option.timestamp)
-DATE=$(echo $TIMESTAMP | sed "s/T.*//")
-echo $DATE
-cd ~/git/MapComplete/ || exit
-npm run create:database -- -- ${DATE/T.*//}
-cd ~/data || exit
-
-rm seeddb.log
-osm2pgsql -O flex -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi.${DATE} planet-latest.osm.pbf >> seeddb.log
-
-# To see the progress
-# tail -f seeddb.log
-
-cd ~/git/MapComplete || exit
-npm run delete:database:old
-nohup npm run server:summary >> summary_server.log &
-cd - || exit
-
-# Restart tileserver
-export DATABASE_URL=postgresql://user:password@localhost:5444/osm-poi.${DATE}
-nohup ./pg_tileserv >> pg_tileserv.log &