Merge branch 'develop' into feature/360-view

.forgejo/workflows/deploy_hosted.yml

@@ -1,4 +1,5 @@
 on:
+  workflow_dispatch:
   push:
     branches-ignore:
       - build/*
@@ -52,27 +53,35 @@ jobs:
 
       - name: Zipping dist file
         run: |
-          mv dist /tmp/${{ github.ref_name }}
+          BRANCH=$(echo ${{ github.ref_name }} | sed 's/\//-/g')
+          mv dist /tmp/${{ BRANCH }}
           cd /tmp
-          zip ${{ github.ref_name }}.zip -r ${{ github.ref_name }}/*
+          zip ${{ BRANCH }}.zip -r ${{ BRANCH }}/*
           cd -
 
       - name: uploading file
-        run: scp /tmp/${{ github.ref_name }}.zip hetzner:/root/staging/
+        run: |
+          BRANCH=$(echo ${{ github.ref_name }} | sed 's/\//-/g')
+          scp /tmp/${BRANCH}.zip hetzner:/root/staging/
 
       - name: cleanup files
-        run: rm /tmp/${{ github.ref_name }}.zip && rm -rf /tmp/${{ github.ref_name }}/
+        run: |
+          BRANCH=$(echo ${{ github.ref_name }} | sed 's/\//-/g')
+          rm /tmp/${BRANCH}.zip && r${GITHUB_REF_NAME}m -rf /tmp/${BRANCH}/
 
       - name: unzipping remote file
-        run: ssh hetzner "cd /root/staging && rm -rf ${{ github.ref_name }} && unzip ${{ github.ref_name }}.zip && rm -rf /root/public/${{ github.ref_name }} && mv /root/staging/${{ github.ref_name }}/ /root/public/ && rm ${{ github.ref_name }}.zip"
+        run: |
+          BRANCH=$(echo ${{ github.ref_name }} | sed 's/\//-/g')
+          ssh hetzner "cd /root/staging && rm -rf ${BRANCH} && unzip ${BRANCH}.zip && rm -rf /root/public/${BRANCH} && mv /root/staging/${BRANCH}/ /root/public/ && rm ${BRANCH}.zip"
 
       - name: Comment on the PR
         run: |
+          BRANCH=$(echo ${{ github.ref_name }} | sed 's/\//-/g')
           PR_NUMBER=$(echo ${{ github.event.pull_request.number || github.event.issue.number }})
           if [[ -n "$PR_NUMBER" ]]
           then
           echo "Found a pull request or issue number, will post to $PR_NUMBER ..."
-          MSG=$(echo "Congratulations! This PR been successfully built and has been deployed. It is (temporarily) available on https://builds.mapcomplete.org/${PR_NUMBER}")
+          MSG=$(echo "Congratulations! This PR been successfully built and has been deployed. It is (temporarily) available on https://builds.mapcomplete.org/${BRANCH}")
           BODY="{\"body\": \"$MSG\"}"
           # Token must have following permissions: issue > read and write
           curl -X POST "https://source.mapcomplete.org/api/v1/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" \

.forgejo/workflows/update_database.yml

@@ -0,0 +1,43 @@
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 1 */2 *"
+
+
+jobs:
+  daily_data_maintenance:
+    runs-on: [ lain ]
+    steps:
+      - uses: https://source.mapcomplete.org/actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: https://source.mapcomplete.org/actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+          cache-dependency-path: package-lock.json
+
+      - name: Init all dependencies and layers
+        shell: bash
+        run: npm ci && npm run init
+
+      - name: Create export script
+        shell: bash
+        run: npm run build:dbscript # output: build_db.lua
+
+
+      - name: Print planet file date
+        shell: bash
+        run: |
+          # LAIN has a weekly updated planet file in /data/planet-latest.osm.pbf
+          # See https://source.mapcomplete.org/MapComplete/planet-file-updater
+          TIMESTAMP=$(osmium fileinfo /data/planet-latest.osm.pbf -g header.option.timestamp)
+          DATE=$(echo $TIMESTAMP | sed "s/T.*//")
+          echo $DATE
+          # Create a new database in postgres
+          npm run create:database -- -- $DATE
+          echo "Seeding database '$DATE'"
+          osm2pgsql -O flex -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi.${DATE} /data/planet-latest.osm.pbf
+          npm run delete:database:old
+
+

.github/FUNDING.yml

@@ -1 +0,0 @@
-liberapay: pietervdvn

Docs/SettingUpPSQL.md

@@ -13,16 +13,9 @@ Increase the max number of connections. osm2pgsql needs connection one per table
 - Validate with `cat /var/lib/postgresql/data/postgresql.conf | grep "max_connections"`
 - `sudo docker restart <ID>`
 
+# Seeding the data
 
-> The following steps are also automated in `update.sh`
-
-## Create a database in the SQL-server
-
-Run `vite-node scripts/osm2pgsql/createNewDatabase.ts -- YYYY-MM-DD` to create a new, appropriate, database
-
-## Create export scripts for every layer
-
-Use `npm run build:dbscript`
+> The following steps are also automated in .forgejo/workflows/update_database.yml
 
 ## Importing data
 

scripts/osm2pgsql/update.sh

@@ -1,38 +0,0 @@
-#! /bin/bash
-
-# Full database update. Downloads the latest from planet.osm.org, build update script, setup and seed it
-# Meant to be run with 'nohup'
-export NVM_DIR="$HOME/.nvm"
-[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
-nvm use 20.12.2
-npm run init # contains a 'npm run generate, which builds the layers'
-npm run generate:buildDbScript
-mv build_db.lua ~/data/
-
-
-cd ~/data || exit
-rm planet-latest.osm.pbf
-
-wget https://planet.osm.org/pbf/planet-latest.osm.pbf
-
-TIMESTAMP=$(osmium fileinfo ~/data/planet-latest.osm.pbf -g header.option.timestamp)
-DATE=$(echo $TIMESTAMP | sed "s/T.*//")
-echo $DATE
-cd ~/git/MapComplete/ || exit
-npm run create:database -- -- ${DATE/T.*//}
-cd ~/data || exit
-
-rm seeddb.log
-osm2pgsql -O flex -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi.${DATE} planet-latest.osm.pbf >> seeddb.log
-
-# To see the progress
-# tail -f seeddb.log
-
-cd ~/git/MapComplete || exit
-npm run delete:database:old
-nohup npm run server:summary >> summary_server.log &
-cd - || exit
-
-# Restart tileserver
-export DATABASE_URL=postgresql://user:password@localhost:5444/osm-poi.${DATE}
-nohup ./pg_tileserv >> pg_tileserv.log &