diff --git a/.forgejo/workflows/update_database.yml b/.forgejo/workflows/update_database.yml new file mode 100644 index 0000000000..ea77ff6ba5 --- /dev/null +++ b/.forgejo/workflows/update_database.yml @@ -0,0 +1,42 @@ +on: + schedule: + - cron: "0 0 1 */2 *" + + +jobs: + daily_data_maintenance: + runs-on: [ lain ] + steps: + - uses: https://source.mapcomplete.org/actions/checkout@v4 + + - name: Set up Node.js + uses: https://source.mapcomplete.org/actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: package-lock.json + + - name: Init all dependencies and layers + shell: bash + run: npm run init + + - name: Create export script + shell: bash + run: npm run build:dbscript # output: build_db.lua + + + - name: Print planet file date + shell: bash + run: | + # LAIN has a weekly updated planet file in /data/planet-latest.osm.pbf + # See https://source.mapcomplete.org/MapComplete/planet-file-updater + TIMESTAMP=$(osmium fileinfo /data/planet-latest.osm.pbf -g header.option.timestamp) + DATE=$(echo $TIMESTAMP | sed "s/T.*//") + echo $DATE + # Create a new database in postgres + vite-node scripts/osm2pgsql/createNewDatabase.ts -- $DATE + echo "Seeding database '$DATE'" + osm2pgsql -O flex -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi.${DATE} /data/planet-latest.osm.pbf + npm run delete:database:old + + diff --git a/Docs/SettingUpPSQL.md b/Docs/SettingUpPSQL.md index 3c9cb19b7b..c8da686795 100644 --- a/Docs/SettingUpPSQL.md +++ b/Docs/SettingUpPSQL.md @@ -13,16 +13,9 @@ Increase the max number of connections. osm2pgsql needs connection one per table - Validate with `cat /var/lib/postgresql/data/postgresql.conf | grep "max_connections"` - `sudo docker restart ` +# Seeding the data -> The following steps are also automated in `update.sh` - -## Create a database in the SQL-server - -Run `vite-node scripts/osm2pgsql/createNewDatabase.ts -- YYYY-MM-DD` to create a new, appropriate, database - -## Create export scripts for every layer - -Use `npm run build:dbscript` +> The following steps are also automated in .forgejo/workflows/update_database.yml ## Importing data diff --git a/scripts/osm2pgsql/update.sh b/scripts/osm2pgsql/update.sh deleted file mode 100755 index 1ed6aaec5c..0000000000 --- a/scripts/osm2pgsql/update.sh +++ /dev/null @@ -1,38 +0,0 @@ -#! /bin/bash - -# Full database update. Downloads the latest from planet.osm.org, build update script, setup and seed it -# Meant to be run with 'nohup' -export NVM_DIR="$HOME/.nvm" -[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm -nvm use 20.12.2 -npm run init # contains a 'npm run generate, which builds the layers' -npm run generate:buildDbScript -mv build_db.lua ~/data/ - - -cd ~/data || exit -rm planet-latest.osm.pbf - -wget https://planet.osm.org/pbf/planet-latest.osm.pbf - -TIMESTAMP=$(osmium fileinfo ~/data/planet-latest.osm.pbf -g header.option.timestamp) -DATE=$(echo $TIMESTAMP | sed "s/T.*//") -echo $DATE -cd ~/git/MapComplete/ || exit -npm run create:database -- -- ${DATE/T.*//} -cd ~/data || exit - -rm seeddb.log -osm2pgsql -O flex -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi.${DATE} planet-latest.osm.pbf >> seeddb.log - -# To see the progress -# tail -f seeddb.log - -cd ~/git/MapComplete || exit -npm run delete:database:old -nohup npm run server:summary >> summary_server.log & -cd - || exit - -# Restart tileserver -export DATABASE_URL=postgresql://user:password@localhost:5444/osm-poi.${DATE} -nohup ./pg_tileserv >> pg_tileserv.log &