Merge develop

scripts/ScriptUtils.ts

@@ -172,6 +172,13 @@ export default class ScriptUtils {
         headers?: any,
         timeoutSecs?: number
     ): Promise<{ content: string } | { redirect: string } | "timeout"> {
+        if (url.startsWith("./assets")) {
+            return Promise.resolve({ content: readFileSync("./public/" + url, "utf8") })
+        }
+        if (url.startsWith("./")) {
+            return Promise.resolve({ content: readFileSync(url, "utf8") })
+        }
+
         const requestPromise = new Promise((resolve, reject) => {
             try {
                 headers = headers ?? {}

scripts/downloadEli.ts

@@ -12,7 +12,8 @@ class DownloadEli extends Script {
     async main(args: string[]): Promise<void> {
         const url = "https://osmlab.github.io/editor-layer-index/imagery.geojson"
         // Target should use '.json' instead of '.geojson', as the latter cannot be imported by the build systems
-        const target = args[0] ?? "src/assets/editor-layer-index.json"
+        const target = args[0] ?? "public/assets/data/editor-layer-index.json"
+        const targetBing = args[0] ?? "src/assets/bing.json"
 
         const eli: Eli = await Utils.downloadJson(url)
         const keptLayers: EliEntry[] = []
@@ -99,8 +100,19 @@ class DownloadEli extends Script {
 
         const contents =
             '{"type":"FeatureCollection",\n  "features": [\n' +
-            keptLayers.map((l) => JSON.stringify(l)).join(",\n") +
+            keptLayers
+                .filter((l) => l.properties.id !== "Bing")
+                .map((l) => JSON.stringify(l))
+                .join(",\n") +
             "\n]}"
+
+        const bing = keptLayers.find((l) => l.properties.id === "Bing")
+        if (bing) {
+            fs.writeFileSync(targetBing, JSON.stringify(bing), { encoding: "utf8" })
+            console.log("Written", targetBing)
+        } else {
+            console.log("No bing entry found")
+        }
         fs.writeFileSync(target, contents, { encoding: "utf8" })
         console.log("Written", keptLayers.length + ", entries to the ELI")
     }

scripts/generateLayerOverview.ts

@@ -9,7 +9,6 @@ import {
     DoesImageExist,
     PrevalidateTheme,
     ValidateLayer,
-    ValidateThemeAndLayers,
     ValidateThemeEnsemble,
 } from "../src/Models/ThemeConfig/Conversion/Validation"
 import { Translation } from "../src/UI/i18n/Translation"
@@ -33,6 +32,8 @@ import { GenerateFavouritesLayer } from "./generateFavouritesLayer"
 import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig"
 import Translations from "../src/UI/i18n/Translations"
 import { Translatable } from "../src/Models/ThemeConfig/Json/Translatable"
+import { ValidateThemeAndLayers } from "../src/Models/ThemeConfig/Conversion/ValidateThemeAndLayers"
+import { ExtractImages } from "../src/Models/ThemeConfig/Conversion/FixImages"
 
 // This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files.
 // It spits out an overview of those to be used to load them
@@ -80,15 +81,16 @@ class ParseLayer extends Conversion<
         }
         const fixed = this._prepareLayer.convert(parsed, context.inOperation("PrepareLayer"))
 
-        if (!fixed.source) {
-            context.enter("source").err("No source is configured")
+        if (!fixed.source && fixed.presets?.length < 1) {
+            context.enter("source").err("No source is configured. (Tags might be automatically derived if presets are given)")
             return undefined
         }
 
         if (
+            fixed.source &&
             typeof fixed.source !== "string" &&
-            fixed.source["osmTags"] &&
-            fixed.source["osmTags"]["and"] === undefined
+            fixed.source?.["osmTags"] &&
+            fixed.source?.["osmTags"]["and"] === undefined
         ) {
             fixed.source["osmTags"] = { and: [fixed.source["osmTags"]] }
         }
@@ -850,6 +852,15 @@ class LayerOverviewUtils extends Script {
                     }
                 }
 
+                const usedImages = Utils.Dedup(
+                    new ExtractImages(true, knownTagRenderings)
+                        .convertStrict(themeFile)
+                        .map((x) => x.path)
+                )
+                usedImages.sort()
+
+                themeFile["_usedImages"] = usedImages
+
                 this.writeTheme(themeFile)
                 fixed.set(themeFile.id, themeFile)
 

scripts/generateLayouts.ts

@@ -12,7 +12,7 @@ import SpecialVisualizations from "../src/UI/SpecialVisualizations"
 import Constants from "../src/Models/Constants"
 import { AvailableRasterLayers, RasterLayerPolygon } from "../src/Models/RasterLayers"
 import { ImmutableStore } from "../src/Logic/UIEventSource"
-import * as eli from "../src/assets/editor-layer-index.json"
+import * as eli from "../public/assets/data/editor-layer-index.json"
 import * as eli_global from "../src/assets/global-raster-layers.json"
 import ValidationUtils from "../src/Models/ThemeConfig/Conversion/ValidationUtils"
 import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
@@ -366,7 +366,7 @@ class GenerateLayouts extends Script {
         hosts.add("https://schema.org")
         const eliLayers: RasterLayerPolygon[] = AvailableRasterLayers.layersAvailableAt(
             new ImmutableStore({ lon: 0, lat: 0 })
-        ).data
+        ).store.data
         {
             const vectorLayers = eliLayers.filter((l) => l.properties.type === "vector")
             const vectorSources = vectorLayers.map((l) => l.properties.url)

scripts/generateSunnyUnlabeled.ts

@@ -7,14 +7,20 @@ export class GenerateSunnyUnlabeled extends Script {
         super("Generates 'sunny-unlabeled.json' based on sunny.json")
     }
 
-    async main(args: string[]): Promise<void> {
+
+    generateUnlabeled() {
         const unlabeled = { "#": "AUTOMATICALLY GENERATED! Do not edit.", ...sunny }
         unlabeled.name = unlabeled.name + "-unlabeled"
         unlabeled.layers = sunny.layers.filter(
-            (l) => l.type !== "symbol" || !l.layout["text-field"]
+            (l) => l.type !== "symbol" || !l.layout["text-field"],
         )
         writeFileSync("public/assets/sunny-unlabeled.json", JSON.stringify(unlabeled, null, "   "))
     }
+
+
+    async main(args: string[]): Promise<void> {
+        this.generateUnlabeled()
+    }
 }
 
 new GenerateSunnyUnlabeled().run()

scripts/handleErrors.ts

@@ -28,7 +28,6 @@ class HandleErrors extends Script {
     constructor() {
         super("Inspects the errors made on a given day. Argument: path to errors")
     }
-
     async main(args: string[]): Promise<void> {
         const osmConnection = new OsmConnection()
         const downloader = new OsmObjectDownloader(osmConnection.Backend(), undefined)
@@ -75,7 +74,12 @@ class HandleErrors extends Script {
         }
 
         for (const parsed of all) {
-            console.log(parsed.message.username, parsed.message.layout, parsed.message.message, parsed.date)
+            console.log(
+                parsed.message.username,
+                parsed.message.layout,
+                parsed.message.message,
+                parsed.date
+            )
 
             const e = parsed.message
             const neededIds = Changes.GetNeededIds(e.pendingChanges)
@@ -113,7 +117,7 @@ class HandleErrors extends Script {
                     "Changes for " + parsed.index + ": empty changeset, not creating a file for it"
                 )*/
             } else if (createdChangesets.has(changeset)) {
-               /* console.log(
+                /* console.log(
                     "Changeset " +
                         parsed.index +
                         " is identical to previously seen changeset, not writing to file"
@@ -127,7 +131,7 @@ ${changeset}`
             }
             const refusedContent = JSON.stringify(refused)
             if (refusedFiles.has(refusedContent)) {
-               /* console.log(
+                /* console.log(
                     "Refused changes for " +
                         parsed.index +
                         " is identical to previously seen changeset, not writing to file"

scripts/osm2pgsql/createNewDatabase.ts

@@ -0,0 +1,16 @@
+import Script from "../Script"
+import { OsmPoiDatabase } from "./osmPoiDatabase"
+
+class CreateNewDatabase extends Script {
+    constructor() {
+        super("Creates a new version of the database. Usage: `createNewDatabase -- YYYY-MM-DD` which will create database `osm-poi.YYYY-MM-DD`")
+    }
+
+    async main(args: string[]): Promise<void> {
+        const db = new OsmPoiDatabase("postgresql://user:password@localhost:5444")
+        await db.createNew(args[0])
+    }
+}
+
+
+new CreateNewDatabase().run()

scripts/osm2pgsql/deleteOldDbs.ts

@@ -0,0 +1,16 @@
+import Script from "../Script"
+import { OsmPoiDatabase } from "./osmPoiDatabase"
+
+class DeleteOldDbs extends Script {
+    constructor() {
+        super("Drops all but the newest `osm-poi.*`")
+    }
+
+    async main(args: string[]): Promise<void> {
+        const db = new OsmPoiDatabase("postgresql://user:password@localhost:5444")
+        await db.deleteAllButLatest()
+    }
+}
+
+
+new DeleteOldDbs().run()

scripts/osm2pgsql/osmPoiDatabase.ts

@@ -0,0 +1,240 @@
+import { Client } from "pg"
+
+/**
+ * Just the OSM2PGSL default database
+ */
+export interface PoiDatabaseMeta {
+    attributes
+    current_timestamp
+    db_format
+    flat_node_file
+    import_timestamp
+    output
+    prefix
+    replication_base_url
+    replication_sequence_number
+    replication_timestamp
+    style
+    updatable
+    version
+}
+
+/**
+ * Connects with a Postgis database, gives back how much items there are within the given BBOX
+ */
+export class OsmPoiDatabase {
+    private static readonly prefixes: ReadonlyArray<string> = ["pois", "lines", "polygons"]
+    private _client: Client
+    private isConnected = false
+    private supportedLayers: Set<string> = undefined
+    private supportedLayersDate: Date = undefined
+    private metaCache: PoiDatabaseMeta = undefined
+    private metaCacheDate: Date = undefined
+    private readonly _connectionString: string
+
+    private static readonly _prefix = "osm-poi" as const
+
+    constructor(connectionString: string) {
+        this._connectionString = connectionString
+    }
+
+    private getMetaClient() {
+        return new Client(this._connectionString + "/postgres")
+    }
+
+    private async connectIfNeeded() {
+        if (this.isConnected) {
+            return
+        }
+        this.metaCache = undefined
+        await this.connectToLatest()
+        this._client.once("end", () => {
+            this.isConnected = false
+        })
+        this._client.once("error", () => {
+            if (this.isConnected) {
+                this.isConnected = false
+                try {
+                    this._client.end()
+                } catch (e) {
+                    console.error("Could not disconnect")
+                }
+            }
+        })
+        this.isConnected = true
+    }
+
+    async findSuitableDatabases(): Promise<string[]> {
+        const metaclient = this.getMetaClient()
+        await metaclient.connect()
+        try {
+
+            const meta = await metaclient.query("SELECT datname FROM pg_database")
+            let latest: string = undefined
+            let latestDate: Date = new Date(0)
+            const dbs: string[] = []
+            for (const row of meta.rows) {
+                const name: string = row["datname"]
+                if (!name.startsWith(OsmPoiDatabase._prefix)) {
+                    continue
+                }
+                const nm = name.substring(OsmPoiDatabase._prefix.length + 1)
+                dbs.push(nm)
+            }
+            dbs.sort()
+            return dbs
+        } finally {
+            await metaclient.end()
+        }
+    }
+
+    async searchLatest() {
+        const dbs = await this.findSuitableDatabases()
+        let latest: string = undefined
+        let latestDate: Date = undefined
+        for (const name of dbs) {
+            const date = new Date(name)
+            if (latestDate.getTime() < date.getTime()) {
+                latest = name
+                latestDate = date
+            }
+        }
+        if (latest === undefined) {
+            throw "No suitable database found"
+        }
+
+        console.log("Latest database is:", latest)
+        return latest
+    }
+
+    async createNew(date: string) {
+        const dbname = `${OsmPoiDatabase._prefix}.${date}`
+        console.log("Attempting to create a new database with name", dbname)
+        const metaclient = this.getMetaClient()
+        await metaclient.connect()
+        try {
+
+            await metaclient.query(`CREATE DATABASE "${dbname}"`)
+            console.log("Database created - installing extensions")
+            const client = new Client(this._connectionString + "/" + dbname)
+            try {
+                await client.connect()
+                await client.query(`CREATE EXTENSION IF NOT EXISTS postgis`)
+                console.log("Created database", dbname, "with postgis")
+            } finally {
+                await client.end()
+            }
+        } finally {
+            await metaclient.end()
+        }
+    }
+
+    async deleteAllButLatest(){
+        const dbs = await this.findSuitableDatabases()
+        for (let i = 0; i < dbs.length - 1; i++) {
+            await this.deleteDatabase(dbs[i])
+        }
+    }
+
+    /**
+     * DANGEROUS
+     * Drops de database with the given name
+     * @param date
+     */
+    async deleteDatabase(date: string) {
+        const metaclient = this.getMetaClient()
+        await metaclient.connect()
+        try {
+            await metaclient.query(`DROP DATABASE "${OsmPoiDatabase._prefix}.${date}"`)
+            console.log(`Dropped database ${OsmPoiDatabase._prefix}.${date}`)
+        } finally {
+            await metaclient.end()
+        }
+    }
+
+
+    async connectToLatest() {
+        const latest = await this.searchLatest()
+        this._client = new Client(this._connectionString + "/" + latest)
+        await this._client.connect()
+    }
+
+    async getCount(
+        layer: string,
+        bbox: [[number, number], [number, number]] = undefined,
+    ): Promise<{ count: number; lat: number; lon: number }> {
+        await this.connectIfNeeded()
+
+        let total: number = 0
+        let latSum = 0
+        let lonSum = 0
+        for (const prefix of OsmPoiDatabase.prefixes) {
+            let query =
+                "SELECT COUNT(*), ST_AsText(ST_Centroid(ST_Collect(geom))) FROM " +
+                prefix +
+                "_" +
+                layer
+
+            if (bbox) {
+                query += ` WHERE ST_MakeEnvelope (${bbox[0][0]}, ${bbox[0][1]}, ${bbox[1][0]}, ${bbox[1][1]}, 4326) ~ geom`
+            }
+            const result = await this._client.query(query)
+            const count = Number(result.rows[0].count)
+            let point = result.rows[0].st_astext
+            if (count === 0) {
+                continue
+            }
+            total += count
+            if (!point) {
+                continue
+            }
+            point = point.substring(6, point.length - 1)
+            const [lon, lat] = point.split(" ")
+            latSum += lat * count
+            lonSum += lon * count
+        }
+
+        return { count: total, lat: latSum / total, lon: lonSum / total }
+    }
+
+    disconnect() {
+        this._client.end()
+    }
+
+    async getLayers(): Promise<Set<string>> {
+        if (
+            this.supportedLayers !== undefined &&
+            new Date().getTime() - this.supportedLayersDate.getTime() < 1000 * 60 * 60 * 24
+        ) {
+            return this.supportedLayers
+        }
+        const q =
+            "SELECT table_name \n" +
+            "FROM information_schema.tables \n" +
+            "WHERE table_schema = 'public' AND table_name LIKE 'lines_%';"
+        const result = await this._client.query(q)
+        const layers = result.rows.map((r) => r.table_name.substring("lines_".length))
+        this.supportedLayers = new Set(layers)
+        this.supportedLayersDate = new Date()
+        return this.supportedLayers
+    }
+
+    async getMeta(): Promise<PoiDatabaseMeta> {
+        const now = new Date()
+        if (this.metaCache !== undefined) {
+            const diffSec = (this.metaCacheDate.getTime() - now.getTime()) / 1000
+            if (diffSec < 120) {
+                return this.metaCache
+            }
+        }
+        await this.connectIfNeeded()
+        const result = await this._client.query("SELECT * FROM public.osm2pgsql_properties")
+        const meta = {}
+        for (const { property, value } of result.rows) {
+            meta[property] = value
+        }
+        this.metaCacheDate = now
+        this.metaCache = <PoiDatabaseMeta>meta
+        return this.metaCache
+    }
+}

scripts/osm2pgsql/tilecountServer.ts

@@ -1,124 +1,8 @@
-import { Client } from "pg"
 import { Tiles } from "../../src/Models/TileRange"
 import { Server } from "../server"
 import Script from "../Script"
+import { OsmPoiDatabase } from "./osmPoiDatabase"
 
-/**
- * Just the OSM2PGSL default database
- */
-interface PoiDatabaseMeta {
-    attributes
-    current_timestamp
-    db_format
-    flat_node_file
-    import_timestamp
-    output
-    prefix
-    replication_base_url
-    replication_sequence_number
-    replication_timestamp
-    style
-    updatable
-    version
-}
-
-/**
- * Connects with a Postgis database, gives back how much items there are within the given BBOX
- */
-class OsmPoiDatabase {
-    private static readonly prefixes: ReadonlyArray<string> = ["pois", "lines", "polygons"]
-    private readonly _client: Client
-    private isConnected = false
-    private supportedLayers: Set<string> = undefined
-    private supportedLayersDate: Date = undefined
-    private metaCache: PoiDatabaseMeta = undefined
-    private metaCacheDate: Date = undefined
-
-    constructor(connectionString: string) {
-        this._client = new Client(connectionString)
-    }
-
-    async getCount(
-        layer: string,
-        bbox: [[number, number], [number, number]] = undefined
-    ): Promise<{ count: number; lat: number; lon: number }> {
-        if (!this.isConnected) {
-            await this._client.connect()
-            this.isConnected = true
-        }
-
-        let total: number = 0
-        let latSum = 0
-        let lonSum = 0
-        for (const prefix of OsmPoiDatabase.prefixes) {
-            let query =
-                "SELECT COUNT(*), ST_AsText(ST_Centroid(ST_Collect(geom))) FROM " +
-                prefix +
-                "_" +
-                layer
-
-            if (bbox) {
-                query += ` WHERE ST_MakeEnvelope (${bbox[0][0]}, ${bbox[0][1]}, ${bbox[1][0]}, ${bbox[1][1]}, 4326) ~ geom`
-            }
-            const result = await this._client.query(query)
-            const count = Number(result.rows[0].count)
-            let point = result.rows[0].st_astext
-            if (count === 0) {
-                continue
-            }
-            total += count
-            if (!point) {
-                continue
-            }
-            point = point.substring(6, point.length - 1)
-            const [lon, lat] = point.split(" ")
-            latSum += lat * count
-            lonSum += lon * count
-        }
-
-        return { count: total, lat: latSum / total, lon: lonSum / total }
-    }
-
-    disconnect() {
-        this._client.end()
-    }
-
-    async getLayers(): Promise<Set<string>> {
-        if (
-            this.supportedLayers !== undefined &&
-            new Date().getTime() - this.supportedLayersDate.getTime() < 1000 * 60 * 60 * 24
-        ) {
-            return this.supportedLayers
-        }
-        const q =
-            "SELECT table_name \n" +
-            "FROM information_schema.tables \n" +
-            "WHERE table_schema = 'public' AND table_name LIKE 'lines_%';"
-        const result = await this._client.query(q)
-        const layers = result.rows.map((r) => r.table_name.substring("lines_".length))
-        this.supportedLayers = new Set(layers)
-        this.supportedLayersDate = new Date()
-        return this.supportedLayers
-    }
-
-    async getMeta(): Promise<PoiDatabaseMeta> {
-        const now = new Date()
-        if (this.metaCache !== undefined) {
-            const diffSec = (this.metaCacheDate.getTime() - now.getTime()) / 1000
-            if (diffSec < 120) {
-                return this.metaCache
-            }
-        }
-        const result = await this._client.query("SELECT * FROM public.osm2pgsql_properties")
-        const meta = {}
-        for (const { property, value } of result.rows) {
-            meta[property] = value
-        }
-        this.metaCacheDate = now
-        this.metaCache = <PoiDatabaseMeta>meta
-        return this.metaCache
-    }
-}
 
 class CachedSqlCount {
     private readonly _cache: Record<
@@ -169,7 +53,7 @@ class TileCountServer extends Script {
     }
 
     async main(args: string[]): Promise<void> {
-        const connectionString = args[0] ?? "postgresql://user:password@localhost:5444/osm-poi"
+        const connectionString = args[0] ?? "postgresql://user:password@localhost:5444"
         const port = Number(args[1] ?? 2345)
         console.log("Connecting to database", connectionString)
         const tcs = new OsmPoiDatabase(connectionString)

scripts/osm2pgsql/update.sh

@@ -1,8 +1,27 @@
 #! /bin/bash
 
-npm run init
-npm run generate
-npm run refresh:layeroverview
+# Full database update. DOwnload latest from planet.osm.org, build update script, setup and seed it
+nvm use
+npm run init # contains a 'npm run generate, which builds the layers'
 npm run generate:buildDbScript
-mv build_db.sh ~/data/
-transmission-cli https://planet.osm.org/pbf/planet-latest.osm.pbf.torrent -f ./on_data_downloaded.sh &>nohup_transmission.log
+mv build_db.lua ~/data/
+TIMESTAMP=$(osmium fileinfo ~/data/planet-latest.osm.pbf -g header.option.timestamp)
+DATE=$(echo $TIMESTAMP | sed "s/T.*//")
+echo $DATE
+npm run create:database -- -- ${DATE/T.*//}
+
+cd ~/data || exit
+rm planet-latest.osm.pbf
+wget https://planet.osm.org/pbf/planet-latest.osm.pbf
+
+rm seeddb.log
+osm2pgsql -O flex -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi planet-latest.osm.pbf >> seeddb.log
+
+# To see the progress
+# tail -f seeddb.log
+
+npm run delete:database:old
+
+# Restart tileserver
+export DATABASE_URL=postgresql://user:password@localhost:5444/osm-poi.${DATE}
+nohup ./pg_tileserv >> pg_tileserv.log &