Scripts: create server script for pmtiles

Docs/ServerConfig/cache/Caddyfile

@@ -1,4 +1,5 @@
 cache.mapcomplete.org {
     reverse_proxy /summary/* 127.0.0.1:2345
+    reverse_proxy path_regexp ^/\d+/\d+/\d+\.pmtiles$ http://localhost:2346
     reverse_proxy /* 127.0.0.1:7800
 }

package.json

@@ -151,6 +151,7 @@
     "server:ldjson": "vite-node scripts/serverLdScrape.ts",
     "server:studio": "vite-node scripts/studioServer -- /root/git/MapComplete/assets",
     "server:errorreport": "vite-node scripts/serverErrorReport.ts -- /root/error_reports/",
+    "server:pmtiles": "vite-node scripts/serverPmTileExtracts.ts",
     "generate:buildDbScript": "vite-node scripts/osm2pgsql/generateBuildDbScript.ts",
     "generate:summaryCache": "vite-node scripts/generateSummaryTileCache.ts",
     "create:database": "vite-node scripts/osm2pgsql/createNewDatabase.ts",

scripts/generatePmTilesExtracts.ts

@@ -1,15 +1,15 @@
 import Script from "./Script"
-import { Tiles } from "../src/Models/TileRange"
-
-import { spawn } from "child_process"
 import { existsSync, mkdirSync, writeFileSync } from "fs"
 import { Utils } from "../src/Utils"
 import { OfflineBasemapManager } from "../src/Logic/OfflineBasemapManager"
+import { PmTilesExtractGenerator } from "./pmTilesExtractGenerator"
 
 class GeneratePmTilesExtracts extends Script {
     private targetDir: string
-    private sourceFile: string
     private skipped: number = 0
+
+    private extractsGenerator: PmTilesExtractGenerator
+
     constructor() {
         super(
             "Generates many pmtiles-archive from planet-latest.pmtiles. Expects the `pmtiles`-executable to be at `/data/pmtiles`." +
@@ -18,56 +18,14 @@ class GeneratePmTilesExtracts extends Script {
         )
     }
 
-    startProcess(script: string, captureStdioChunks?: (string: string) => void): Promise<void> {
+    public *generateColumnIfNeeded(
-        return new Promise((resolve, reject) => {
-            const child = spawn("/data/pmtiles", script.split(" "), {
-                stdio: captureStdioChunks === undefined ? "ignore" : ["pipe", "pipe", "pipe"],
-                cwd: this.targetDir,
-            })
-
-            if (captureStdioChunks !== undefined) {
-                child.stdout.on('data', data => {
-                    captureStdioChunks(data)
-                });
-            }
-
-            /*child.stderr.on('data', data => {
-                console.error(`stderr: ${data}`);
-            });*/
-
-            child.on("close", (code) => {
-                if (code === 0) resolve()
-                else reject(new Error(`Process exited with code ${code}`))
-            })
-
-            child.on("error", reject)
-        })
-    }
-
-    private generateArchive(z: number, x: number, y: number, maxzoom?: number): Promise<void> {
-        const [[max_lat, min_lon], [min_lat, max_lon]] = Tiles.tile_bounds(z, x, y)
-        let maxzoomflag = ""
-        if (maxzoom !== undefined) {
-            maxzoomflag = " --maxzoom=" + maxzoom
-        }
-        return this.startProcess(
-            `extract ${this.sourceFile} --download-threads=1 --minzoom=${z}${maxzoomflag} --bbox=${[
-                min_lon,
-                min_lat + 0.0001,
-                max_lon,
-                max_lat,
-            ].join(",")} ${this.getFilename(z, x, y)}`
-        )
-    }
-
-    private *generateColumnIfNeeded(
         z: number,
         x: number,
         boundary: number,
         maxzoom?: number
-    ): Generator<Promise<void>> {
+    ): Generator<Promise<any>> {
-        const lastFileForColumn = this.getFilename(z, x, boundary - 1)
+        const lastFileForColumn = this.extractsGenerator.getFilename(z, x, boundary - 1)
-        if (existsSync(this.targetDir + "/" + lastFileForColumn)) {
+        if (existsSync(lastFileForColumn)) {
             // Skip this column, already exists
             console.log("Skipping column ", x, "at zoom", z)
             this.skipped += boundary
@@ -79,12 +37,12 @@ class GeneratePmTilesExtracts extends Script {
             "at zoom",
             z,
             "as",
-            this.targetDir + "/" + lastFileForColumn,
+            lastFileForColumn,
             "does not exist"
         )
 
         for (let y = 0; y < boundary; y++) {
-            yield this.generateArchive(z, x, y, maxzoom)
+            yield this.extractsGenerator.generateArchive(z, x, y, maxzoom)
         }
     }
 
@@ -100,9 +58,7 @@ class GeneratePmTilesExtracts extends Script {
         }
     }
 
-    private getFilename(z: number, x: number, y: number) {
+
-        return `${z}/${x}/${y}.pmtiles`
-    }
 
     private *generateAll(): Generator<Promise<void>> {
         const zoomlevels: Record<number, number> = OfflineBasemapManager.zoomelevels
@@ -132,11 +88,13 @@ class GeneratePmTilesExtracts extends Script {
 
     async main(args: string[]): Promise<void> {
         this.targetDir = args[0]
-        this.sourceFile = this.targetDir+"/planet-latest.pmtiles"
+        const sourceFile = this.targetDir+"/planet-latest.pmtiles"
         if (!this.targetDir) {
             console.log("Please specify a target directory. Did you forget '--' in vite-node?")
             return
         }
+        this.extractsGenerator = new PmTilesExtractGenerator(sourceFile, this.targetDir)
+
         let estimate = 0
         for (const key in OfflineBasemapManager.zoomelevels) {
             const z: number = Number(key)

scripts/pmTilesExtractGenerator.ts

@@ -0,0 +1,64 @@
+import { spawn } from "child_process"
+import { Tiles } from "../src/Models/TileRange"
+
+export class PmTilesExtractGenerator {
+    private readonly _targetDir: string
+    private readonly _sourceFile: string
+    private readonly _executeableLocation: string
+
+    constructor(sourceFile: string, targetDir: string, executeableLocation: string = "/data/pmtiles") {
+        this._sourceFile = sourceFile
+        this._targetDir = targetDir
+        this._executeableLocation = executeableLocation
+    }
+
+    startProcess(script: string, captureStdioChunks?: (string: string) => void): Promise<void> {
+        return new Promise((resolve, reject) => {
+            const child = spawn(this._executeableLocation, script.split(" "), {
+                stdio: captureStdioChunks === undefined ? "ignore" : ["pipe", "pipe", "pipe"],
+                cwd: this._targetDir,
+            })
+
+            if (captureStdioChunks !== undefined) {
+                child.stdout.on("data", data => {
+                    captureStdioChunks(data)
+                })
+            }
+
+            /*child.stderr.on('data', data => {
+                console.error(`stderr: ${data}`);
+            });*/
+
+            child.on("close", (code) => {
+                if (code === 0) resolve()
+                else reject(new Error(`Process exited with code ${code}`))
+            })
+
+            child.on("error", reject)
+        })
+    }
+
+    getFilename(z: number, x: number, y: number) {
+        return `${this._targetDir}/${z}/${x}/${y}.pmtiles`
+    }
+
+    async generateArchive(z: number, x: number, y: number, maxzoom?: number): Promise<string> {
+        const [[max_lat, min_lon], [min_lat, max_lon]] = Tiles.tile_bounds(z, x, y)
+        let maxzoomflag = ""
+        if (maxzoom !== undefined) {
+            maxzoomflag = " --maxzoom=" + maxzoom
+        }
+        const outputFileName = this.getFilename(z, x, y)
+        await this.startProcess(
+            `extract ${this._sourceFile} --download-threads=1 --minzoom=${z}${maxzoomflag} --bbox=${[
+                min_lon,
+                min_lat + 0.0001,
+                max_lon,
+                max_lat,
+            ].join(",")} ${outputFileName}`,
+        )
+        return outputFileName
+    }
+
+
+}

scripts/server.ts

@@ -1,14 +1,22 @@
 import http from "node:http"
+import { ServerResponse } from "http"
+import { createReadStream } from "node:fs"
 
 export interface Handler {
     mustMatch: string | RegExp
     mimetype: string
     addHeaders?: Record<string, string>
+    /**
+     * IF set, do not write headers and result; only close.
+     * Use this if you want to send binary data
+     */
+    unmanaged?: boolean
     handle: (
         path: string,
         queryParams: URLSearchParams,
         req: http.IncomingMessage,
-        body: string | undefined
+        body: string | undefined,
+        res: http.ServerResponse
     ) => Promise<string>
 }
 
@@ -112,7 +120,11 @@ export class Server {
                 }
 
                 try {
-                    const result = await handler.handle(path, url.searchParams, req, body)
+                    const result = await handler.handle(path, url.searchParams, req, body, res)
+                    if(handler.unmanaged){
+                        res.end()
+                        return
+                    }
                     if (result === undefined) {
                         res.writeHead(500)
                         res.write("Could not fetch this website, probably blocked by them")
@@ -129,14 +141,18 @@ export class Server {
                             result
                         )
                     }
+
                     const extraHeaders = handler.addHeaders ?? {}
                     res.writeHead(200, { "Content-Type": handler.mimetype, ...extraHeaders })
                     res.write("" + result)
                     res.end()
+
+
                 } catch (e) {
                     console.error("Could not handle request:", e)
                     res.writeHead(500)
-                    res.write(e)
+                    res.write("Internal server error - something went wrong:")
+                    res.write(e.toString())
                     res.end()
                 }
             } catch (e) {
@@ -145,8 +161,19 @@ export class Server {
             }
         }).listen(port)
         console.log(
-            "Server is running on port " + port,
+            "Server is running on http://127.0.0.1:" + port,
             ". Supported endpoints are: " + handle.map((h) => h.mustMatch).join(", ")
         )
     }
+
+    /**
+     * Sends a file straight from disk.
+     * Assumes that headers have been set on res
+     * @param path
+     * @param res
+     */
+    public static sendFile(path: string, res: ServerResponse){
+        createReadStream(path).pipe(res);
+    }
+
 }

scripts/serverPmTileExtracts.ts

@@ -0,0 +1,64 @@
+import { Server } from "./server"
+import Script from "./Script"
+import { OfflineBasemapManager } from "../src/Logic/OfflineBasemapManager"
+import http from "node:http"
+import { ServerResponse } from "http"
+import { existsSync } from "fs"
+import ScriptUtils from "./ScriptUtils"
+import { PmTilesExtractGenerator } from "./pmTilesExtractGenerator"
+
+class ServerPmTileExtracts extends Script {
+    constructor() {
+        super("Starts a server that serves PMtiles. Usage:\n" +
+            "sourceFile cachedir [portnumber??2346]")
+    }
+
+    async main(args: string[]): Promise<void> {
+        if(args.length < 2){
+            this.printHelp()
+            return
+        }
+        const sourcefile = args[0]
+        const targetDir = args[1]
+        const port = Number(args[2] ?? "2346")
+
+        const zoomlevels = OfflineBasemapManager.zoomelevels
+        const generator = new PmTilesExtractGenerator(sourcefile, targetDir)
+
+        new Server(port, {},
+            [
+                {
+                    mustMatch: /\d+\/\d+\/\d+.pmtiles/,
+                    unmanaged: true,
+                    mimetype: "application/octet-stream",
+                    handle: async (path: string,
+                                   queryParams: URLSearchParams,
+                                   req: http.IncomingMessage,
+                                   body: string,
+                                   res: ServerResponse) => {
+                        const [z,x,y] = path.split(".")[0].split("/").map(x => Number(x))
+                        const maxzoom = zoomlevels[z]
+                        if(!maxzoom){
+                            throw "Invalid zoomlevel, must be one of "+Array.from(Object.keys(zoomlevels)).join(", ")
+                        }
+
+                        const targetFile = generator.getFilename(z, x, y)
+                        if(!existsSync(targetFile)){
+                            ScriptUtils.createParentDir(targetFile)
+                            console.log("Creating", targetFile)
+                            await generator.generateArchive(z, x, y)
+                        }
+
+                        res.writeHead(200, { "Content-Type": "application/octet-stream" })
+                        Server.sendFile(targetFile, res)
+
+                        return null
+                    },
+                },
+            ],
+        )
+    }
+
+}
+
+new ServerPmTileExtracts().run()