From 676bb8a250311123ef19d2ee659099afef88457a Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Fri, 26 Sep 2025 01:53:34 +0200 Subject: [PATCH] Scripts: create server script for pmtiles --- Docs/ServerConfig/cache/Caddyfile | 1 + package.json | 1 + scripts/generatePmTilesExtracts.ts | 70 ++++++------------------------ scripts/pmTilesExtractGenerator.ts | 64 +++++++++++++++++++++++++++ scripts/server.ts | 35 +++++++++++++-- scripts/serverPmTileExtracts.ts | 64 +++++++++++++++++++++++++++ 6 files changed, 175 insertions(+), 60 deletions(-) create mode 100644 scripts/pmTilesExtractGenerator.ts create mode 100644 scripts/serverPmTileExtracts.ts diff --git a/Docs/ServerConfig/cache/Caddyfile b/Docs/ServerConfig/cache/Caddyfile index e54c42d99..1b90544e4 100644 --- a/Docs/ServerConfig/cache/Caddyfile +++ b/Docs/ServerConfig/cache/Caddyfile @@ -1,4 +1,5 @@ cache.mapcomplete.org { reverse_proxy /summary/* 127.0.0.1:2345 + reverse_proxy path_regexp ^/\d+/\d+/\d+\.pmtiles$ http://localhost:2346 reverse_proxy /* 127.0.0.1:7800 } diff --git a/package.json b/package.json index 418dbfd64..b8de0fec8 100644 --- a/package.json +++ b/package.json @@ -151,6 +151,7 @@ "server:ldjson": "vite-node scripts/serverLdScrape.ts", "server:studio": "vite-node scripts/studioServer -- /root/git/MapComplete/assets", "server:errorreport": "vite-node scripts/serverErrorReport.ts -- /root/error_reports/", + "server:pmtiles": "vite-node scripts/serverPmTileExtracts.ts", "generate:buildDbScript": "vite-node scripts/osm2pgsql/generateBuildDbScript.ts", "generate:summaryCache": "vite-node scripts/generateSummaryTileCache.ts", "create:database": "vite-node scripts/osm2pgsql/createNewDatabase.ts", diff --git a/scripts/generatePmTilesExtracts.ts b/scripts/generatePmTilesExtracts.ts index c3f26079f..09442db4f 100644 --- a/scripts/generatePmTilesExtracts.ts +++ b/scripts/generatePmTilesExtracts.ts @@ -1,15 +1,15 @@ import Script from "./Script" -import { Tiles } from "../src/Models/TileRange" - -import { spawn } from "child_process" import { existsSync, mkdirSync, writeFileSync } from "fs" import { Utils } from "../src/Utils" import { OfflineBasemapManager } from "../src/Logic/OfflineBasemapManager" +import { PmTilesExtractGenerator } from "./pmTilesExtractGenerator" class GeneratePmTilesExtracts extends Script { private targetDir: string - private sourceFile: string private skipped: number = 0 + + private extractsGenerator: PmTilesExtractGenerator + constructor() { super( "Generates many pmtiles-archive from planet-latest.pmtiles. Expects the `pmtiles`-executable to be at `/data/pmtiles`." + @@ -18,56 +18,14 @@ class GeneratePmTilesExtracts extends Script { ) } - startProcess(script: string, captureStdioChunks?: (string: string) => void): Promise { - return new Promise((resolve, reject) => { - const child = spawn("/data/pmtiles", script.split(" "), { - stdio: captureStdioChunks === undefined ? "ignore" : ["pipe", "pipe", "pipe"], - cwd: this.targetDir, - }) - - if (captureStdioChunks !== undefined) { - child.stdout.on('data', data => { - captureStdioChunks(data) - }); - } - - /*child.stderr.on('data', data => { - console.error(`stderr: ${data}`); - });*/ - - child.on("close", (code) => { - if (code === 0) resolve() - else reject(new Error(`Process exited with code ${code}`)) - }) - - child.on("error", reject) - }) - } - - private generateArchive(z: number, x: number, y: number, maxzoom?: number): Promise { - const [[max_lat, min_lon], [min_lat, max_lon]] = Tiles.tile_bounds(z, x, y) - let maxzoomflag = "" - if (maxzoom !== undefined) { - maxzoomflag = " --maxzoom=" + maxzoom - } - return this.startProcess( - `extract ${this.sourceFile} --download-threads=1 --minzoom=${z}${maxzoomflag} --bbox=${[ - min_lon, - min_lat + 0.0001, - max_lon, - max_lat, - ].join(",")} ${this.getFilename(z, x, y)}` - ) - } - - private *generateColumnIfNeeded( + public *generateColumnIfNeeded( z: number, x: number, boundary: number, maxzoom?: number - ): Generator> { - const lastFileForColumn = this.getFilename(z, x, boundary - 1) - if (existsSync(this.targetDir + "/" + lastFileForColumn)) { + ): Generator> { + const lastFileForColumn = this.extractsGenerator.getFilename(z, x, boundary - 1) + if (existsSync(lastFileForColumn)) { // Skip this column, already exists console.log("Skipping column ", x, "at zoom", z) this.skipped += boundary @@ -79,12 +37,12 @@ class GeneratePmTilesExtracts extends Script { "at zoom", z, "as", - this.targetDir + "/" + lastFileForColumn, + lastFileForColumn, "does not exist" ) for (let y = 0; y < boundary; y++) { - yield this.generateArchive(z, x, y, maxzoom) + yield this.extractsGenerator.generateArchive(z, x, y, maxzoom) } } @@ -100,9 +58,7 @@ class GeneratePmTilesExtracts extends Script { } } - private getFilename(z: number, x: number, y: number) { - return `${z}/${x}/${y}.pmtiles` - } + private *generateAll(): Generator> { const zoomlevels: Record = OfflineBasemapManager.zoomelevels @@ -132,11 +88,13 @@ class GeneratePmTilesExtracts extends Script { async main(args: string[]): Promise { this.targetDir = args[0] - this.sourceFile = this.targetDir+"/planet-latest.pmtiles" + const sourceFile = this.targetDir+"/planet-latest.pmtiles" if (!this.targetDir) { console.log("Please specify a target directory. Did you forget '--' in vite-node?") return } + this.extractsGenerator = new PmTilesExtractGenerator(sourceFile, this.targetDir) + let estimate = 0 for (const key in OfflineBasemapManager.zoomelevels) { const z: number = Number(key) diff --git a/scripts/pmTilesExtractGenerator.ts b/scripts/pmTilesExtractGenerator.ts new file mode 100644 index 000000000..4f8f162e7 --- /dev/null +++ b/scripts/pmTilesExtractGenerator.ts @@ -0,0 +1,64 @@ +import { spawn } from "child_process" +import { Tiles } from "../src/Models/TileRange" + +export class PmTilesExtractGenerator { + private readonly _targetDir: string + private readonly _sourceFile: string + private readonly _executeableLocation: string + + constructor(sourceFile: string, targetDir: string, executeableLocation: string = "/data/pmtiles") { + this._sourceFile = sourceFile + this._targetDir = targetDir + this._executeableLocation = executeableLocation + } + + startProcess(script: string, captureStdioChunks?: (string: string) => void): Promise { + return new Promise((resolve, reject) => { + const child = spawn(this._executeableLocation, script.split(" "), { + stdio: captureStdioChunks === undefined ? "ignore" : ["pipe", "pipe", "pipe"], + cwd: this._targetDir, + }) + + if (captureStdioChunks !== undefined) { + child.stdout.on("data", data => { + captureStdioChunks(data) + }) + } + + /*child.stderr.on('data', data => { + console.error(`stderr: ${data}`); + });*/ + + child.on("close", (code) => { + if (code === 0) resolve() + else reject(new Error(`Process exited with code ${code}`)) + }) + + child.on("error", reject) + }) + } + + getFilename(z: number, x: number, y: number) { + return `${this._targetDir}/${z}/${x}/${y}.pmtiles` + } + + async generateArchive(z: number, x: number, y: number, maxzoom?: number): Promise { + const [[max_lat, min_lon], [min_lat, max_lon]] = Tiles.tile_bounds(z, x, y) + let maxzoomflag = "" + if (maxzoom !== undefined) { + maxzoomflag = " --maxzoom=" + maxzoom + } + const outputFileName = this.getFilename(z, x, y) + await this.startProcess( + `extract ${this._sourceFile} --download-threads=1 --minzoom=${z}${maxzoomflag} --bbox=${[ + min_lon, + min_lat + 0.0001, + max_lon, + max_lat, + ].join(",")} ${outputFileName}`, + ) + return outputFileName + } + + +} diff --git a/scripts/server.ts b/scripts/server.ts index d60419ea8..74364b982 100644 --- a/scripts/server.ts +++ b/scripts/server.ts @@ -1,14 +1,22 @@ import http from "node:http" +import { ServerResponse } from "http" +import { createReadStream } from "node:fs" export interface Handler { mustMatch: string | RegExp mimetype: string addHeaders?: Record + /** + * IF set, do not write headers and result; only close. + * Use this if you want to send binary data + */ + unmanaged?: boolean handle: ( path: string, queryParams: URLSearchParams, req: http.IncomingMessage, - body: string | undefined + body: string | undefined, + res: http.ServerResponse ) => Promise } @@ -112,7 +120,11 @@ export class Server { } try { - const result = await handler.handle(path, url.searchParams, req, body) + const result = await handler.handle(path, url.searchParams, req, body, res) + if(handler.unmanaged){ + res.end() + return + } if (result === undefined) { res.writeHead(500) res.write("Could not fetch this website, probably blocked by them") @@ -129,14 +141,18 @@ export class Server { result ) } + const extraHeaders = handler.addHeaders ?? {} res.writeHead(200, { "Content-Type": handler.mimetype, ...extraHeaders }) res.write("" + result) res.end() + + } catch (e) { console.error("Could not handle request:", e) res.writeHead(500) - res.write(e) + res.write("Internal server error - something went wrong:") + res.write(e.toString()) res.end() } } catch (e) { @@ -145,8 +161,19 @@ export class Server { } }).listen(port) console.log( - "Server is running on port " + port, + "Server is running on http://127.0.0.1:" + port, ". Supported endpoints are: " + handle.map((h) => h.mustMatch).join(", ") ) } + + /** + * Sends a file straight from disk. + * Assumes that headers have been set on res + * @param path + * @param res + */ + public static sendFile(path: string, res: ServerResponse){ + createReadStream(path).pipe(res); + } + } diff --git a/scripts/serverPmTileExtracts.ts b/scripts/serverPmTileExtracts.ts new file mode 100644 index 000000000..6a5300924 --- /dev/null +++ b/scripts/serverPmTileExtracts.ts @@ -0,0 +1,64 @@ +import { Server } from "./server" +import Script from "./Script" +import { OfflineBasemapManager } from "../src/Logic/OfflineBasemapManager" +import http from "node:http" +import { ServerResponse } from "http" +import { existsSync } from "fs" +import ScriptUtils from "./ScriptUtils" +import { PmTilesExtractGenerator } from "./pmTilesExtractGenerator" + +class ServerPmTileExtracts extends Script { + constructor() { + super("Starts a server that serves PMtiles. Usage:\n" + + "sourceFile cachedir [portnumber??2346]") + } + + async main(args: string[]): Promise { + if(args.length < 2){ + this.printHelp() + return + } + const sourcefile = args[0] + const targetDir = args[1] + const port = Number(args[2] ?? "2346") + + const zoomlevels = OfflineBasemapManager.zoomelevels + const generator = new PmTilesExtractGenerator(sourcefile, targetDir) + + new Server(port, {}, + [ + { + mustMatch: /\d+\/\d+\/\d+.pmtiles/, + unmanaged: true, + mimetype: "application/octet-stream", + handle: async (path: string, + queryParams: URLSearchParams, + req: http.IncomingMessage, + body: string, + res: ServerResponse) => { + const [z,x,y] = path.split(".")[0].split("/").map(x => Number(x)) + const maxzoom = zoomlevels[z] + if(!maxzoom){ + throw "Invalid zoomlevel, must be one of "+Array.from(Object.keys(zoomlevels)).join(", ") + } + + const targetFile = generator.getFilename(z, x, y) + if(!existsSync(targetFile)){ + ScriptUtils.createParentDir(targetFile) + console.log("Creating", targetFile) + await generator.generateArchive(z, x, y) + } + + res.writeHead(200, { "Content-Type": "application/octet-stream" }) + Server.sendFile(targetFile, res) + + return null + }, + }, + ], + ) + } + +} + +new ServerPmTileExtracts().run()