Scripts(offline): add offline generation script

scripts/generatePmTilesExtractionScript.ts

@@ -1,39 +0,0 @@
-import Script from "./Script"
-import { Tiles } from "../src/Models/TileRange"
-
-class GeneratePmTilesExtractionScript extends Script {
-
-    constructor() {
-        super("Generates a bash script to extract all subpyramids of maxzoom=8 from planet-latest.pmtiles")
-    }
-
-    private emitRange(z: number, x: number, y: number, maxzoom?: number): string {
-        const [[max_lat, min_lon], [min_lat, max_lon]] = Tiles.tile_bounds(z, x, y)
-        let maxzoomflag = ""
-        if(maxzoom !== undefined){
-            maxzoomflag = " --maxzoom="+maxzoom
-        }
-        return (`./pmtiles extract planet-latest.pmtiles --minzoom=${z}${maxzoomflag} --bbox=${[min_lon, min_lat + 0.0001, max_lon, max_lat].join(",")} ${z}-${x}-${y}.pmtiles`)
-    }
-
-    private generateField(z: number, maxzoom?:number){
-        const boundary = 2 << z
-        for (let x = 0; x < boundary; x++) {
-            const xCommands = []
-            for (let y = 0; y < boundary; y++) {
-                xCommands.push(this.emitRange(z, x, y,maxzoom))
-            }
-            console.log(xCommands.join(" && ") + " && echo 'All pyramids for x = " + x + " are generated' & ")
-        }
-
-    }
-
-    async main(): Promise<void> {
-        this.generateField(0, 4)
-        this.generateField(5, 8)
-        this.generateField(9)
-    }
-
-}
-
-new GeneratePmTilesExtractionScript().run()

scripts/generatePmTilesExtracts.ts

@@ -0,0 +1,80 @@
+import Script from "./Script"
+import { Tiles } from "../src/Models/TileRange"
+import { OfflineBasemapManager } from "../src/service-worker/OfflineBasemapManager"
+
+import { spawn } from "child_process"
+
+function startProcess(script: string): Promise<void> {
+    return new Promise((resolve, reject) => {
+        const child = spawn("node", [script], { stdio: "inherit" })
+
+        child.on("close", (code) => {
+            if (code === 0) resolve()
+            else reject(new Error(`Process exited with code ${code}`))
+        })
+
+        child.on("error", reject)
+    })
+}
+
+
+class GeneratePmTilesExtracts extends Script {
+
+    constructor() {
+        super("Generates many pmtiles-archive from planet-latest.pmtiles. Must be started from the directory where planet-latest.pmtiles resides, archives will be created next to it")
+    }
+
+    private generateArchive(z: number, x: number, y: number, maxzoom?: number): Promise<void> {
+        const [[max_lat, min_lon], [min_lat, max_lon]] = Tiles.tile_bounds(z, x, y)
+        let maxzoomflag = ""
+        if(maxzoom !== undefined){
+            maxzoomflag = " --maxzoom="+maxzoom
+        }
+        return startProcess(`./pmtiles extract planet-latest.pmtiles --download-threads=1 --minzoom=${z}${maxzoomflag} --bbox=${[min_lon, min_lat + 0.0001, max_lon, max_lat].join(",")} ${z}-${x}-${y}.pmtiles`)
+    }
+
+    private* generateField(z: number, maxzoom?: number): Generator<Promise<void>> {
+        const boundary = 2 << z
+        for (let x = 0; x < boundary; x++) {
+            for (let y = 0; y < boundary; y++) {
+                yield this.generateArchive(z, x, y, maxzoom)
+            }
+        }
+    }
+
+    private* generateAll(): Generator<Promise<void>> {
+        const zoomlevels: Record<number, number> = OfflineBasemapManager.zoomelevels
+        for (const key in zoomlevels) {
+            const minzoom: number = Number(key)
+            const maxzoom: number | undefined = zoomlevels[key]
+            for (const promise of this.generateField(minzoom, maxzoom)) {
+                yield promise
+            }
+        }
+    }
+
+    createBatch<T>(generator: Generator<T>, length: number): T[] {
+        const batch = []
+        do {
+            const next = generator.next()
+            if (next.done) {
+                return batch
+            }
+            batch.push(next.value)
+        } while (batch.length < length)
+        return batch
+    }
+
+    async main(): Promise<void> {
+        const numberOfThreads = 512
+        const generator = this.generateAll()
+        let batch: Promise<void>[] = []
+        do {
+            batch = this.createBatch(generator, numberOfThreads)
+            await Promise.all(batch)
+        } while (batch.length > 0)
+    }
+
+}
+
+new GeneratePmTilesExtracts().run()