Scripts(offline): add offline generation script

scripts/generatePmTilesExtractionScript.ts

@@ -1,39 +0,0 @@
-import Script from "./Script"
-import { Tiles } from "../src/Models/TileRange"
-
-class GeneratePmTilesExtractionScript extends Script {
-
-    constructor() {
-        super("Generates a bash script to extract all subpyramids of maxzoom=8 from planet-latest.pmtiles")
-    }
-
-    private emitRange(z: number, x: number, y: number, maxzoom?: number): string {
-        const [[max_lat, min_lon], [min_lat, max_lon]] = Tiles.tile_bounds(z, x, y)
-        let maxzoomflag = ""
-        if(maxzoom !== undefined){
-            maxzoomflag = " --maxzoom="+maxzoom
-        }
-        return (`./pmtiles extract planet-latest.pmtiles --minzoom=${z}${maxzoomflag} --bbox=${[min_lon, min_lat + 0.0001, max_lon, max_lat].join(",")} ${z}-${x}-${y}.pmtiles`)
-    }
-
-    private generateField(z: number, maxzoom?:number){
-        const boundary = 2 << z
-        for (let x = 0; x < boundary; x++) {
-            const xCommands = []
-            for (let y = 0; y < boundary; y++) {
-                xCommands.push(this.emitRange(z, x, y,maxzoom))
-            }
-            console.log(xCommands.join(" && ") + " && echo 'All pyramids for x = " + x + " are generated' & ")
-        }
-
-    }
-
-    async main(): Promise<void> {
-        this.generateField(0, 4)
-        this.generateField(5, 8)
-        this.generateField(9)
-    }
-
-}
-
-new GeneratePmTilesExtractionScript().run()

scripts/generatePmTilesExtracts.ts

@@ -0,0 +1,80 @@
+import Script from "./Script"
+import { Tiles } from "../src/Models/TileRange"
+import { OfflineBasemapManager } from "../src/service-worker/OfflineBasemapManager"
+
+import { spawn } from "child_process"
+
+function startProcess(script: string): Promise<void> {
+    return new Promise((resolve, reject) => {
+        const child = spawn("node", [script], { stdio: "inherit" })
+
+        child.on("close", (code) => {
+            if (code === 0) resolve()
+            else reject(new Error(`Process exited with code ${code}`))
+        })
+
+        child.on("error", reject)
+    })
+}
+
+
+class GeneratePmTilesExtracts extends Script {
+
+    constructor() {
+        super("Generates many pmtiles-archive from planet-latest.pmtiles. Must be started from the directory where planet-latest.pmtiles resides, archives will be created next to it")
+    }
+
+    private generateArchive(z: number, x: number, y: number, maxzoom?: number): Promise<void> {
+        const [[max_lat, min_lon], [min_lat, max_lon]] = Tiles.tile_bounds(z, x, y)
+        let maxzoomflag = ""
+        if(maxzoom !== undefined){
+            maxzoomflag = " --maxzoom="+maxzoom
+        }
+        return startProcess(`./pmtiles extract planet-latest.pmtiles --download-threads=1 --minzoom=${z}${maxzoomflag} --bbox=${[min_lon, min_lat + 0.0001, max_lon, max_lat].join(",")} ${z}-${x}-${y}.pmtiles`)
+    }
+
+    private* generateField(z: number, maxzoom?: number): Generator<Promise<void>> {
+        const boundary = 2 << z
+        for (let x = 0; x < boundary; x++) {
+            for (let y = 0; y < boundary; y++) {
+                yield this.generateArchive(z, x, y, maxzoom)
+            }
+        }
+    }
+
+    private* generateAll(): Generator<Promise<void>> {
+        const zoomlevels: Record<number, number> = OfflineBasemapManager.zoomelevels
+        for (const key in zoomlevels) {
+            const minzoom: number = Number(key)
+            const maxzoom: number | undefined = zoomlevels[key]
+            for (const promise of this.generateField(minzoom, maxzoom)) {
+                yield promise
+            }
+        }
+    }
+
+    createBatch<T>(generator: Generator<T>, length: number): T[] {
+        const batch = []
+        do {
+            const next = generator.next()
+            if (next.done) {
+                return batch
+            }
+            batch.push(next.value)
+        } while (batch.length < length)
+        return batch
+    }
+
+    async main(): Promise<void> {
+        const numberOfThreads = 512
+        const generator = this.generateAll()
+        let batch: Promise<void>[] = []
+        do {
+            batch = this.createBatch(generator, numberOfThreads)
+            await Promise.all(batch)
+        } while (batch.length > 0)
+    }
+
+}
+
+new GeneratePmTilesExtracts().run()

src/Models/TileRange.ts

@@ -82,7 +82,7 @@ export class Tiles {
     }
 
     static asGeojson(index: number): Feature<Polygon>
-    static asGeojson(x: number, y: number, z: number): Feature<Polygon>
+    static asGeojson(z: number, x: number, y: number): Feature<Polygon>
     static asGeojson(zIndex: number, x?: number, y?: number): Feature<Polygon> {
         let z = zIndex
         if (x === undefined) {

src/service-worker/OfflineBasemapManager.ts

@@ -1,7 +1,7 @@
 import { PMTiles, RangeResponse, Source } from "pmtiles"
 
 
-interface AreaDescription {
+export interface AreaDescription {
     /**
      * Thie filename at the host and in the indexedDb
      * Host name is not included
@@ -26,13 +26,17 @@ interface AreaDescription {
      */
     dataVersion?: string
 
+    /**
+     * Blob.size
+     */
+    size?: number
+
 }
 
 class TypedIdb<T> {
     private readonly _db: Promise<IDBDatabase>
     private readonly _name: string
 
-
     constructor(db: string) {
         this._name = db
         this._db = TypedIdb.openDb(db)
@@ -102,6 +106,20 @@ class TypedIdb<T> {
             request.onerror = () => reject(request.error)
         })
     }
+
+    async del(key: string): Promise<void> {
+        const db = await this._db
+        return new Promise((resolve, reject) => {
+            const tx = db.transaction([this._name], "readwrite")
+            const store = tx.objectStore(this._name)
+            const request = store.delete(key)
+
+            request.onsuccess = () => resolve()
+            request.onerror = () => reject(request.error)
+        })
+
+
+    }
 }
 
 class BlobSource implements Source {
@@ -145,6 +163,13 @@ export class OfflineBasemapManager {
      */
     private readonly _host: string
 
+    public static readonly zoomelevels = {
+        0: 4,
+        5: 7,
+        8: 9,
+        10: undefined
+    }
+
     private readonly blobs: TypedIdb<any>
     private readonly meta: TypedIdb<AreaDescription>
     private metaCached: AreaDescription[] = []
@@ -171,8 +196,9 @@ export class OfflineBasemapManager {
         this.meta = new TypedIdb<AreaDescription>("OfflineBasemapMeta")
     }
 
-    private async updateCachedMeta() {
+    public async updateCachedMeta(): Promise<AreaDescription[]> {
         this.metaCached = await this.meta.getAllValues()
+        return this.metaCached
     }
 
     /**
@@ -181,12 +207,15 @@ export class OfflineBasemapManager {
      */
     public async installArea(areaDescription: AreaDescription) {
         const target = this._host + areaDescription.name
-        console.log(">>><<< installing area from "+target)
+        console.log("Installing area from " + target)
         const response = await fetch(target)
-
+        if (!response.ok) {
+            return
+        }
         const blob = await response.blob()
         await this.blobs.set(areaDescription.name, blob)
         areaDescription.dataVersion = await new BlobSource(areaDescription.name, blob).getDataVersion()
+        areaDescription.size = blob.size
         await this.meta.set(areaDescription.name, areaDescription)
         await this.updateCachedMeta()
     }
@@ -254,4 +283,11 @@ export class OfflineBasemapManager {
             }
         )
     }
+
+    deleteArea(description: AreaDescription): Promise<AreaDescription[]> {
+        this.blobs.del(description.name)
+        this.meta.del(description.name)
+
+        return this.updateCachedMeta()
+    }
 }