diff --git a/scripts/generatePmTilesExtractionScript.ts b/scripts/generatePmTilesExtractionScript.ts deleted file mode 100644 index 5184186700..0000000000 --- a/scripts/generatePmTilesExtractionScript.ts +++ /dev/null @@ -1,39 +0,0 @@ -import Script from "./Script" -import { Tiles } from "../src/Models/TileRange" - -class GeneratePmTilesExtractionScript extends Script { - - constructor() { - super("Generates a bash script to extract all subpyramids of maxzoom=8 from planet-latest.pmtiles") - } - - private emitRange(z: number, x: number, y: number, maxzoom?: number): string { - const [[max_lat, min_lon], [min_lat, max_lon]] = Tiles.tile_bounds(z, x, y) - let maxzoomflag = "" - if(maxzoom !== undefined){ - maxzoomflag = " --maxzoom="+maxzoom - } - return (`./pmtiles extract planet-latest.pmtiles --minzoom=${z}${maxzoomflag} --bbox=${[min_lon, min_lat + 0.0001, max_lon, max_lat].join(",")} ${z}-${x}-${y}.pmtiles`) - } - - private generateField(z: number, maxzoom?:number){ - const boundary = 2 << z - for (let x = 0; x < boundary; x++) { - const xCommands = [] - for (let y = 0; y < boundary; y++) { - xCommands.push(this.emitRange(z, x, y,maxzoom)) - } - console.log(xCommands.join(" && ") + " && echo 'All pyramids for x = " + x + " are generated' & ") - } - - } - - async main(): Promise { - this.generateField(0, 4) - this.generateField(5, 8) - this.generateField(9) - } - -} - -new GeneratePmTilesExtractionScript().run() diff --git a/scripts/generatePmTilesExtracts.ts b/scripts/generatePmTilesExtracts.ts new file mode 100644 index 0000000000..cb25db1145 --- /dev/null +++ b/scripts/generatePmTilesExtracts.ts @@ -0,0 +1,80 @@ +import Script from "./Script" +import { Tiles } from "../src/Models/TileRange" +import { OfflineBasemapManager } from "../src/service-worker/OfflineBasemapManager" + +import { spawn } from "child_process" + +function startProcess(script: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn("node", [script], { stdio: "inherit" }) + + child.on("close", (code) => { + if (code === 0) resolve() + else reject(new Error(`Process exited with code ${code}`)) + }) + + child.on("error", reject) + }) +} + + +class GeneratePmTilesExtracts extends Script { + + constructor() { + super("Generates many pmtiles-archive from planet-latest.pmtiles. Must be started from the directory where planet-latest.pmtiles resides, archives will be created next to it") + } + + private generateArchive(z: number, x: number, y: number, maxzoom?: number): Promise { + const [[max_lat, min_lon], [min_lat, max_lon]] = Tiles.tile_bounds(z, x, y) + let maxzoomflag = "" + if(maxzoom !== undefined){ + maxzoomflag = " --maxzoom="+maxzoom + } + return startProcess(`./pmtiles extract planet-latest.pmtiles --download-threads=1 --minzoom=${z}${maxzoomflag} --bbox=${[min_lon, min_lat + 0.0001, max_lon, max_lat].join(",")} ${z}-${x}-${y}.pmtiles`) + } + + private* generateField(z: number, maxzoom?: number): Generator> { + const boundary = 2 << z + for (let x = 0; x < boundary; x++) { + for (let y = 0; y < boundary; y++) { + yield this.generateArchive(z, x, y, maxzoom) + } + } + } + + private* generateAll(): Generator> { + const zoomlevels: Record = OfflineBasemapManager.zoomelevels + for (const key in zoomlevels) { + const minzoom: number = Number(key) + const maxzoom: number | undefined = zoomlevels[key] + for (const promise of this.generateField(minzoom, maxzoom)) { + yield promise + } + } + } + + createBatch(generator: Generator, length: number): T[] { + const batch = [] + do { + const next = generator.next() + if (next.done) { + return batch + } + batch.push(next.value) + } while (batch.length < length) + return batch + } + + async main(): Promise { + const numberOfThreads = 512 + const generator = this.generateAll() + let batch: Promise[] = [] + do { + batch = this.createBatch(generator, numberOfThreads) + await Promise.all(batch) + } while (batch.length > 0) + } + +} + +new GeneratePmTilesExtracts().run() diff --git a/src/Models/TileRange.ts b/src/Models/TileRange.ts index d4494da6df..7b0b01b39c 100644 --- a/src/Models/TileRange.ts +++ b/src/Models/TileRange.ts @@ -82,7 +82,7 @@ export class Tiles { } static asGeojson(index: number): Feature - static asGeojson(x: number, y: number, z: number): Feature + static asGeojson(z: number, x: number, y: number): Feature static asGeojson(zIndex: number, x?: number, y?: number): Feature { let z = zIndex if (x === undefined) { diff --git a/src/service-worker/OfflineBasemapManager.ts b/src/service-worker/OfflineBasemapManager.ts index 2805b21e0f..18e3d542ad 100644 --- a/src/service-worker/OfflineBasemapManager.ts +++ b/src/service-worker/OfflineBasemapManager.ts @@ -1,7 +1,7 @@ import { PMTiles, RangeResponse, Source } from "pmtiles" -interface AreaDescription { +export interface AreaDescription { /** * Thie filename at the host and in the indexedDb * Host name is not included @@ -26,13 +26,17 @@ interface AreaDescription { */ dataVersion?: string + /** + * Blob.size + */ + size?: number + } class TypedIdb { private readonly _db: Promise private readonly _name: string - constructor(db: string) { this._name = db this._db = TypedIdb.openDb(db) @@ -102,6 +106,20 @@ class TypedIdb { request.onerror = () => reject(request.error) }) } + + async del(key: string): Promise { + const db = await this._db + return new Promise((resolve, reject) => { + const tx = db.transaction([this._name], "readwrite") + const store = tx.objectStore(this._name) + const request = store.delete(key) + + request.onsuccess = () => resolve() + request.onerror = () => reject(request.error) + }) + + + } } class BlobSource implements Source { @@ -145,6 +163,13 @@ export class OfflineBasemapManager { */ private readonly _host: string + public static readonly zoomelevels = { + 0: 4, + 5: 7, + 8: 9, + 10: undefined + } + private readonly blobs: TypedIdb private readonly meta: TypedIdb private metaCached: AreaDescription[] = [] @@ -171,8 +196,9 @@ export class OfflineBasemapManager { this.meta = new TypedIdb("OfflineBasemapMeta") } - private async updateCachedMeta() { + public async updateCachedMeta(): Promise { this.metaCached = await this.meta.getAllValues() + return this.metaCached } /** @@ -181,12 +207,15 @@ export class OfflineBasemapManager { */ public async installArea(areaDescription: AreaDescription) { const target = this._host + areaDescription.name - console.log(">>><<< installing area from "+target) + console.log("Installing area from " + target) const response = await fetch(target) - + if (!response.ok) { + return + } const blob = await response.blob() await this.blobs.set(areaDescription.name, blob) areaDescription.dataVersion = await new BlobSource(areaDescription.name, blob).getDataVersion() + areaDescription.size = blob.size await this.meta.set(areaDescription.name, areaDescription) await this.updateCachedMeta() } @@ -254,4 +283,11 @@ export class OfflineBasemapManager { } ) } + + deleteArea(description: AreaDescription): Promise { + this.blobs.del(description.name) + this.meta.del(description.name) + + return this.updateCachedMeta() + } }