Feature: offline basemaps: first working version (without ability to install basemaps from the UI)

src/InstallServiceWorker.ts

@@ -5,7 +5,7 @@ window.addEventListener("load", async () => {
         return
     }
     try {
-        await navigator.serviceWorker.register("/service-worker.js")
+        await navigator.serviceWorker.register("/service-worker.js", { type: "module" })
         console.log("Service worker registration successful")
     } catch (err) {
         console.error("Service worker registration failed", err)

src/UI/Test.svelte

@@ -1,63 +1,4 @@
 <script lang="ts">
-  import { PMTiles } from "pmtiles"
-
-  // What piece of the map to download and cache locally for the current map view?
-  const gentBBox = {
-    "type": "Feature",
-    "properties": {},
-    "geometry": {
-      "coordinates": [
-        [
-          [
-            3.6911333519349228,
-            51.069673467969096
-          ],
-          [
-            3.6911333519349228,
-            51.02886180987505
-          ],
-          [
-            3.773873676856965,
-            51.02886180987505
-          ],
-          [
-            3.773873676856965,
-            51.069673467969096
-          ],
-          [
-            3.6911333519349228,
-            51.069673467969096
-          ]
-        ]
-      ],
-      "type": "Polygon"
-    }
-  }
-
-  // URL or fetch source of the PMTiles archive
-  let source = new PMTiles("https://cache.mapcomplete.org/planet-latest.pmtiles")
-
-  async function test(z: number, x: number, y: number) {
-
-
-    const meta = await source.getMetadata()
-    console.log(meta)
-    const minzoom = meta["minzoom"]
-    const maxzoom = meta["maxzoom"]
-    console.log("Range is", minzoom, maxzoom)
-    for (const l of meta["vector_layers"]) {
-      //    console.log(l)
-    }
-//    const header = await source.getHeader()
-//    console.log(header)
-    let resp = await source.getZxy(z, x, y)
-    console.log(resp)
-//    let tileData = resp?.data
-    // do something with tileData
-  }
-
-  test(14, 8338, 5469) // Elf-julistraat Brugge
-  test(15, 12233, 10534)
 </script>
 
 Nothing to test

src/service-worker.ts

@@ -1,84 +0,0 @@
-const version = "0.0.0"
-
-interface ServiceWorkerFetchEvent extends Event {
-    request: RequestInfo & { url: string }
-    respondWith: (response: any | PromiseLike<Response>) => Promise<void>
-}
-
-async function install() {
-    console.log("Installing service worker!")
-}
-
-addEventListener("install", (e) => (<any>e).waitUntil(install()))
-addEventListener("activate", (e) => (<any>e).waitUntil(activate()))
-
-async function clearCaches(exceptVersion = undefined) {
-    const keys = await caches.keys()
-    await Promise.all(keys.map((k) => k !== version && caches.delete(k)))
-    console.log("Cleared caches")
-}
-
-async function activate() {
-    console.log("Activating service worker")
-    await clearCaches(version)
-}
-
-async function fetchAndCache(event: ServiceWorkerFetchEvent): Promise<Response> {
-    const networkResponse = await fetch(event.request)
-    const cache = await caches.open(version)
-    await cache.put(event.request, networkResponse.clone())
-    console.log("Cached", event.request)
-    return networkResponse
-}
-
-async function cacheFirst(event: ServiceWorkerFetchEvent, attemptUpdate: boolean = false) {
-    const cacheResponse = await caches.match(event.request, { ignoreSearch: true })
-    if (cacheResponse === undefined) {
-        return fetchAndCache(event)
-    }
-    console.debug("Loaded from cache: ", event.request)
-    if (attemptUpdate) {
-        fetchAndCache(event)
-    }
-    return cacheResponse
-}
-
-const neverCache: RegExp[] = [/\.html$/, /service-worker/]
-const neverCacheHost: RegExp[] = [/127\.0\.0\.[0-9]+/, /\.local/, /\.gitpod\.io/, /localhost/]
-
-async function handleRequest(event: ServiceWorkerFetchEvent) {
-    const origin = new URL(self.origin)
-    const requestUrl = new URL(event.request.url)
-    if (requestUrl.pathname.endsWith("service-worker-version")) {
-        console.log("Sending version number...")
-        await event.respondWith(new Response(JSON.stringify({ "service-worker-version": version })))
-        return
-    }
-    if (requestUrl.pathname.endsWith("/service-worker-clear")) {
-        await clearCaches()
-        await event.respondWith(new Response(JSON.stringify({ "cache-cleared": true })))
-        return
-    }
-
-    const shouldBeCached =
-        origin.host === requestUrl.host &&
-        !neverCacheHost.some((blacklisted) => origin.host.match(blacklisted)) &&
-        !neverCache.some((blacklisted) => event.request.url.match(blacklisted))
-    if (!shouldBeCached) {
-        console.debug("Not intercepting ", requestUrl.toString(), origin.host, requestUrl.host)
-        // We return _without_ calling event.respondWith, which signals the browser that it'll have to handle it himself
-        return
-    }
-    await event.respondWith(await cacheFirst(event))
-}
-
-self.addEventListener("fetch", async (e) => {
-    // Important: this lambda must run synchronously, as the browser will otherwise handle the request
-    const event: ServiceWorkerFetchEvent = <ServiceWorkerFetchEvent>e
-    try {
-        await handleRequest(event)
-    } catch (e) {
-        console.error("CRASH IN SW:", e)
-        await event.respondWith(fetch(event.request.url))
-    }
-})

src/service-worker/OfflineBasemapManager.ts

@@ -0,0 +1,257 @@
+import { PMTiles, RangeResponse, Source } from "pmtiles"
+
+
+interface AreaDescription {
+    /**
+     * Thie filename at the host and in the indexedDb
+     * Host name is not included
+     */
+    name: string
+    /**
+     * Minzoom that is covered, inclusive
+     */
+    minzoom: number,
+    /**
+     * Maxzoom that is covered, inclusive
+     */
+    maxzoom: number,
+    /**
+     * The x, y of the tile that is covered (at minzoom)
+     */
+    x: number,
+    y: number,
+
+    /**
+     * ISO-datestring of when the data was processed
+     */
+    dataVersion?: string
+
+}
+
+class TypedIdb<T> {
+    private readonly _db: Promise<IDBDatabase>
+    private readonly _name: string
+
+
+    constructor(db: string) {
+        this._name = db
+        this._db = TypedIdb.openDb(db)
+    }
+
+    private static openDb(name: string): Promise<IDBDatabase> {
+        return new Promise((resolve, reject) => {
+
+            const request: IDBOpenDBRequest = indexedDB.open(name)
+            request.onerror = (event) => {
+                console.error("Could not open the Database: ", event)
+                reject(event)
+            }
+            request.onupgradeneeded = () => {
+                const db = request.result
+                db.createObjectStore(name)
+            }
+            request.onsuccess = () => {
+                resolve(request.result)
+            }
+        })
+    }
+
+
+    async set(key: string, value: T): Promise<void> {
+        const db = await this._db
+        return new Promise((resolve, reject) => {
+            const tx = db.transaction([this._name], "readwrite")
+            const store = tx.objectStore(this._name)
+            const request = store.put(value, key)
+
+            request.onsuccess = () => resolve()
+            request.onerror = () => reject(request.error)
+        })
+    }
+
+    async get(key: string): Promise<T> {
+        const db = await this._db
+        return new Promise((resolve, reject) => {
+            const tx = db.transaction([this._name], "readonly")
+            const store = tx.objectStore(this._name)
+            const request: IDBRequest<T> = store.get(key)
+
+            request.onsuccess = () => resolve(request.result)
+            request.onerror = () => reject(request.error)
+        })
+    }
+
+    async getAllValues(): Promise<T[]> {
+        const db = await this._db
+        return new Promise((resolve, reject) => {
+            const tx = db.transaction([this._name], "readonly")
+            const store = tx.objectStore(this._name)
+            const values: T[] = []
+            const request = store.openCursor()
+
+            request.onsuccess = () => {
+                const cursor = request.result
+                if (cursor) {
+                    values.push(cursor.value)
+                    cursor.continue() // Triggers a new 'onsuccess' event
+                } else {
+                    resolve(values)
+                }
+            }
+
+            request.onerror = () => reject(request.error)
+        })
+    }
+}
+
+class BlobSource implements Source {
+    private readonly _blob: Blob
+    private readonly _name: string
+    public readonly pmtiles: PMTiles
+
+    constructor(name: string, blob: Blob) {
+        this._name = name
+        this._blob = blob
+        this.pmtiles = new PMTiles(this)
+    }
+
+    async getBytes(offset: number, length: number): Promise<RangeResponse> {
+        const sliced = this._blob.slice(offset, offset + length)
+        return {
+            data: await sliced.arrayBuffer()
+        }
+    }
+
+    getKey() {
+        return this._name
+    }
+
+    /**
+     * The underlying data was created with an extract from OpenStreetMap,
+     * extracted at a certain date. This date can be used as version indicator.
+     */
+    async getDataVersion(): Promise<string> {
+        const meta: Record<string, string> = <any>await this.pmtiles.getMetadata()
+        return meta["planetiler:osm:osmosisreplicationtime"]
+
+    }
+
+}
+
+export class OfflineBasemapManager {
+    /**
+     * Where to get the initial map tiles
+     * @private
+     */
+    private readonly _host: string
+
+    private readonly blobs: TypedIdb<any>
+    private readonly meta: TypedIdb<AreaDescription>
+    private metaCached: AreaDescription[] = []
+
+    /**
+     * The 'offline base map manager' is responsible for keeping track of the locally installed 'protomaps' subpyramids.
+     *
+     * Roughly taken, a 'protomaps pyramid' is a collection of MVT-tiles in a single file, all belonging to a certain bbox.
+     * The 'cache.mapcomplete.org' server has a collection of sliced pyramids:
+     *
+     * global-basemap.pmtiles: all tiles from z=0 to z=4, ~15MB
+     * 5-{x}-{y}.pmtiles: zoom levels 5 'till 8 (up to 9MB)
+     * 9-{x}-{y}.pmtimes: zoom levels 8 till 15 for a region, up to 90MB
+     *
+     * When a user downloads an offline map, they download a 9-* subpyramid, the corresponding 5-8 pyramid and the 'global-basemap'
+     *
+     */
+    public constructor(host: string) {
+        if (!host.endsWith("/")) {
+            host += "/"
+        }
+        this._host = host
+        this.blobs = new TypedIdb("OfflineBasemap")
+        this.meta = new TypedIdb<AreaDescription>("OfflineBasemapMeta")
+    }
+
+    private async updateCachedMeta() {
+        this.metaCached = await this.meta.getAllValues()
+    }
+
+    /**
+     * ! Must be called from a fetch event in the service worker !
+     * @param areaDescription
+     */
+    public async installArea(areaDescription: AreaDescription) {
+        const target = this._host + areaDescription.name
+        console.log(">>><<< installing area from "+target)
+        const response = await fetch(target)
+
+        const blob = await response.blob()
+        await this.blobs.set(areaDescription.name, blob)
+        areaDescription.dataVersion = await new BlobSource(areaDescription.name, blob).getDataVersion()
+        await this.meta.set(areaDescription.name, areaDescription)
+        await this.updateCachedMeta()
+    }
+
+    /**
+     * Knows about the cache structure of cache.mapcomplete.org
+     * @see GeneratePmTilesExtractionScript
+     */
+    public static getAreaDescriptionForMapcomplete(name: string): AreaDescription {
+
+        if (!name.endsWith(".pmtiles")) {
+            throw "Invalid filename, should end with .pmtiles"
+        }
+        const [z, x, y] = name.substring(0, name.length - ".pmtiles".length).split("-").map(Number)
+        const maxzooms: Record<number, number> = { 0: 4, 5: 8, 9: 15 }
+        return {
+            name,
+            minzoom: z,
+            maxzoom: maxzooms[z],
+            x, y
+        }
+    }
+
+    getMeta(): AreaDescription[] {
+        return this.metaCached
+    }
+
+    private determineArea(z: number, x: number, y: number): AreaDescription | undefined {
+        for (const areaDescription of this.metaCached) {
+            if (areaDescription.minzoom > z) {
+                continue
+            }
+            if (areaDescription.maxzoom < z) {
+                continue
+            }
+            const zdiff = z - areaDescription.minzoom
+            const xAtMinz = x >> zdiff
+            const yAtMinZ = y >> zdiff
+            if (xAtMinz === areaDescription.x && yAtMinZ === areaDescription.y) {
+                return areaDescription
+            }
+        }
+        return undefined
+    }
+
+    async getTileResponse(z: number, x: number, y: number): Promise<Response> {
+        if (this.metaCached.length === 0) {
+            await this.updateCachedMeta()
+        }
+        const area = this.determineArea(z, x, y)
+        if (!area) {
+            return new Response("Not found", { status: 404 })
+        }
+        const blob = await this.blobs.get(area.name)
+        const pmtiles = new BlobSource(area.name, blob)
+        const tileData = await pmtiles.pmtiles.getZxy(z, x, y)
+        if (!tileData) {
+            return new Response("Not found (not in tile archive, should not happen)", { status: 404 })
+        }
+
+        return new Response(
+            tileData.data,
+            {
+                    headers: { 'Content-Type': 'application/x.protobuf' }
+            }
+        )
+    }
+}

src/service-worker/index.ts

@@ -0,0 +1,47 @@
+// This file must have worker types, but not DOM types.
+// The global should be that of a service worker.
+
+// This fixes `self`'s type.
+declare var self: ServiceWorkerGlobalScope
+export {}
+import { OfflineBasemapManager } from "./OfflineBasemapManager"
+
+const offlinemaps = new OfflineBasemapManager("https://cache.mapcomplete.org/")
+
+
+function routeOffline(event: FetchEvent) {
+    const url = event.request.url
+    const rest = url.split("/service-worker/offline-basemap/")[1]
+    if (rest.indexOf("install") >= 0) {
+        const filename = url.split("/").pop()
+        if (!filename) {
+            return
+        }
+        const description = OfflineBasemapManager.getAreaDescriptionForMapcomplete(filename)
+        event.respondWith(
+            offlinemaps.installArea(description).then(
+                () => {
+                    return new Response(
+                        JSON.stringify({
+                            "status": "installed",
+                            installed_previously: offlinemaps.getMeta()
+                        }), { headers: { "Content-Type": "application/json" } })
+                })
+        )
+        return
+    }
+    const tileRegex =/(\d+-\d+-\d+).mvt$/
+    const tileMatch = rest.match(tileRegex)
+    if (tileMatch) {
+        const [z, x, y] = tileMatch[1].split("-").map(Number)
+        event.respondWith(offlinemaps.getTileResponse(z, x, y))
+    }
+}
+
+self.addEventListener("fetch", (event) => {
+    const url = event.request.url
+    if (url.indexOf("/service-worker/offline-basemap/") >= 0) {
+        routeOffline(event)
+
+    }
+})

src/service-worker/rollup.config.js

@@ -0,0 +1,12 @@
+import typescript from "@rollup/plugin-typescript"
+import resolve from "@rollup/plugin-node-resolve"
+import commonjs from "@rollup/plugin-commonjs"
+
+export default {
+  input: "index.ts",
+  output: {
+    file: "../../public/service-worker.js",
+    format: "es"
+  },
+  plugins: [resolve(), commonjs(), typescript()]
+}

src/service-worker/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "esnext",
+    "module": "esnext",
+    "strict": true,
+    "moduleResolution": "node",
+    "rootDir": ".",
+    "outDir": "../../public/service-worker/",
+    "composite": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "resolveJsonModule": true,
+    "lib": [
+      "esnext",
+      "webworker"
+    ],
+    "types": ["pmtiles"]
+  },
+  "include": ["index.ts", "OfflineBasemapManager.ts"]
+}