Feature(offline): add management module for offline basemaps

2025-07-31 12:30:09 +02:00 · 2025-07-31 12:30:09 +02:00 · 2dd0240ce8
commit 2dd0240ce8
parent 31eb9b5587
9 changed files with 171 additions and 41 deletions
--- a/.gitignore
+++ b/.gitignore
@ -10,6 +10,7 @@ assets/editor-layer-index.json
 assets/generated/*
 assets/layers/favourite/favourite.json
 public/*.webmanifest
+public/service-worker/tsconfig.tsbuildinfo
 /*.html
 !/index.html
 !/customGenerator.html
--- a/public/css/index-tailwind-output.css
+++ b/public/css/index-tailwind-output.css
@ -1770,6 +1770,10 @@ input[type="range"].range-lg::-moz-range-thumb {
  height: 0.875rem;
 }

+.h-3\/4 {
+  height: 75%;
+}
+
 .h-32 {
  height: 8rem;
 }
--- a/src/Models/MenuState.ts
+++ b/src/Models/MenuState.ts
@ -26,6 +26,7 @@ export class MenuState {
        "favourites",
        "filter",
        "hotkeys",
+        "manageOffline",
        "menu",
        "privacy",
        "search",
--- a/src/UI/BigComponents/MenuDrawer.svelte
+++ b/src/UI/BigComponents/MenuDrawer.svelte
@ -18,7 +18,7 @@
  import RasterLayerOverview from "../Map/RasterLayerOverview.svelte"
  import ThemeIntroPanel from "./ThemeIntroPanel.svelte"
  import Marker from "../Map/Marker.svelte"
-  import { ShareIcon } from "@babeard/svelte-heroicons/mini"
+  import { BoltIcon, ShareIcon } from "@babeard/svelte-heroicons/mini"
  import SidebarUnit from "../Base/SidebarUnit.svelte"
  import PanoramaxLink from "./PanoramaxLink.svelte"
  import { UIEventSource } from "../../Logic/UIEventSource"
@ -27,6 +27,7 @@
  import Hotkeys from "../Base/Hotkeys"
  import MenuDrawerIndex from "./MenuDrawerIndex.svelte"
  import ThemeViewState from "../../Models/ThemeViewState"
+  import HotkeyTable from "./HotkeyTable.svelte"

  export let onlyLink: boolean
  export let state: ThemeViewState
@ -49,6 +50,8 @@
      })
    }
  })
+  let hotkeys = Hotkeys._docs
+
 </script>

 <div class:h-0={!onlyLink} class:h-full={onlyLink} class="overflow-hidden">
@ -111,6 +114,18 @@
              />
            </a>

+            {#if $hotkeys.length > 0}
+              <div class="hidden-on-mobile w-full">
+                <Page {onlyLink} shown={pg.hotkeys}>
+                  <svelte:fragment slot="header">
+                    <BoltIcon />
+                    <Tr t={Translations.t.hotkeyDocumentation.title} />
+                  </svelte:fragment>
+                  <HotkeyTable />
+                </Page>
+              </div>
+            {/if}
+
            <a class="flex" href={Utils.OsmChaLinkFor(31, theme.id)} target="_blank">
              <QueueList class="h-6 w-6" />
              <Tr t={Translations.t.general.attribution.openOsmcha.Subs({ theme: theme.title })} />
--- a/src/UI/BigComponents/MenuDrawerIndex.svelte
+++ b/src/UI/BigComponents/MenuDrawerIndex.svelte
@ -16,7 +16,6 @@
  import Tr from "../Base/Tr.svelte"
  import LoginToggle from "../Base/LoginToggle.svelte"
  import { CloseButton } from "flowbite-svelte"
-  import HotkeyTable from "./HotkeyTable.svelte"
  import { Utils } from "../../Utils"
  import Constants from "../../Models/Constants"
  import Mastodon from "../../assets/svg/Mastodon.svelte"
@ -61,6 +60,8 @@
  import QueuedImagesView from "../Image/QueuedImagesView.svelte"
  import InsetSpacer from "../Base/InsetSpacer.svelte"
  import UserCircle from "@rgossiaux/svelte-heroicons/solid/UserCircle"
+  import OfflineManagement from "./OfflineManagement.svelte"
+  import { GlobeEuropeAfrica } from "@babeard/svelte-heroicons/solid/GlobeEuropeAfrica"

  export let state: {
    favourites: FavouritesFeatureSource
@ -71,7 +72,6 @@
    mapProperties?: MapProperties
    userRelatedState?: UserRelatedState
  }
-  let hotkeys = Hotkeys._docs
  let userdetails = state.osmConnection.userDetails

  let usersettingslayer = new LayerConfig(<LayerConfigJson>usersettings, "usersettings", true)
@ -134,13 +134,17 @@
    <SidebarUnit>
      <LoginToggle {state}>
        <LoginButton osmConnection={state.osmConnection} slot="not-logged-in" />
-        <div class="flex items-center gap-x-4 w-fit m-2">
+        <div class="flex items-center gap-x-4 w-full m-2">
          {#if $userdetails.img}
            <img alt="avatar" src={$userdetails.img} class="h-12 w-12 rounded-full" />
            {:else}
            <UserCircle class="h-14 w-14" color="gray"/>
          {/if}
-          <b>{$userdetails.name}</b>
+          <div class="flex flex-col w-full gap-y-2">
+
+            <b>{$userdetails.name}</b>
+            <LogoutButton clss="as-link small subtle text-sm" osmConnection={state.osmConnection} />
+          </div>
        </div>
      </LoginToggle>

@ -194,22 +198,16 @@
            </div>
          </Page>
        {/if}
-        {#if $hotkeys.length > 0}
-          <div class="hidden-on-mobile w-full">
-            <Page {onlyLink} shown={pg.hotkeys}>
-              <svelte:fragment slot="header">
-                <BoltIcon />
-                <Tr t={Translations.t.hotkeyDocumentation.title} />
-              </svelte:fragment>
-              <HotkeyTable />
-            </Page>
-          </div>
-        {/if}
-
-        <div class="self-end">
-          <LogoutButton osmConnection={state.osmConnection} />
-        </div>
      </LoginToggle>
+
+      <Page {onlyLink} shown={pg.manageOffline} fullscreen>
+        <svelte:fragment slot="header">
+          <GlobeEuropeAfrica />
+          Manage offline basemap
+        </svelte:fragment>
+        <OfflineManagement {state} />
+      </Page>
+
      <LanguagePicker
        preferredLanguages={state.userRelatedState.osmConnection.userDetails.mapD(
          (ud) => ud.languages
@ -268,6 +266,7 @@
        <Forgejo class="h-6 w-6" />
        <Tr t={Translations.t.general.attribution.gotoSourceCode} />
      </a>
+
      <a class="flex" href={`${Constants.weblate}projects/mapcomplete/`} target="_blank">
        <TranslateIcon class="h-6 w-6" />
        <Tr t={Translations.t.translations.activateButton} />
--- a/src/Utils.ts
+++ b/src/Utils.ts
@ -1882,6 +1882,14 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
        return [].concat(...param)
    }

+    public static sum(list: number[]): number {
+        let total = 0
+        for (const number of list) {
+            total += number
+        }
+        return total
+    }
+
    /**
     *
     * JSON.stringify(Utils.reorder({b: "0", a: "1"}, ["a", "b"])) // => '{"a":"1","b":"0"}'
--- a/src/assets/global-raster-layers.json
+++ b/src/assets/global-raster-layers.json
@ -23,14 +23,14 @@
      ],
      "best": true,
      "id": "protomaps.sunny-self",
-      "name": "Protomaps Sunny (Hosted by pietervdvn)",
+      "name": "Protomaps Sunny (Offline)",
      "type": "vector",
      "category": "osmbasedmap",
      "attribution": {
        "text": "Protomaps",
        "url": "https://protomaps.com/"
      },
-      "url": "https://cache.mapcomplete.org/global-basemap.pmtiles"
+      "url": "https://127.0.0.1/service-worker/offline-basemap/{z}-{x}-{y}.mvt"
    },
    {
      "name": "OpenStreetMap Carto",
--- a/src/service-worker/OfflineBasemapManager.ts
+++ b/src/service-worker/OfflineBasemapManager.ts
@ -163,12 +163,12 @@ export class OfflineBasemapManager {
     */
    private readonly _host: string

-    public static readonly zoomelevels = {
+    public static readonly zoomelevels: Record<number, number | undefined> = {
        0: 4,
        5: 7,
        8: 9,
        10: undefined
-    }
+    } as const

    private readonly blobs: TypedIdb<any>
    private readonly meta: TypedIdb<AreaDescription>
@ -201,6 +201,41 @@ export class OfflineBasemapManager {
        return this.metaCached
    }

+    public isInstalled(toCompare: { z?: number, minzoom?: number, x: number, y: number }): boolean {
+        return this.metaCached.some(area => area.x === toCompare.x && area.y === toCompare.y && (toCompare.minzoom ?? toCompare.z) === area.minzoom)
+    }
+
+
+    /**
+     * Returns all AreaDescriptions needed for the specified tile. Most specific zoom level last.
+     * Already installed area descriptions are _not_ returned
+     * @param tile
+     */
+    public* getInstallCandidates(tile: { z: number, x: number, y: number }): Generator<AreaDescription, void, unknown> {
+
+        for (const k in OfflineBasemapManager.zoomelevels) {
+            const z = Number(k)
+
+            const zDiff = tile.z - z
+            if (zDiff < 0) {
+                continue
+            }
+            const x = tile.x >> zDiff
+            const y = tile.y >> zDiff
+            if (!this.isInstalled({ z, x, y })) {
+                console.log("Installing level " + z + " archive")
+                yield <AreaDescription>{
+                    name: `${z}-${x}-${y}.pmtiles`,
+                    minzoom: z,
+                    maxzoom: OfflineBasemapManager.zoomelevels[z] ?? 15,
+                    x, y
+                }
+            }
+        }
+
+
+    }
+
    /**
     * ! Must be called from a fetch event in the service worker !
     * @param areaDescription
@ -230,11 +265,11 @@ export class OfflineBasemapManager {
            throw "Invalid filename, should end with .pmtiles"
        }
        const [z, x, y] = name.substring(0, name.length - ".pmtiles".length).split("-").map(Number)
-        const maxzooms: Record<number, number> = { 0: 4, 5: 8, 9: 15 }
+        const maxzooms: Record<number, number | undefined> = this.zoomelevels
        return {
            name,
            minzoom: z,
-            maxzoom: maxzooms[z],
+            maxzoom: maxzooms[z] ?? 15,
            x, y
        }
    }
@ -243,6 +278,13 @@ export class OfflineBasemapManager {
        return this.metaCached
    }

+    /**
+     * Looks through the 'installedMeta' where a tile can be extracted from
+     * @param z
+     * @param x
+     * @param y
+     * @private
+     */
    private determineArea(z: number, x: number, y: number): AreaDescription | undefined {
        for (const areaDescription of this.metaCached) {
            if (areaDescription.minzoom > z) {
@ -261,13 +303,30 @@ export class OfflineBasemapManager {
        return undefined
    }

-    async getTileResponse(z: number, x: number, y: number): Promise<Response> {
+    async getFallback(z: number, x: number, y: number): Promise<Response> {
+        const url = `https://api.protomaps.com/tiles/v4/${z}/${x}/${y}.mvt?key=2af8b969a9e8b692`
+        return fetch(url)
+    }
+
+    /**
+     *
+     * @param z
+     * @param x
+     * @param y
+     * @param fallback: if set and no local tile could be found: ask protomaps instead
+     */
+    async getTileResponse(z: number, x: number, y: number, options?: {
+        autoInstall?: boolean, fallback?: boolean
+    }): Promise<Response> {
        if (this.metaCached.length === 0) {
            await this.updateCachedMeta()
        }
        const area = this.determineArea(z, x, y)
        if (!area) {
-            return new Response("Not found", { status: 404 })
+            if (options?.fallback) {
+                return this.getFallback(z, x, y)
+            }
+            return new Response("Not found: no suitable area found", { status: 404 })
        }
        const blob = await this.blobs.get(area.name)
        const pmtiles = new BlobSource(area.name, blob)
--- a/src/service-worker/index.ts
+++ b/src/service-worker/index.ts
@ -6,35 +6,75 @@ declare var self: ServiceWorkerGlobalScope
 export {}
 import { OfflineBasemapManager } from "./OfflineBasemapManager"

+
 const offlinemaps = new OfflineBasemapManager("https://cache.mapcomplete.org/")

+function jsonResponse(object: object | []): Response {
+    return new Response(JSON.stringify(object), {
+        headers: {
+            "Content-Type": "application/json"
+        }
+    })
+}

 function routeOffline(event: FetchEvent) {
-    const url = event.request.url
-    const rest = url.split("/service-worker/offline-basemap/")[1]
-    if (rest.indexOf("install") >= 0) {
-        const filename = url.split("/").pop()
+    const url = new URL(event.request.url)
+    const rest = url.pathname.split("/service-worker/offline-basemap/")[1]
+
+    if (rest === "meta.json") {
+        event.respondWith(
+            offlinemaps.updateCachedMeta().then(meta => jsonResponse(meta)))
+        return
+    }
+    if (rest.indexOf("delete") >= 0) {
+        const filename = rest.split("/").pop()
        if (!filename) {
            return
        }
        const description = OfflineBasemapManager.getAreaDescriptionForMapcomplete(filename)
        event.respondWith(
-            offlinemaps.installArea(description).then(
-                () => {
-                    return new Response(
-                        JSON.stringify({
-                            "status": "installed",
-                            installed_previously: offlinemaps.getMeta()
-                        }), { headers: { "Content-Type": "application/json" } })
-                })
+            offlinemaps.deleteArea(description)
+                .then(() =>
+                    offlinemaps.updateCachedMeta())
+                .then(meta => jsonResponse({
+                    "status": "installed",
+                    installed: meta
+                })))
+    }
+    if (rest.indexOf("update") >= 0) {
+        event.respondWith(
+            offlinemaps.updateCachedMeta()
+                .then(meta => jsonResponse({
+                    "status": "installed",
+                    installed: meta
+                })))
+        return
+    }
+    if (rest.indexOf("install") >= 0) {
+        const filename = rest.split("/").pop()
+        if (!filename) {
+            return
+        }
+        const description = OfflineBasemapManager.getAreaDescriptionForMapcomplete(filename)
+        event.respondWith(
+            offlinemaps.installArea(description)
+                .then(() =>
+                    offlinemaps.updateCachedMeta())
+                .then(meta => jsonResponse({
+                    "status": "installed",
+                    installed: meta
+                }))
        )
        return
    }
    const tileRegex =/(\d+-\d+-\d+).mvt$/
    const tileMatch = rest.match(tileRegex)
    if (tileMatch) {
+        const fallback = url.searchParams.get("fallback") === "true"
+        const autoInstall = url.searchParams.get("auto") === "true"
+
        const [z, x, y] = tileMatch[1].split("-").map(Number)
-        event.respondWith(offlinemaps.getTileResponse(z, x, y))
+        event.respondWith(offlinemaps.getTileResponse(z, x, y, { fallback, autoInstall }))
    }
 }

@ -45,3 +85,6 @@ self.addEventListener("fetch", (event) => {

    }
 })
+
+self.addEventListener("install", () => self.skipWaiting())
+self.addEventListener("activate", event => event.waitUntil(self.clients.claim()))