Feature: offline: more features to be able to work fully offline

assets/layers/icons/icons.json

@@ -430,10 +430,10 @@
       }
     },
     {
-      "condition": "_favourite=yes",
+      "id": "favourite_icon",
       "description": "Only for rendering",
       "icon": "circle:white;heart:red",
-      "id": "favourite_icon",
+      "condition": "_favourite=yes",
       "metacondition": "__showTimeSensitiveIcons!=no"
     },
     {

assets/layers/last_click/last_click.json

@@ -217,8 +217,8 @@
     },
     {
       "id": "debug",
-      "metacondition": "__featureSwitchIsDebugging=true",
+      "render": "{all_tags()}",
-      "render": "{all_tags()}"
+      "metacondition": "__featureSwitchIsDebugging=true"
     }
   ],
   "filter": [

assets/layers/note/note.json

@@ -114,9 +114,9 @@
   "lineRendering": [],
   "tagRenderings": [
     {
-      "classes": "p-0",
       "id": "conversation",
-      "render": "{visualize_note_comments()}"
+      "render": "{visualize_note_comments()}",
+      "classes": "p-0"
     },
     {
       "id": "add_image",

assets/layers/osm_community_index/osm_community_index.json

@@ -66,16 +66,16 @@
   ],
   "tagRenderings": [
     {
-      "condition": "level=country",
-      "description": "The name of the country",
       "id": "country_name",
-      "render": "{nameEn} {emojiFlag}"
+      "description": "The name of the country",
+      "render": "{nameEn} {emojiFlag}",
+      "condition": "level=country"
     },
     {
-      "condition": "_community_links~*",
-      "description": "Community Links (Discord, meetups, Slack groups, IRC channels, mailing lists etc...)",
       "id": "community_links",
-      "render": "{_community_links}"
+      "description": "Community Links (Discord, meetups, Slack groups, IRC channels, mailing lists etc...)",
+      "render": "{_community_links}",
+      "condition": "_community_links~*"
     }
   ],
   "filter": [

assets/layers/usersettings/usersettings.json

@@ -1898,6 +1898,46 @@
       "render": {
         "*": "{storage_all_tags()}"
       }
+    },
+    {
+      "id": "debug_serviceworker_accordeon",
+      "render": {
+        "special": {
+          "header": "debug_serviceworker_accordeon_title",
+          "labels": "debug_serviceworker",
+          "type": "group"
+        }
+      },
+      "condition": "mapcomplete-show_debug=yes"
+    },
+    {
+      "id": "debug_serviceworker_accordeon_title",
+      "labels": [
+        "hidden"
+      ],
+      "render": {
+        "en": "Debug information about the service worker"
+      }
+    },
+    {
+      "id": "expl",
+      "labels": [
+        "debug_serviceworker",
+        "hidden"
+      ],
+      "render": {
+        "en": "To clear the service worker data, use the 'clear caches' button"
+      }
+    },
+    {
+      "id": "service_worker_tags",
+      "labels": [
+        "debug_serviceworker",
+        "hidden"
+      ],
+      "render": {
+        "*": "{serviceworker_all_tags()}"
+      }
     }
   ],
   "allowMove": false

index.html

@@ -43,8 +43,6 @@
 <script src="./src/Logic/Web/AndroidPolyfill.ts" type="module"></script>
 <script type="module" src="./src/all_themes_index.ts"></script>
 
-<script async src="./src/InstallServiceWorker.ts" type="module"></script>
-
 
 </body>
 </html>

langs/en.json

@@ -335,6 +335,7 @@
         "next": "Next",
         "noTagsSelected": "No tags selected",
         "number": "number",
+        "offline": "Your device is offline",
         "openTheMap": "Open the map",
         "openTheMapReason": "to view, edit and add information",
         "opening_hours": {

langs/layers/en.json

@@ -14095,6 +14095,9 @@
             "debug_accordeon_title": {
                 "render": "Debug information"
             },
+            "debug_serviceworker_accordeon_title": {
+                "render": "Debug information about the service worker"
+            },
             "debug_storage_accordeon_title": {
                 "render": "Debug information about local storage"
             },
@@ -14105,6 +14108,9 @@
                     }
                 }
             },
+            "expl": {
+                "render": "To clear the service worker data, use the 'clear caches' button"
+            },
             "fixate-north": {
                 "mappings": {
                     "0": {

scripts/generateIncludedImages.ts

@@ -1,7 +1,7 @@
 import * as fs from "fs"
 import Script from "./Script"
 
-function genImages(dryrun = false) {
+function genImages() {
     console.log("Generating images")
     const dir = fs.readdirSync("./assets/svg")
     for (const path of dir) {
@@ -64,7 +64,7 @@ class GenerateIncludedImages extends Script {
         super("Converts all images from assets/svg into svelte-classes.")
     }
 
-    async main(args: string[]): Promise<void> {
+    async main(): Promise<void> {
         genImages()
     }
 }

scripts/generateLayerOverview.ts

@@ -8,7 +8,7 @@ import {
     DoesImageExist,
     PrevalidateTheme,
     ValidateLayer,
-    ValidateThemeEnsemble,
+    ValidateThemeEnsemble
 } from "../src/Models/ThemeConfig/Conversion/Validation"
 import { Translation } from "../src/UI/i18n/Translation"
 import { OrderLayer, PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
@@ -19,7 +19,7 @@ import {
     DesugaringStep,
     Each,
     Fuse,
-    On,
+    On
 } from "../src/Models/ThemeConfig/Conversion/Conversion"
 import { Utils } from "../src/Utils"
 import Script from "./Script"
@@ -182,7 +182,7 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
         return `./assets/layers/${id}/${id}.json`
     }
 
-    writeLayer(layer: LayerConfigJson) {
+    public writeLayer(layer: LayerConfigJson) {
         if (layer.labels?.some((l) => this._labelBlacklist.has(l))) {
             console.log("Not writing layer " + layer.id + ", censored")
             return
@@ -191,6 +191,15 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
         if (!existsSync(LayerOverviewUtils.layerPath)) {
             mkdirSync(LayerOverviewUtils.layerPath)
         }
+
+        const usedImages = Lists.dedup(new ExtractImages(true, new Set(this._desugaringState.tagRenderings.keys()))
+            .convertStrict({ layers: [layer], id: "dummy", icon: undefined, title: undefined })
+            .map((x) => x.path))
+        usedImages.sort()
+
+        layer["_usedImages"] = usedImages
+
+
         writeFileSync(LayerBuilder.targetPath(layer.id), JSON.stringify(layer, null, "  "), {
             encoding: "utf8",
         })

scripts/prepareServiceWorker.ts

@@ -9,7 +9,7 @@ class PrepareServiceWorker extends Script {
     }
 
     public async main() {
-        const v = Constants.vNumber
+        const v = Constants.vNumber + "-" + new Date().getTime()
         writeFileSync("./src/service-worker/SWGenerated.ts",
             ["export class SWGenerated {",
                 "// generated by scripts/prepareServiceWorker.ts",

src/InstallServiceWorker.ts

@@ -1,13 +1,17 @@
-export {}
+export class InstallServiceWorker {
-window.addEventListener("load", async () => {
+
-    if (!("serviceWorker" in navigator)) {
+    static async installServiceWorker() {
-        console.log("Service workers are not supported")
+        if (!("serviceWorker" in navigator)) {
-        return
+            throw ("Service workers are not supported")
-    }
+        }
-    try {
         await navigator.serviceWorker.register("/service-worker.js", { type: "module" })
         console.log("Service worker registration successful")
-    } catch (err) {
+
-        console.error("Service worker registration failed", err)
     }
-})
+
+    static async precache(assets: string[]) {
+        if (assets?.length > 0) {
+            await fetch("./service-worker/precache?assets=" + assets.join(";"))
+        }
+    }
+}

src/Logic/Osm/OsmConnection.ts

@@ -8,6 +8,7 @@ import Constants from "../../Models/Constants"
 import { Feature, Point } from "geojson"
 import { AndroidPolyfill } from "../Web/AndroidPolyfill"
 import { QueryParameters } from "../Web/QueryParameters"
+import { IsOnline } from "../Web/IsOnline"
 
 interface OsmUserInfo {
     id: number
@@ -131,6 +132,7 @@ export class OsmConnection {
      * Details of the currently logged-in user; undefined if not logged in
      */
     public userDetails: UIEventSource<UserDetails | undefined>
+
     public isLoggedIn: Store<boolean>
     public gpxServiceIsOnline: UIEventSource<OsmServiceState> = new UIEventSource<OsmServiceState>(
         "unknown"
@@ -182,7 +184,7 @@ export class OsmConnection {
             this._oauth_config.oauth_secret = import.meta.env.VITE_OSM_OAUTH_SECRET
         }
 
-        this.userDetails = new UIEventSource<UserDetails>(undefined, "userDetails")
+        this.userDetails = UIEventSource.asObject<UserDetails>(LocalStorageSource.get("user_details"), undefined)
         if (options.fakeUser) {
             const ud = this.userDetails.data
             ud.csCount = 5678
@@ -197,13 +199,7 @@ export class OsmConnection {
         }
         this.updateCapabilities()
 
-        this.isLoggedIn = this.userDetails.map(
+        this.isLoggedIn = this.userDetails.map((user) => !!user)
-            (user) =>
-                !!user &&
-                (this.apiIsOnline.data === "unknown" || this.apiIsOnline.data === "online"),
-            [this.apiIsOnline]
-        )
-
         this._dryRun = options.dryRun ?? new UIEventSource<boolean>(false)
 
         if (options?.shared_cookie) {
@@ -284,6 +280,9 @@ export class OsmConnection {
     }
 
     public async AttemptLogin() {
+        if (!IsOnline.isOnline.data) {
+            return
+        }
         this.updateCapabilities()
         if (this.loadingStatus.data !== "logged-in") {
             this.loadingStatus.setData("loading")
@@ -308,6 +307,9 @@ export class OsmConnection {
     }
 
     private async loadUserInfo() {
+        if (!IsOnline.isOnline.data) {
+            return
+        }
         try {
             const result = await this.interact("user/details.json")
             if (result === null) {

src/UI/Base/LoginToggle.svelte

@@ -14,10 +14,15 @@
     osmConnection: OsmConnection
     featureSwitches?: { featureSwitchEnableLogin?: UIEventSource<boolean> }
   }
+  /**
+   * Do show this element when in offline mode
+   */
+  export let offline = false
   /**
    * If set, 'loading' will act as if we are already logged in.
    */
-  export let ignoreLoading: boolean = false
+  export let ignoreLoading: boolean = offline // If it works in offline mode, it'll work while we are logging in too
+
   /**
    * If set and the OSM-api fails, do _not_ show any error messages nor the successful state, just hide.
    * Will still show the "not-logged-in"-slot
@@ -32,23 +37,26 @@
     unknown: t.loginFailedUnreachableMode,
     readonly: t.loginFailedReadonlyMode,
   }
-  const apiState: Store<string> =
+  const apiState: Store<OsmServiceState> =
     state?.osmConnection?.apiIsOnline ?? new ImmutableStore<OsmServiceState>("online")
   const online = IsOnline.isOnline
+  let loggedIn = state?.osmConnection?.isLoggedIn
 </script>
 
 {#if $badge}
-  {#if !$online}
+  {#if !$online && !offline}
     {#if !hiddenFail}
       <div class="alert">
-        Your device is offline
+        <Tr t={t.offline} />
       </div>
     {/if}
   {:else if !ignoreLoading && !hiddenFail && $loadingStatus === "loading"}
     <slot name="loading">
       <Loading />
     </slot>
-  {:else if ($loadingStatus === "error" || $apiState === "readonly" || $apiState === "offline")}
+  {:else if $loggedIn}
+    <slot />
+  {:else if ($loadingStatus === "error" || $apiState === "readonly" || $apiState === "offline" || $apiState === "unreachable")}
     {#if !hiddenFail}
       <slot name="error">
         <div class="alert flex flex-col items-center">
@@ -63,8 +71,7 @@
         </div>
       </slot>
     {/if}
-  {:else if $loadingStatus === "logged-in"}
+
-    <slot />
   {:else if $loadingStatus === "not-attempted"}
     <slot name="not-logged-in" />
   {/if}

src/UI/BigComponents/MenuDrawerIndex.svelte

@@ -77,11 +77,9 @@
 
   let usersettingslayer = new LayerConfig(<LayerConfigJson>usersettings, "usersettings", true)
 
-  let theme = state.theme
   let featureSwitches = state.featureSwitches
   let showHome = featureSwitches?.featureSwitchBackToThemeOverview
   let pg = state.guistate.pageStates
-  let location = state.mapProperties?.location
   export let onlyLink: boolean
   const t = Translations.t.general.menu
   let shown = new UIEventSource(state.guistate.pageStates.menu.data || !onlyLink)
@@ -133,7 +131,7 @@
 
     <!-- User related: avatar, settings, favourits, logout -->
     <SidebarUnit>
-      <LoginToggle {state}>
+      <LoginToggle {state} offline>
         <LoginButton osmConnection={state.osmConnection} slot="not-logged-in" />
         <div class="flex items-center gap-x-4 w-full m-2">
           {#if $userdetails.img}
@@ -143,7 +141,7 @@
           {/if}
           <div class="flex flex-col w-full gap-y-2">
 
-            <b>{$userdetails.name}</b>
+            <b>{$userdetails?.name ?? '<Username>'}</b>
             <LogoutButton clss="as-link small subtle text-sm" osmConnection={state.osmConnection} />
           </div>
         </div>
@@ -281,7 +279,7 @@
         <Tr t={Translations.t.inspector.menu} />
       </a>
 
-      {#if !state.theme}
+      {#if !state?.theme}
         <a class="flex" href={($isAndroid ? "https://mapcomplete.org" : ".") +`/statistics.html`}
            target="_blank">
           <ChartBar class="h-6 w-6" />

src/UI/Popup/ClearCaches.svelte

@@ -4,7 +4,7 @@
 
   function clearCaches() {
     IdbLocalStorage.clearAll()
-    Utils.download("./service-worker-clear")
+    Utils.download("./service-worker/clear_caches.json")
     window.location.reload()
   }
   export let msg: string

src/UI/Popup/TagRendering/TagRenderingEditable.svelte

@@ -128,7 +128,7 @@
           {layer}
           extraClasses="my-2"
         />
-        {#if (!editingEnabled || $editingEnabled) && $apiState !== "readonly" && $apiState !== "offline"}
+        {#if !editingEnabled || $editingEnabled}
           <EditButton
             arialabel={config.editButtonAriaLabel}
             ariaLabelledBy={answerId}

src/UI/Popup/TagRendering/TagRenderingQuestion.svelte

@@ -359,7 +359,7 @@
 {#if question !== undefined && $apiState !== "readonly" && $apiState !== "offline"}
   <div class={clss}>
     {#if layer?.isNormal()}
-      <LoginToggle {state}>
+      <LoginToggle {state} offline hiddenFail>
         <DotMenu hideBackground={true} open={menuIsOpened}>
           <SidebarUnit>
             {#if $disabledInTheme.indexOf(config.id) >= 0}
@@ -538,7 +538,7 @@
         {/if}
 
         <!-- Save and cancel buttons, in a logintoggle -->
-        <LoginToggle {state} ignoreLoading>
+        <LoginToggle {state} ignoreLoading offline>
           <div class="flex w-full justify-end" slot="not-logged-in">
             {#if config.alwaysForceSaveButton}
               <button

src/UI/SingleThemeGui.svelte

@@ -13,6 +13,8 @@
   import { WithSearchState } from "../Models/ThemeViewState/WithSearchState"
   import ThemeConfig from "../Models/ThemeConfig/ThemeConfig"
   import { AndroidPolyfill } from "../Logic/Web/AndroidPolyfill"
+  import { InstallServiceWorker } from "../InstallServiceWorker"
+  import type { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
 
   function webgl_support() {
     try {
@@ -51,6 +53,26 @@
 
   let availableLayers = UIEventSource.fromPromise(getAvailableLayers())
   const state = new WithSearchState(theme, availableLayers)
+  InstallServiceWorker.installServiceWorker().then(() => {
+    if (theme.source._usedImages) {
+
+      window.requestIdleCallback(() => {
+        InstallServiceWorker.precache(theme.source._usedImages?.filter(i => i.startsWith("./")))
+      })
+    }
+    for (const layer of (<LayerConfigJson[]>theme.source.layers)) {
+      if (!Constants.isPriviliged(layer)) {
+        continue
+      }
+      if (!layer["_usedImages"]) {
+        continue
+      }
+      // The priviliged layers, which are injected, might have assets not included in the '_usedImages' of the theme
+      window.requestIdleCallback(() => {
+        InstallServiceWorker.precache(layer["_usedImages"]?.filter(i => i.startsWith("./")))
+      })
+    }
+  }).catch(e => console.error("Could not install service worker:", e))
 </script>
 
 {#if !webgl_supported}

src/UI/SpecialVisualisations/SettingsVisualisations.ts

@@ -79,7 +79,7 @@ class GpsAllTags extends SpecialVisualizationSvelte {
     }
 }
 
-class StorageAlLTags extends SpecialVisualizationSvelte {
+class StorageAllTags extends SpecialVisualizationSvelte {
     funcName = "storage_all_tags"
     group = "settings"
     docs = "Shows the current state of storage"
@@ -103,6 +103,27 @@ class StorageAlLTags extends SpecialVisualizationSvelte {
     }
 }
 
+
+class ServiceWorkerAllTags extends SpecialVisualizationSvelte {
+    funcName = "serviceworker_all_tags"
+    group = "settings"
+    docs = "Shows the current state of service worker"
+    args = []
+
+    constr(state: SpecialVisualizationState): SvelteUIElement {
+        const data = {}
+        for (const key in localStorage) {
+            data[key] = localStorage[key]
+        }
+        const tags = UIEventSource.fromPromise<Record<string, any>>(
+            Utils.downloadJson(
+                "./service-worker/status.json")
+        )
+
+        return new SvelteUIElement(AllTagsPanel, { state, tags })
+    }
+}
+
 export class ClearCachesVis extends SpecialVisualizationSvelte {
     funcName = "clear_caches"
     docs =
@@ -234,7 +255,8 @@ export class SettingsVisualisations {
             new DisabledQuestionsVis(),
             new GyroscopeAllTags(),
             new GpsAllTags(),
-            new StorageAlLTags(),
+            new StorageAllTags(),
+            new ServiceWorkerAllTags(),
             new ClearCachesVis(),
             new LoginButtonVis(),
             new QrLogin(),

src/all_themes_index.ts

@@ -1,5 +1,6 @@
 import { QueryParameters } from "./Logic/Web/QueryParameters"
 import AllThemesGui from "./UI/AllThemesGui.svelte"
+import { InstallServiceWorker } from "./InstallServiceWorker"
 
 const theme = QueryParameters.GetQueryParameter("layout", undefined).data ?? ""
 const customLayout = QueryParameters.GetQueryParameter("userlayout", undefined).data ?? ""
@@ -26,7 +27,7 @@ if (theme !== "") {
         l.protocol + "//" + window.location.host + "/theme.html" + l.search + l.hash
     )
 }
-
+InstallServiceWorker.installServiceWorker().catch(e => console.error(e))
 new AllThemesGui({
     target: document.getElementById("main"),
 })

src/service-worker/SWGenerated.ts

@@ -1,4 +1,4 @@
 export class SWGenerated {
 // generated by scripts/prepareServiceWorker.ts
-static vNumber = "0.54.4"
+static vNumber = "0.54.6-1754230245885"
 }

src/service-worker/index.ts

@@ -37,6 +37,14 @@ async function listCachedRequests(): Promise<string[]> {
     return requests.map(req => req.url)
 }
 
+async function clearCaches(): Promise<void> {
+    const cache = await caches.open(SWGenerated.vNumber)
+    const keys = await cache.keys()
+    for (const key of keys) {
+        await cache.delete(key)
+    }
+}
+
 class Router {
     private readonly _endpoints: Record<string, (event: FetchEvent) => void>
     private readonly _subpaths: Record<string, (event: FetchEvent, rest: string) => void>
@@ -71,12 +79,33 @@ class Router {
 
 
 const allOffline = new Router({
+    "clear_caches.Json": (event) => {
+        event.respondWith(
+            clearCaches().then(() =>
+                jsonResponse({ status: "ok" })
+            )
+        )
+    },
+    "precache": (event) => {
+        const url = new URL(event.request.url)
+        const assets = url.searchParams.get("assets")?.split(";")
+        if (assets) {
+            console.log("Precaching:", assets)
+            event.waitUntil(caches.open(SWGenerated.vNumber).then(cache =>
+                cache.addAll(assets)
+            ))
+        }
+        event.respondWith(jsonResponse({ status: "ok" }))
+    },
         "status.json": (event) => {
             event.respondWith(
                 listCachedRequests().then(cached =>
                     jsonResponse(
                         {
-                            status: "ok", cached,
+                            status: "ok",
+                            vnumber: SWGenerated.vNumber,
+                            domain: selfDomain,
+                            cached
                         }
                     ))
             )
@@ -89,7 +118,6 @@ self.addEventListener("fetch", (event) => {
     if (url.endsWith("/service-worker.js")) {
         return // Installation of a new version, we don't interfere
     }
-    console.log("Intercepting event", event.request.url)
     if (url.indexOf("/service-worker/") >= 0) {
         allOffline.route(event)
         return
@@ -99,6 +127,11 @@ self.addEventListener("fetch", (event) => {
         respondFromCache(event)
         return
     }
+    if (urlObj.hostname === "data.mapcomplete.org") {
+        respondFromCache(event)
+        return
+    }
+
 })
 
 self.addEventListener("install", () => self.skipWaiting())