From 988643dafad71c3c618c05c49791009ce56de9fa Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Tue, 2 Sep 2025 03:13:11 +0200 Subject: [PATCH 1/4] Feature: first version of offline foreground data management --- langs/en.json | 4 +- .../Actors/OfflineForegroundDataManager.ts | 84 +++++++++ src/Logic/BBox.ts | 39 +++- src/Logic/FeatureSource/FeatureSource.ts | 4 +- .../Sources/OverpassFeatureSource.ts | 73 ++++---- .../FeatureSource/Sources/ThemeSource.ts | 14 +- src/Logic/OfflineBasemapManager.ts | 20 ++- src/Logic/Osm/Overpass.ts | 13 +- src/Logic/State/GeolocationControlState.ts | 2 + src/Logic/UIEventSource.ts | 15 +- src/Logic/Web/IsOnline.ts | 8 +- .../ThemeViewState/WithLayoutSourceState.ts | 4 + src/Models/TileRange.ts | 19 +- src/UI/BigComponents/MenuDrawerIndex.svelte | 4 +- .../OfflineForegroundManagement.svelte | 166 ++++++++++++++++++ src/UI/BigComponents/OfflineManagement.svelte | 101 ++++++----- src/UI/Map/MapLibreAdaptor.ts | 9 +- 17 files changed, 470 insertions(+), 109 deletions(-) create mode 100644 src/Logic/Actors/OfflineForegroundDataManager.ts create mode 100644 src/UI/BigComponents/OfflineForegroundManagement.svelte diff --git a/langs/en.json b/langs/en.json index 196b2de20..578c6b862 100644 --- a/langs/en.json +++ b/langs/en.json @@ -774,7 +774,7 @@ "installing": "Data is being downloaded", "localOnMap": "Offline basemaps on the map", "name": "Name", - "overview": "Offline basemaps overview", + "overview": "Offline background map", "range": "Zoom ranges", "size": "Size" }, @@ -839,7 +839,7 @@ "reviews": { "affiliated_reviewer_warning": "(Affiliated review)", "attribution": "By Mangrove.reviews", - "averageRating": "Average rating of {n} stars", + "averageRating": "Average ting of {n} stars", "delete": "Delete review", "deleteConfirm": "Permanently delete this review", "deleteText": "This cannot be undone", diff --git a/src/Logic/Actors/OfflineForegroundDataManager.ts b/src/Logic/Actors/OfflineForegroundDataManager.ts new file mode 100644 index 000000000..749492f82 --- /dev/null +++ b/src/Logic/Actors/OfflineForegroundDataManager.ts @@ -0,0 +1,84 @@ +/** + * Keeps track of what foreground (mostly OSM) data is loaded and should be _reloaded_ automatically when internet is restored + * + */ +import ThemeSource from "../FeatureSource/Sources/ThemeSource" +import { LocalStorageSource } from "../Web/LocalStorageSource" +import { Store, UIEventSource } from "../UIEventSource" +import { BBox } from "../BBox" +import { WithLayoutSourceState } from "../../Models/ThemeViewState/WithLayoutSourceState" +import { Lists } from "../../Utils/Lists" +import { IsOnline } from "../Web/IsOnline" + +export class OfflineForegroundDataManager { + private _themeSource: ThemeSource + private readonly _bboxesForOffline: UIEventSource<[[number, number], [number, number]][]> + public readonly bboxesForOffline: UIEventSource<[[number, number], [number, number]][]> + private readonly _isUpdating: UIEventSource = new UIEventSource(false) + public readonly isUpdating: Store = this._isUpdating + + public readonly updateWhenReconnected: UIEventSource + public readonly updateOnLoad: UIEventSource + + constructor(state: WithLayoutSourceState) { + this._themeSource = state.indexedFeatures + this._bboxesForOffline = + UIEventSource.asObject<[[number, number], [number, number]][]>(LocalStorageSource.get("bboxes_for_offline_" + state.theme.id, "[]"), []) + this._bboxesForOffline.update(bboxes => OfflineForegroundDataManager.clean(bboxes)) + this.updateWhenReconnected = LocalStorageSource.getParsed("updateWhenReconnected_" + state.theme.id, false) + this.updateOnLoad = LocalStorageSource.getParsed("updateOnLoad_" + state.theme.id, false) + + + this.bboxesForOffline = new UIEventSource(this._bboxesForOffline.data) + this.bboxesForOffline.addCallbackD(bboxes => { + bboxes = OfflineForegroundDataManager.clean(bboxes) + this._bboxesForOffline.set(bboxes) // Will trigger an update + }) + + this._bboxesForOffline.addCallbackD(bboxes => this.updateBboxes(bboxes)) + + this.updateWhenReconnected.once(() => { + IsOnline.isOnline.addCallbackD(isOnline => { + if (isOnline) { + this.updateAll() + } + }) + }, v => v === true) + + this.updateOnLoad.once(() => { + this.updateAll() + }, v => v === true) + + } + + public async updateAll() { + await this.updateBboxes(this._bboxesForOffline.data) + } + + private async updateBboxes(bboxes: [[number, number], [number, number]][]) { + if (this._isUpdating.data) { + console.trace("Duplicate updateBboxes") + return + } + this._isUpdating.set(true) + for (const bboxCoor of bboxes) { + console.trace("Downloading ", bboxCoor.flatMap(x => x).join("; ")) + await this._themeSource.downloadAll(new BBox(bboxCoor)) + } + this._isUpdating.set(false) + + } + + private static clean(bboxes: [[number, number], [number, number]][]) { + const asBbox = bboxes.map(bbox => { + try { + return new BBox(bbox) + } catch (e) { + console.error("Got an invalid bbox:", bbox) + return undefined + } + }) + const cleaned = BBox.dropFullyContained(Lists.noNull(asBbox)) + return cleaned.map(bbox => bbox.toLngLat()) + } +} diff --git a/src/Logic/BBox.ts b/src/Logic/BBox.ts index c1b629549..9ba6aeec7 100644 --- a/src/Logic/BBox.ts +++ b/src/Logic/BBox.ts @@ -298,7 +298,7 @@ export class BBox { * const expanded = bbox.expandToTileBounds(15) * !isNaN(expanded.minLat) // => true */ - expandToTileBounds(zoomlevel: number): BBox { + public expandToTileBounds(zoomlevel: number): BBox { if (zoomlevel === undefined) { return this } @@ -309,7 +309,7 @@ export class BBox { return new BBox([].concat(boundsul, boundslr)) } - toMercator(): { minLat: number; maxLat: number; minLon: number; maxLon: number } { + public toMercator(): { minLat: number; maxLat: number; minLon: number; maxLon: number } { const [minLon, minLat] = GeoOperations.ConvertWgs84To900913([this.minLon, this.minLat]) const [maxLon, maxLat] = GeoOperations.ConvertWgs84To900913([this.maxLon, this.maxLat]) @@ -341,7 +341,40 @@ export class BBox { return GeoOperations.calculateOverlap(this.asGeoJson({}), [f]).length > 0 } - center() { + public center() { return [(this.minLon + this.maxLon) / 2, (this.minLat + this.maxLat) / 2] } + + /** + * Simplifies a list of bboxes so that bboxes that are fully contained are filtered out. + * Elements will keep the same relative order + * + * const bbox1 = new BBox([ 1,1,2,2]) + * const bbox2 = new BBox([0,0,3,3]) + * const bbox3 = new BBox([0,1,3,2]) + * const bbox4 = new BBox([1,0,2,3]) + * BBox.dropFullyContained([bbox1, bbox2]) // => [bbox2] + * BBox.dropFullyContained([bbox2, bbox1]) // => [bbox2] + * BBox.dropFullyContained([bbox2, bbox1, bbox3]) // => [bbox2] + * BBox.dropFullyContained([bbox4, bbox3]) // => [bbox4, bbox3] + * BBox.dropFullyContained([bbox3, bbox3]) // => [bbox3] + * + */ + static dropFullyContained(bboxes: ReadonlyArray): BBox[] { + const newBboxes: BBox[] = [] + for (const bbox of bboxes) { + if (newBboxes.some(newBbox => bbox.isContainedIn(newBbox))) { + continue + } + for (let i = newBboxes.length - 1; i >= 0; i--) { + const newBbox = newBboxes[i] + if (newBbox.isContainedIn(bbox)) { + newBboxes.splice(i, 1) + } + } + newBboxes.push(bbox) + } + return newBboxes + } + } diff --git a/src/Logic/FeatureSource/FeatureSource.ts b/src/Logic/FeatureSource/FeatureSource.ts index db1b0be0d..61aa4b5f5 100644 --- a/src/Logic/FeatureSource/FeatureSource.ts +++ b/src/Logic/FeatureSource/FeatureSource.ts @@ -36,6 +36,8 @@ export interface FeatureSourceForTile extends Featu /** * A feature source which is aware of the indexes it contains */ -export interface IndexedFeatureSource extends FeatureSource { +export interface IndexedFeatureSource & { + id: string +}>> extends FeatureSource { readonly featuresById: Store> } diff --git a/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts b/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts index 7cbad0368..f815d5bdd 100644 --- a/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts +++ b/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts @@ -22,7 +22,6 @@ export default class OverpassFeatureSource im public readonly features: UIEventSource = new UIEventSource(undefined) public readonly runningQuery: UIEventSource = new UIEventSource(false) - public readonly timeout: UIEventSource = new UIEventSource(0) private readonly retries: UIEventSource = new UIEventSource(0) @@ -102,19 +101,11 @@ export default class OverpassFeatureSource im } /** - * Download the relevant data from overpass. Attempt to use a different server if one fails; only downloads the relevant layers - * Will always attempt to download, even is 'options.isActive.data' is 'false', the zoom level is incorrect, ... - * @private + * Attempts to download the data from an overpass server; + * fails over to the next server in case of failure */ - public async updateAsync(overrideBounds?: BBox): Promise { - if (!navigator.onLine) { - return - } - let data: { features: T[] } = undefined - let lastUsed = 0 - const start = new Date() + private async attemptDownload(overrideBounds?: BBox, serverIndexStart?: number): Promise<{ features: T[] }> { const layersToDownload = this._layersToDownload.data - if (layersToDownload.length == 0) { return } @@ -123,6 +114,7 @@ export default class OverpassFeatureSource im if (overpassUrls === undefined || overpassUrls.length === 0) { throw "Panic: overpassFeatureSource didn't receive any overpassUrls" } + let serverToTry = serverIndexStart ?? 0 // Index in overpassUrls // Note: the bounds are updated between attempts, in case that the user zoomed around let bounds: BBox do { @@ -136,37 +128,51 @@ export default class OverpassFeatureSource im return } - const overpass = this.GetFilter(overpassUrls[lastUsed], layersToDownload) + const overpass = this.getFilter(overpassUrls[serverToTry], layersToDownload) if (overpass === undefined) { return undefined } - this.runningQuery.setData(true) - data = (await overpass.queryGeoJson(bounds))[0] + const data = (await overpass.queryGeoJson(bounds))[0] + if (data) { + this._lastQueryBBox = bounds + this._lastRequestedLayers = layersToDownload + return data // Success! + } } catch (e) { - this.retries.data++ - this.retries.ping() + this.retries.update(i => i + 1) console.error(`QUERY FAILED due to`, e) await Utils.waitFor(1000) - if (lastUsed + 1 < overpassUrls.length) { - lastUsed++ - console.log("Trying next time with", overpassUrls[lastUsed]) - } else { - lastUsed = 0 - this.timeout.setData(this.retries.data * 5) + serverToTry++ + serverToTry %= overpassUrls.length + console.log("Trying next time with", overpassUrls[serverToTry]) - while (this.timeout.data > 0) { - await Utils.waitFor(1000) - this.timeout.data-- - this.timeout.ping() - } + if (serverToTry === 0) { + // We do a longer timeout as we cycled through all our overpass instances + await Utils.waitFor(1000 * this.retries.data) } + } finally { + this.retries.set(0) } - } while (data === undefined && this._isActive.data) + } while (this._isActive.data) + } + + /** + * Download the relevant data from overpass. Attempt to use a different server if one fails; only downloads the relevant layers + * Will always attempt to download, even is 'options.isActive.data' is 'false', the zoom level is incorrect, ... + * @private + */ + public async updateAsync(overrideBounds?: BBox): Promise { + if (!navigator.onLine) { + return + } + const start = new Date() try { + this.runningQuery.setData(true) + const data: { features: T[] } = await this.attemptDownload(overrideBounds) if (data === undefined) { return undefined } @@ -183,8 +189,7 @@ export default class OverpassFeatureSource im "seconds" ) this.features.setData(data.features) - this._lastQueryBBox = bounds - this._lastRequestedLayers = layersToDownload + } catch (e) { console.error("Got the overpass response, but could not process it: ", e, e.stack) } finally { @@ -200,7 +205,7 @@ export default class OverpassFeatureSource im * @constructor * @private */ - private GetFilter(interpreterUrl: string, layersToDownload: LayerConfig[]): Overpass { + private getFilter(interpreterUrl: string, layersToDownload: LayerConfig[]): Overpass { let filters: TagsFilter[] = layersToDownload.map((layer) => layer.source.osmTags) filters = Lists.noNull(filters) if (filters.length === 0) { @@ -222,10 +227,6 @@ export default class OverpassFeatureSource im return undefined } - if (this.timeout.data > 0) { - console.log("Still in timeout - not updating") - return undefined - } const requestedBounds = this.state.bounds.data if ( this._lastQueryBBox !== undefined && diff --git a/src/Logic/FeatureSource/Sources/ThemeSource.ts b/src/Logic/FeatureSource/Sources/ThemeSource.ts index a9c4a91ed..35d561cd7 100644 --- a/src/Logic/FeatureSource/Sources/ThemeSource.ts +++ b/src/Logic/FeatureSource/Sources/ThemeSource.ts @@ -21,7 +21,10 @@ import { IsOnline } from "../../Web/IsOnline" * * Note that special layers (with `source=null` will be ignored) */ -export default class ThemeSource & {id: string}>> implements IndexedFeatureSource { +export default class ThemeSource & { + id: string +}> = Feature & { id: string }>> + implements IndexedFeatureSource { /** * Indicates if a data source is loading something */ @@ -82,8 +85,9 @@ export default class ThemeSource }) } - public async downloadAll() { - return this.core.data.downloadAll() + public async downloadAll(bbox?: BBox) { + const core = await this.core.awaitValue() + await core.downloadAll(bbox) } public addSource(source: FeatureSource) { @@ -308,9 +312,9 @@ class ThemeSourceCore extends FeatureSourceMerger { ) } - public async downloadAll() { + public async downloadAll(bbox?: BBox) { console.log("Downloading all data:") - await this._downloadAll.updateAsync(this._mapBounds.data) + await this._downloadAll.updateAsync(bbox ?? this._mapBounds.data) // await Promise.all(this.supportsForceDownload.map((i) => i.updateAsync())) console.log("Done") } diff --git a/src/Logic/OfflineBasemapManager.ts b/src/Logic/OfflineBasemapManager.ts index d2e59eb82..703780c3b 100644 --- a/src/Logic/OfflineBasemapManager.ts +++ b/src/Logic/OfflineBasemapManager.ts @@ -4,6 +4,8 @@ import { IsOnline } from "./Web/IsOnline" import Constants from "../Models/Constants" import { Store, UIEventSource } from "./UIEventSource" import { Utils } from "../Utils" +import { BBox } from "./BBox" +import { Tiles } from "../Models/TileRange" export interface AreaDescription { /** @@ -170,7 +172,7 @@ export class OfflineBasemapManager { * Where to get the initial map tiles * @private */ - private readonly _host: string + public readonly host: string public static readonly zoomelevels: Record = { 0: 4, @@ -207,7 +209,7 @@ export class OfflineBasemapManager { if (!host.endsWith("/")) { host += "/" } - this._host = host + this.host = host this.blobs = new TypedIdb("OfflineBasemap") this.meta = new TypedIdb("OfflineBasemapMeta") this.updateCachedMeta() @@ -264,7 +266,7 @@ export class OfflineBasemapManager { * @private */ public async installArea(areaDescription: AreaDescription) { - const target = this._host + areaDescription.name + const target = this.host + areaDescription.name if (this.isInstalled(areaDescription)) { // Already installed return true @@ -448,4 +450,16 @@ export class OfflineBasemapManager { } return await this.fallback(params, abortController) } + + public async installBbox(bbox: BBox) { + const maxZoomlevel = Math.max(...Object.keys(OfflineBasemapManager.zoomelevels).map(s => Number(s))) + const range = Tiles.TileRangeBetween(maxZoomlevel, bbox.maxLat, bbox.maxLon, bbox.minLat, bbox.minLon) + if (range.total > 100) { + throw "BBox is to big to isntall" + } + await Tiles.mapRangeAsync(range, (x, y) => { + const z = maxZoomlevel + return this.autoInstall({ z, x, y }) + }) + } } diff --git a/src/Logic/Osm/Overpass.ts b/src/Logic/Osm/Overpass.ts index 3c9edd61b..72c2f2153 100644 --- a/src/Logic/Osm/Overpass.ts +++ b/src/Logic/Osm/Overpass.ts @@ -49,21 +49,26 @@ export class Overpass { bounds.getEast() + "]" const query = this.buildScript(bbox) - return await this.ExecuteQuery(query) + return await this.executeQuery(query) } public buildUrl(query: string) { return `${this._interpreterUrl}?data=${encodeURIComponent(query)}` } - private async ExecuteQuery( + private async executeQuery( query: string ): Promise<[{features: T[]}, Date]> { - const json = await Utils.downloadJson<{ + const jsonResult = await Utils.downloadJsonAdvanced<{ elements: [] remark osm3s: { timestamp_osm_base: string } - }>(this.buildUrl(query), {}) + }>(this.buildUrl(query), {}, 1) + + if (jsonResult["error"]) { + throw jsonResult["error"] + } + const json = jsonResult["content"] if (json.elements.length === 0 && json.remark !== undefined) { console.warn("Timeout or other runtime error while querying overpass", json.remark) diff --git a/src/Logic/State/GeolocationControlState.ts b/src/Logic/State/GeolocationControlState.ts index 2052a759d..aeff0e764 100644 --- a/src/Logic/State/GeolocationControlState.ts +++ b/src/Logic/State/GeolocationControlState.ts @@ -6,6 +6,8 @@ import { MapProperties } from "../../Models/MapProperties" /** * Does the user interaction state with a geolocation button, such as keeping track of the last click, * and lock status + moving the map when clicked + * + * Note: _not_ part of the big hierarchy */ export class GeolocationControlState { public readonly lastClick = new UIEventSource(undefined) diff --git a/src/Logic/UIEventSource.ts b/src/Logic/UIEventSource.ts index 8d31037ec..b7e25196b 100644 --- a/src/Logic/UIEventSource.ts +++ b/src/Logic/UIEventSource.ts @@ -346,16 +346,27 @@ export abstract class Store implements Readable { * @param callback * @param condition */ - public once(callback: () => void, condition?: (v: T) => boolean) { + public once(callback: (t: T) => void, condition?: (v: T) => boolean) { condition ??= (v) => !!v this.addCallbackAndRunD((v) => { if (condition(v)) { - callback() + callback(v) return true } }) } + /** + * awaits until there is a value meeting the condition (default: not null, not undefined); return this value + * If the store contains a value already matching the criteria, the promise will resolve immediately + */ + public awaitValue(condition?: (v: T) => boolean): Promise { + return new Promise(resolve => { + this.once(value => resolve(value), condition) + }) + } + + /** * Create a new UIEVentSource. Whenever 'this.data' changes, the returned UIEventSource will get this value as well. * However, this value can be overridden without affecting source diff --git a/src/Logic/Web/IsOnline.ts b/src/Logic/Web/IsOnline.ts index a2c0cb300..dff6ae2f6 100644 --- a/src/Logic/Web/IsOnline.ts +++ b/src/Logic/Web/IsOnline.ts @@ -1,5 +1,6 @@ import { Store, UIEventSource } from "../UIEventSource" import { Utils } from "../../Utils" +import { LocalStorageSource } from "./LocalStorageSource" export class IsOnline { private static readonly _isOnline: UIEventSource = new UIEventSource( @@ -17,5 +18,10 @@ export class IsOnline { } } - public static readonly isOnline: Store = IsOnline._isOnline + /** + * Doesn't yet properly work - many features don't use 'isOnline' yet to guard features but rather attempt to connect and fallback through failure + */ + public static readonly forceOffline: UIEventSource = UIEventSource.asBoolean(LocalStorageSource.get("forceOffline", "false")) + + public static readonly isOnline: Store = IsOnline._isOnline.map(online => online && !IsOnline.forceOffline.data, [IsOnline.forceOffline]) } diff --git a/src/Models/ThemeViewState/WithLayoutSourceState.ts b/src/Models/ThemeViewState/WithLayoutSourceState.ts index d09f41d96..22abec73b 100644 --- a/src/Models/ThemeViewState/WithLayoutSourceState.ts +++ b/src/Models/ThemeViewState/WithLayoutSourceState.ts @@ -10,6 +10,7 @@ import { Tag } from "../../Logic/Tags/Tag" import Hotkeys from "../../UI/Base/Hotkeys" import Translations from "../../UI/i18n/Translations" import { Feature } from "geojson" +import { OfflineForegroundDataManager } from "../../Logic/Actors/OfflineForegroundDataManager" export class WithLayoutSourceState extends WithSelectedElementState { readonly layerState: LayerState @@ -23,6 +24,8 @@ export class WithLayoutSourceState extends WithSelectedElementState { */ readonly floors: Store + readonly offlineForegroundDataManager: OfflineForegroundDataManager + constructor(theme: ThemeConfig, mvtAvailableLayers: Store>) { super(theme) /* Set up the layout source @@ -49,6 +52,7 @@ export class WithLayoutSourceState extends WithSelectedElementState { this.dataIsLoading = layoutSource.isLoading this.indexedFeatures = layoutSource this.featureProperties = new FeaturePropertiesStore(layoutSource) + this.offlineForegroundDataManager = new OfflineForegroundDataManager(this) this.floors = WithLayoutSourceState.initFloors(this.featuresInView) diff --git a/src/Models/TileRange.ts b/src/Models/TileRange.ts index a2b50dee2..8d7f0a424 100644 --- a/src/Models/TileRange.ts +++ b/src/Models/TileRange.ts @@ -26,6 +26,21 @@ export class Tiles { return result } + public static async mapRangeAsync(tileRange: TileRange, f: (x: number, y: number) => Promise): Promise { + const result: T[] = [] + const total = tileRange.total + if (total > 100000) { + throw `Tilerange too big (z is ${tileRange.zoomlevel}, total tiles needed: ${tileRange.total})` + } + for (let x = tileRange.xstart; x <= tileRange.xend; x++) { + for (let y = tileRange.ystart; y <= tileRange.yend; y++) { + const t = await f(x, y) + result.push(t) + } + } + return result + } + /** * Calculates the tile bounds of the * @param z @@ -63,7 +78,7 @@ export class Tiles { static centerPointOf(zOrId: number, x?: number, y?: number): [number, number] { let z: number if (x === undefined) { - ;[z, x, y] = Tiles.tile_from_index(zOrId) + [z, x, y] = Tiles.tile_from_index(zOrId) } else { z = zOrId } @@ -95,7 +110,7 @@ export class Tiles { static asGeojson(zIndex: number, x?: number, y?: number): Feature { let z = zIndex if (x === undefined) { - ;[z, x, y] = Tiles.tile_from_index(zIndex) + [z, x, y] = Tiles.tile_from_index(zIndex) } const bounds = Tiles.tile_bounds_lon_lat(z, x, y) return new BBox(bounds).asGeoJson() diff --git a/src/UI/BigComponents/MenuDrawerIndex.svelte b/src/UI/BigComponents/MenuDrawerIndex.svelte index b6a0465b0..98ceedc3c 100644 --- a/src/UI/BigComponents/MenuDrawerIndex.svelte +++ b/src/UI/BigComponents/MenuDrawerIndex.svelte @@ -137,7 +137,7 @@
- {$userdetails.name} + {$userdetails?.name}
@@ -201,7 +201,7 @@ - Manage offline basemap + Manage map data for offline use diff --git a/src/UI/BigComponents/OfflineForegroundManagement.svelte b/src/UI/BigComponents/OfflineForegroundManagement.svelte new file mode 100644 index 000000000..e09fe1f3a --- /dev/null +++ b/src/UI/BigComponents/OfflineForegroundManagement.svelte @@ -0,0 +1,166 @@ + + +
+
+ +
+
+ +
+
+{#if !$isOnline} +
Offline mode - cannot update now
+{:else if $isUpdating} + Updating data... +{:else} + +{/if} + + Automatically update the foreground data of marked areas when internet connection is connected again + + + + Automatically update the foreground data of marked areas when loading MapComplete + + + diff --git a/src/UI/BigComponents/OfflineManagement.svelte b/src/UI/BigComponents/OfflineManagement.svelte index 7d37a3e6c..fa728fe86 100644 --- a/src/UI/BigComponents/OfflineManagement.svelte +++ b/src/UI/BigComponents/OfflineManagement.svelte @@ -24,13 +24,15 @@ import { default as Trans } from "../Base/Tr.svelte" import AccordionSingle from "../Flowbite/AccordionSingle.svelte" import { Lists } from "../../Utils/Lists" + import OfflineForegroundManagement from "./OfflineForegroundManagement.svelte" + import { IsOnline } from "../../Logic/Web/IsOnline" export let state: ThemeViewState & SpecialVisualizationState = undefined export let autoDownload = state.autoDownloadOfflineBasemap let focusZ = Math.max(...Object.keys(OfflineBasemapManager.zoomelevels).map(Number)) let map: UIEventSource = new UIEventSource(undefined) - let mapProperties: MapProperties = new MapLibreAdaptor(map) + let mapProperties: MapProperties = new MapLibreAdaptor(map).installQuicklocation() state?.showCurrentLocationOn(map) mapProperties.maxzoom.set(focusZ - 1) mapProperties.zoom.set(Math.min(focusZ - 1, state.mapProperties.zoom.data)) @@ -77,9 +79,9 @@ id: "center_point_" + z + "_" + x + "_" + y, txt: "Tile " + x + " " + y, } - return [f] + return [>f] }) - let installedFeature: Store[]> = installed.map((meta) => + let installedFeature: Store[]> = installed.map((meta) => (meta ?? []).map((area) => { const f = Tiles.asGeojson(area.minzoom, area.x, area.y) f.properties = { @@ -92,11 +94,11 @@ " " + Utils.toHumanByteSize(Number(area.size)), } - return f + return >f }) ) new ShowDataLayer(map, { - features: new StaticFeatureSource(installedFeature), + features: new StaticFeatureSource>(installedFeature), layer: new LayerConfig({ id: "downloaded", source: "special", @@ -132,7 +134,7 @@ }), }) new ShowDataLayer(map, { - features: new StaticFeatureSource(focusTileFeature), + features: new StaticFeatureSource>(focusTileFeature), layer: new LayerConfig({ id: "focustile", source: "special", @@ -152,57 +154,62 @@ }), }) const t = Translations.t.offline + -
- - - - - -
- -
- -
+
+
+ +
{#if $installed === undefined} {:else}
- - -
-
- -
-
-
- {#if $focusTileIsInstalling} -
- - - -
- {:else} - - {/if} -
-
- - + The offline background map is the basemap that is shown as background under the clickable map features + (protomaps sunny). + The data for this map is shared between all MapComplete themes. + These are downloaded from {state.offlineMapManager.host} which has a weekly update schedule. + + + + + +
+ +
+ +
+
+
+ +
+
+
+ {#if $focusTileIsInstalling} +
+ + + +
+ {:else} + + {/if} +
+
+ {Utils.toHumanByteSize(Lists.sum($installed.map((area) => area.size)))} + {#if minzoom > $zoom} +
+ +
+ {:else if $areaContained} +
+ +
+ {:else} + + {/if}
-{#if !$isOnline} -
Offline mode - cannot update now
+
+ + {#if !$isOnline} +
+ +
{:else if $isUpdating} - Updating data... + + + {:else} - {/if} + + + +
- Automatically update the foreground data of marked areas when internet connection is connected again + - - Automatically update the foreground data of marked areas when loading MapComplete + - From ce435765f2885f1149bc82a11a34c4d5ff4cc94c Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Tue, 14 Oct 2025 22:04:40 +0200 Subject: [PATCH 3/4] Offline: add translation --- langs/en.json | 12 +++++++++++- src/UI/BigComponents/MenuDrawerIndex.svelte | 2 -- src/UI/BigComponents/OfflineManagement.svelte | 6 ++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/langs/en.json b/langs/en.json index 33a13b44a..5213c68c7 100644 --- a/langs/en.json +++ b/langs/en.json @@ -778,19 +778,29 @@ }, "offline": { "actions": "Actions", + "alreadyOffline": "This area is already offline", + "areaToBig": "This area is to big to keep offline", "autoCheckmark": "Automatically download the basemap when browsing around", "autoExplanation": "If checked, MapComplete will automatically download the basemap to the cache for the area. This results in bigger initial data loads, but requires less internet over the long run. If you plan to visit a region with less connectivity, you can also select the area you want to download below.", "autoExplanationIntro": "What does automatically downloading basemaps mean?", "date": "Map generation data", "delete": "Delete basemap", "deleteAll": "Delete all basemaps", + "deleteAreas": "Clear offline areas", "download": "Download area", "installing": "Data is being downloaded", + "intro": " The offline background map is the basemap that is shown as background under the clickable map features (protomaps sunny). he data for this map is shared between all MapComplete themes. These are downloaded from {host} which has a weekly update schedule.", "localOnMap": "Offline basemaps on the map", + "markArea": "Keep this area available offline", "name": "Name", + "offlineNoUpdate": "Updating is currently not possible as you are offline", "overview": "Offline background map", "range": "Zoom ranges", - "size": "Size" + "size": "Size", + "updateAll": "Update all offline areas", + "updateOnLoad": "Update the areas when loading", + "updateOnReconnect": "Update the available areas when connection is restored", + "updating": "Updating..." }, "plantDetection": { "back": "Back to species overview", diff --git a/src/UI/BigComponents/MenuDrawerIndex.svelte b/src/UI/BigComponents/MenuDrawerIndex.svelte index 4b522c3c9..644920f2f 100644 --- a/src/UI/BigComponents/MenuDrawerIndex.svelte +++ b/src/UI/BigComponents/MenuDrawerIndex.svelte @@ -55,8 +55,6 @@ import ImageUploadQueue from "../../Logic/ImageProviders/ImageUploadQueue" import QueuedImagesView from "../Image/QueuedImagesView.svelte" import InsetSpacer from "../Base/InsetSpacer.svelte" - import OfflineManagement from "./OfflineManagement.svelte" - import { GlobeEuropeAfrica } from "@babeard/svelte-heroicons/solid/GlobeEuropeAfrica" import { onDestroy } from "svelte" import Avatar from "../Base/Avatar.svelte" import { Changes } from "../../Logic/Osm/Changes" diff --git a/src/UI/BigComponents/OfflineManagement.svelte b/src/UI/BigComponents/OfflineManagement.svelte index 4690b595f..5c1ef5e7d 100644 --- a/src/UI/BigComponents/OfflineManagement.svelte +++ b/src/UI/BigComponents/OfflineManagement.svelte @@ -169,10 +169,8 @@ - The offline background map is the basemap that is shown as background under the clickable map features - (protomaps sunny). - The data for this map is shared between all MapComplete themes. - These are downloaded from {state.offlineMapManager.host} which has a weekly update schedule. + + From 7d420b07654ac7a18ecc6f07ab91845dafa158ad Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Wed, 15 Oct 2025 16:44:42 +0200 Subject: [PATCH 4/4] Fix tile generation, don't generate files from 100GB anymore --- android | 2 +- scripts/pmTilesExtractGenerator.ts | 7 ++++++- scripts/serverPmTileExtracts.ts | 9 +++++++-- src/Logic/Actors/OfflineForegroundDataManager.ts | 7 +++++-- src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts | 2 +- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/android b/android index dc3f3f5ac..472b6da88 160000 --- a/android +++ b/android @@ -1 +1 @@ -Subproject commit dc3f3f5ac3d4d42bed7aeb58ff5386a063dbac06 +Subproject commit 472b6da88e038d34e2915ebd36350376acf0c58c diff --git a/scripts/pmTilesExtractGenerator.ts b/scripts/pmTilesExtractGenerator.ts index 74ec5c0b2..50ab37b4a 100644 --- a/scripts/pmTilesExtractGenerator.ts +++ b/scripts/pmTilesExtractGenerator.ts @@ -46,12 +46,17 @@ export class PmTilesExtractGenerator { return `${this._targetDir}/${z}/${x}/${y}.pmtiles` } - async generateArchive(z: number, x: number, y: number, maxzoom?: number): Promise { + async generateArchive(z: number, x: number, y: number, maxzoom: number): Promise { const [[max_lat, min_lon], [min_lat, max_lon]] = Tiles.tile_bounds(z, x, y) let maxzoomflag = "" if (maxzoom !== undefined) { maxzoomflag = " --maxzoom=" + maxzoom } + + if(!maxzoom && z < 15){ + throw "No maxzoom for a pretty low zoom detected. This will result in a big archive" + } + const outputFileName = this.getFilename(z, x, y) await this.startProcess( `extract ${ diff --git a/scripts/serverPmTileExtracts.ts b/scripts/serverPmTileExtracts.ts index 14ce96539..5c77f22f9 100644 --- a/scripts/serverPmTileExtracts.ts +++ b/scripts/serverPmTileExtracts.ts @@ -66,7 +66,8 @@ class ServerPmTileExtracts extends Script { ScriptUtils.createParentDir(targetFile) console.log("Creating", targetFile) const start = new Date() - await generator.generateArchive(z, x, y) + const maxzoom = OfflineBasemapManager.zoomelevels[z] + await generator.generateArchive(z, x, y, maxzoom) const stop = new Date() console.log( "Creating ", @@ -82,7 +83,11 @@ class ServerPmTileExtracts extends Script { if (req.destroyed) { return null } - res.writeHead(200, { "Content-Type": "application/octet-stream" }) + const stats = statSync(targetFile) + + res.writeHead(200, { "Content-Type": "application/octet-stream" , + "Content-Length": stats.size + }) Server.sendFile(targetFile, res) return null diff --git a/src/Logic/Actors/OfflineForegroundDataManager.ts b/src/Logic/Actors/OfflineForegroundDataManager.ts index dc079610c..34bd67743 100644 --- a/src/Logic/Actors/OfflineForegroundDataManager.ts +++ b/src/Logic/Actors/OfflineForegroundDataManager.ts @@ -73,7 +73,6 @@ export class OfflineForegroundDataManager { } this._bboxesForOffline.data.push(data) this._bboxesForOffline.update(offl => OfflineForegroundDataManager.clean(offl)) - // this._bboxesForOffline.update(data => OfflineForegroundDataManager.clean(data)) this._isUpdating.set(true) await this.updateSingleBbox(data) this._isUpdating.set(false) @@ -100,7 +99,6 @@ export class OfflineForegroundDataManager { // As 'offlBbox' is included in the _bboxForOffline, pinging it will update downstream UI elements console.log(">>> Updating bbox preping", offlBbox, this._bboxesForOffline.data.indexOf(offlBbox)) this._bboxesForOffline.ping() - console.log(">>> Updating bbox", offlBbox, this._bboxesForOffline.data.indexOf(offlBbox)) try { await OfflineBasemapManager.singleton.installBbox(bbox) console.log(">>> BBox update started") @@ -117,6 +115,11 @@ export class OfflineForegroundDataManager { console.log(">>> bboxes for offline are now:", this._bboxesForOffline.data) } + /** + * Merges fully contained bboxes into the bigger bbox + * @param bboxes + * @private + */ private static clean(bboxes: OfflineBbox[]): OfflineBbox[] { const validated = bboxes.filter(bbox => { try { diff --git a/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts b/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts index a4aaafb17..f07ba22fd 100644 --- a/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts +++ b/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts @@ -145,7 +145,7 @@ export default class OverpassFeatureSource } catch (e) { this.retries.update(i => i + 1) console.error(`QUERY FAILED (attempt ${this.retries.data}, will retry: ${this._isActive.data}) due to`, e) - if(options.noRetries){ + if(options?.noRetries){ throw e } await Utils.waitFor(1000)