ilja_space/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Fix: change inner workings of how history is kept, should improve performance

Browse source
This commit is contained in:
Pieter Vander Vennet 2025-04-01 14:10:13 +02:00
parent 27ae31afe7
commit 7c57047d30
3 changed files with 102 additions and 91 deletions
Download patch file Download diff file Expand all files Collapse all files

55
src/Logic/Osm/OsmPreferences.ts
View file

@ -80,17 +80,10 @@ export class OsmPreferences {
private async loadBulkPreferences() {
const prefs = await this.getPreferencesDictDirectly()
this.seenKeys = Object.keys(prefs)
const legacy = OsmPreferences.getLegacyCombinedItems(prefs)
const merged = OsmPreferences.mergeDict(prefs)
if (Object.keys(legacy).length > 0) {
await this.removeLegacy(legacy)
}
for (const key in merged) {
this.initPreference(key, prefs[key], true)
}
for (const key in legacy) {
this.initPreference(key, legacy[key], true)
}
this._allPreferences.ping()
}
@ -124,6 +117,7 @@ export class OsmPreferences {
key,
localStorage.data ?? defaultValue
) // cached
if (this.localStorageInited.has(key)) {
return pref
}
@ -140,15 +134,6 @@ export class OsmPreferences {
this.removeAllWithPrefix("")
}
public async removeLegacy(legacyDict: Record<string, string>) {
for (const k in legacyDict) {
const v = legacyDict[k]
console.log("Upgrading legacy preference", k)
await this.removeAllWithPrefix(k)
this.osmConnection.getPreference(k).set(v)
}
}
/**
*
* OsmPreferences.mergeDict({abc: "123", def: "123", "def:0": "456", "def:1":"789"}) // => {abc: "123", def: "123456789"}
@ -160,7 +145,6 @@ export class OsmPreferences {
const normalKeys = allKeys.filter((k) => !k.match(/[a-z-_0-9A-Z]*:[0-9]+/))
for (const normalKey of normalKeys) {
if (normalKey.match(/-combined-[0-9]*$/) || normalKey.match(/-combined-length$/)) {
// Ignore legacy keys
continue
}
const partKeys = OsmPreferences.keysStartingWith(allKeys, normalKey)
@ -171,42 +155,7 @@ export class OsmPreferences {
}
/**
* Gets all items which have a 'combined'-string, the legacy long preferences
*
* const input = {
* "extra-noncombined-key":"xyz",
* "mapcomplete-unofficial-theme-httpsrawgithubusercontentcomosm-catalanwikidataimgmainwikidataimgjson-combined-0":
* "{\"id\":\"https://raw.githubusercontent.com/osm-catalan/wikidataimg/main/wikidataimg.json\",\"icon\":\"https://upload.wikimedia.org/wikipedia/commons/5/50/Yes_Check_Circle.svg\",\"title\":{\"ca\":\"wikidataimg\",\"_context\":\"themes:wikidataimg.title\"},\"shortDescription\"",
* "mapcomplete-unofficial-theme-httpsrawgithubusercontentcomosm-catalanwikidataimgmainwikidataimgjson-combined-1":
* ":{\"ca\":\"Afegeix imatges d'articles de wikimedia\",\"_context\":\"themes:wikidataimg\"}}",
* }
* const merged = OsmPreferences.getLegacyCombinedItems(input)
* const data = merged["mapcomplete-unofficial-theme-httpsrawgithubusercontentcomosm-catalanwikidataimgmainwikidataimgjson"]
* JSON.parse(data) // => {"id": "https://raw.githubusercontent.com/osm-catalan/wikidataimg/main/wikidataimg.json", "icon": "https://upload.wikimedia.org/wikipedia/commons/5/50/Yes_Check_Circle.svg","title": { "ca": "wikidataimg", "_context": "themes:wikidataimg.title" }, "shortDescription": {"ca": "Afegeix imatges d'articles de wikimedia","_context": "themes:wikidataimg"}}
* merged["extra-noncombined-key"] // => undefined
*/
public static getLegacyCombinedItems(dict: Record<string, string>): Record<string, string> {
const merged: Record<string, string> = {}
const keys = Object.keys(dict)
const toCheck = Utils.NoNullInplace(
Utils.Dedup(keys.map((k) => k.match(/(.*)-combined-[0-9]+$/)?.[1]))
)
for (const key of toCheck) {
let i = 0
let str = ""
let v: string
do {
v = dict[key + "-combined-" + i]
str += v ?? ""
i++
} while (v !== undefined)
merged[key] = str
}
return merged
}
/**
* Bulk-downloads all preferences
* Bulk-downloads all preferences, creates a simple record from all
* @private
*/
private async getPreferencesDictDirectly(): Promise<Record<string, string>> {

134
src/Logic/State/UserRelatedState.ts
View file

@ -21,10 +21,49 @@ import Showdown from "showdown"
import { LocalStorageSource } from "../Web/LocalStorageSource"
import { GeocodeResult } from "../Search/GeocodingProvider"
export class OptionallySyncedHistory<T> {
class RoundRobinStore<T> {
private readonly _store: UIEventSource<T[]>
private readonly _index: UIEventSource<number>
private readonly _value: UIEventSource<T[]>
public readonly value: Store<T[]>
constructor(store: UIEventSource<T[]>, index: UIEventSource<number>) {
this._store = store
this._index = index
this._value = new UIEventSource([])
this.value = this._value
this._store.addCallbackD(() => this.set())
this.set()
}
private set() {
const v = this._store.data
const i = this._index.data
const newList = Utils.NoNull(v.slice(i + 1, v.length).concat(v.slice(0, i + 1)))
this._value.set(newList)
}
/**
* Add a value to the underlying store
* @param t
*/
public add(t: T) {
const i = this._index.data
this._index.set((i + 1) % this._store.data.length)
this._store.data[i] = t
console.trace(">>> Setting", i, this._store.data, t, "new index:", this._index.data)
this._store.ping()
}
}
export class OptionallySyncedHistory<T extends object | string> {
public readonly syncPreference: UIEventSource<"sync" | "local" | "no">
public readonly value: Store<T[]>
private readonly synced: UIEventSource<T[]>
private readonly syncedBackingStore: UIEventSource<T[]>
private readonly syncedOrdered: RoundRobinStore<T>
private readonly local: UIEventSource<T[]>
private readonly thisSession: UIEventSource<T[]>
private readonly _maxHistory: number
@ -41,10 +80,14 @@ export class OptionallySyncedHistory<T> {
this._maxHistory = maxHistory
this._isSame = isSame
this.syncPreference = osmconnection.getPreference("preference-" + key + "-history", "sync")
const synced = (this.synced = UIEventSource.asObject<T[]>(
osmconnection.getPreference(key + "-history"),
[]
))
this.syncedBackingStore = Stores.fromArray(
Utils.TimesT(maxHistory, (i) =>
UIEventSource.asObject<T>(osmconnection.getPreference(key + "-history-" + i), undefined)
))
this.syncedOrdered = new RoundRobinStore<T>(this.syncedBackingStore,
UIEventSource.asInt(osmconnection.getPreference(key + "-history-round-robin", "0"))
)
const local = (this.local = LocalStorageSource.getParsed<T[]>(key + "-history", []))
const thisSession = (this.thisSession = new UIEventSource<T[]>(
[],
@ -52,7 +95,7 @@ export class OptionallySyncedHistory<T> {
))
this.syncPreference.addCallback((syncmode) => {
if (syncmode === "sync") {
const list = [...thisSession.data, ...synced.data].slice(0, maxHistory)
const list = [...thisSession.data, ...this.syncedOrdered.value.data].slice(0, maxHistory)
if (this._isSame) {
for (let i = 0; i < list.length; i++) {
for (let j = i + 1; j < list.length; j++) {
@ -62,12 +105,12 @@ export class OptionallySyncedHistory<T> {
}
}
}
synced.set(list)
this.syncedBackingStore.set(list)
} else if (syncmode === "local") {
local.set(synced.data?.slice(0, maxHistory))
synced.set([])
local.set(this.syncedOrdered.value.data?.slice(0, maxHistory))
this.syncedBackingStore.set([])
} else {
synced.set([])
this.syncedBackingStore.set([])
local.set([])
}
})
@ -75,10 +118,10 @@ export class OptionallySyncedHistory<T> {
this.value = this.syncPreference.bind((syncPref) => this.getAppropriateStore(syncPref))
}
private getAppropriateStore(syncPref?: string) {
private getAppropriateStore(syncPref?: string): Store<T[]> {
syncPref ??= this.syncPreference.data
if (syncPref === "sync") {
return this.synced
return this.syncedOrdered.value
}
if (syncPref === "local") {
return this.local
@ -87,12 +130,29 @@ export class OptionallySyncedHistory<T> {
}
public add(t: T) {
const store = this.getAppropriateStore()
let oldList = store.data ?? []
if (this._isSame) {
oldList = oldList.filter((x) => !this._isSame(t, x))
const alreadyNoted = this.getAppropriateStore().data.some(item => this._isSame(item, t))
if (alreadyNoted) {
return
}
}
if (this.syncPreference.data === "local") {
const ls = this.local.data
ls.unshift(t)
if (ls.length >= this._maxHistory) {
ls.splice(this._maxHistory, 1)
}
this.local.ping()
} else if (this.syncPreference.data === "sync") {
this.osmconnection.isLoggedIn.addCallbackAndRun(loggedIn => {
// Wait until we are logged in and the settings are downloaded before adding the preference
if (loggedIn) {
this.syncedOrdered.add(t)
return true
}
})
}
store.set([t, ...oldList].slice(0, this._maxHistory))
}
/**
@ -113,7 +173,8 @@ export class OptionallySyncedHistory<T> {
}
clear() {
this.getAppropriateStore().set([])
this.syncedBackingStore.set([])
this.local.set([])
}
}
@ -229,6 +290,7 @@ export default class UserRelatedState {
10,
(a, b) => a === b
)
this.recentlyVisitedThemes.value.addCallbackAndRunD(th => console.log(">>> themes", th))
this.recentlyVisitedSearch = new OptionallySyncedHistory<GeocodeResult>(
"places",
this.osmConnection,
@ -298,10 +360,10 @@ export default class UserRelatedState {
try {
return JSON.parse(str)
} catch (e) {
console.warn(
e.warn(
"Removing theme " +
id +
" as it could not be parsed from the preferences; the content is:",
id +
" as it could not be parsed from the preferences; the content is:",
str
)
pref.setData(null)
@ -331,7 +393,7 @@ export default class UserRelatedState {
icon: layout.icon,
title: layout.title.translations,
shortDescription: layout.shortDescription.translations,
definition: layout["definition"],
definition: layout["definition"]
})
)
}
@ -386,13 +448,13 @@ export default class UserRelatedState {
id: "home",
"user:home": "yes",
_lon: homeLonLat[0],
_lat: homeLonLat[1],
_lat: homeLonLat[1]
},
geometry: {
type: "Point",
coordinates: homeLonLat,
},
},
coordinates: homeLonLat
}
}
]
})
return new StaticFeatureSource(feature)
@ -414,7 +476,7 @@ export default class UserRelatedState {
_applicationOpened: new Date().toISOString(),
_supports_sharing:
typeof window === "undefined" ? "no" : window.navigator.share ? "yes" : "no",
_iframe: Utils.isIframe ? "yes" : "no",
_iframe: Utils.isIframe ? "yes" : "no"
})
if (!Utils.runningFromConsole) {
amendedPrefs.data["_host"] = window.location.host
@ -462,18 +524,18 @@ export default class UserRelatedState {
const zenLinks: { link: string; id: string }[] = Utils.NoNull([
hasMissingTheme
? {
id: "theme:" + layout.id,
link: LinkToWeblate.hrefToWeblateZen(
language,
"themes",
layout.id
),
}
id: "theme:" + layout.id,
link: LinkToWeblate.hrefToWeblateZen(
language,
"themes",
layout.id
)
}
: undefined,
...missingLayers.map((id) => ({
id: "layer:" + id,
link: LinkToWeblate.hrefToWeblateZen(language, "layers", id),
})),
link: LinkToWeblate.hrefToWeblateZen(language, "layers", id)
}))
])
const untranslated_count = untranslated.length
amendedPrefs.data["_translation_total"] = "" + total

4
src/Logic/UIEventSource.ts
View file

@ -807,7 +807,7 @@ export class UIEventSource<T> extends Store<T> implements Writable<T> {
)
}
static asObject<T extends object>(
static asObject<T extends object | string>(
stringUIEventSource: UIEventSource<string>,
defaultV: T
): UIEventSource<T> {
@ -819,7 +819,7 @@ export class UIEventSource<T> extends Store<T> implements Writable<T> {
try {
return <T>JSON.parse(str)
} catch (e) {
console.error("Could not parse value", str, "due to", e)
console.error("Could not parse value", str, "due to", e, "; the underlying data store has tag", stringUIEventSource.tag)
return defaultV
}
},