diff --git a/scripts/Script.ts b/scripts/Script.ts index 65b2fcfa3..ee71041a9 100644 --- a/scripts/Script.ts +++ b/scripts/Script.ts @@ -24,7 +24,7 @@ export default abstract class Script { }) .catch((e) => { console.log(`ERROR in script ${process.argv[1]}:`, e) - process.exit(1) + // process.exit(1) }) } diff --git a/src/Logic/Osm/OsmConnection.ts b/src/Logic/Osm/OsmConnection.ts index 1f98c6015..03ab61fdf 100644 --- a/src/Logic/Osm/OsmConnection.ts +++ b/src/Logic/Osm/OsmConnection.ts @@ -154,7 +154,6 @@ export class OsmConnection { console.log("Not authenticated") } } - public GetPreference( key: string, defaultValue: string = undefined, @@ -162,12 +161,20 @@ export class OsmConnection { prefix?: string } ): UIEventSource { - options ??= {prefix: "mapcomplete-"} - return >this.preferencesHandler.GetPreference(key, defaultValue, options) + const prefix =options?.prefix ?? "mapcomplete-" + return >this.preferencesHandler.getPreference(key, defaultValue, prefix) + + } + public getPreference( + key: string, + defaultValue: string = undefined, + prefix: string = "mapcomplete-" + ): UIEventSource { + return >this.preferencesHandler.getPreference(key, defaultValue, prefix) } public GetLongPreference(key: string, prefix: string = "mapcomplete-"): UIEventSource { - return this.preferencesHandler.GetLongPreference(key, prefix) + return this.preferencesHandler.getPreference(key, prefix) } public OnLoggedIn(action: (userDetails: UserDetails) => void) { diff --git a/src/Logic/Osm/OsmPreferences.ts b/src/Logic/Osm/OsmPreferences.ts index d076e09ee..c8f69f3ed 100644 --- a/src/Logic/Osm/OsmPreferences.ts +++ b/src/Logic/Osm/OsmPreferences.ts @@ -2,12 +2,18 @@ import { Store, UIEventSource } from "../UIEventSource" import { OsmConnection } from "./OsmConnection" import { LocalStorageSource } from "../Web/LocalStorageSource" import OSMAuthInstance = OSMAuth.osmAuth +import { Utils } from "../../Utils" export class OsmPreferences { - private normalPreferences: Record> = {} - private longPreferences: Record> = {} + private preferences: Record> = {} + private localStorageInited: Set = new Set() + /** + * Contains all the keys as returned by the OSM-preferences. + * Used to clean up old preferences + */ + private seenKeys: string[] = [] private readonly _allPreferences: UIEventSource> = new UIEventSource({}) public readonly allPreferences: Store>> = this._allPreferences @@ -25,15 +31,6 @@ export class OsmPreferences { }) } - private getLongValue(allPrefs: Record, key: string): string { - const count = Number(allPrefs[key + "-length"]) - let str = "" - for (let i = 0; i < count; i++) { - str += allPrefs[key + i] - } - return str - - } private setPreferencesAll(key: string, value: string) { if (this._allPreferences.data[key] !== value) { @@ -42,98 +39,50 @@ export class OsmPreferences { } } - private initPreference(key: string, value: string = "", excludeFromAll: boolean = false): UIEventSource { - if (this.normalPreferences[key] !== undefined) { - return this.normalPreferences[key] + private initPreference(key: string, value: string = ""): UIEventSource { + if (this.preferences[key] !== undefined) { + return this.preferences[key] } - const pref = this.normalPreferences[key] = new UIEventSource(value, "preference: " + key) - if(value && !excludeFromAll){ + const pref = this.preferences[key] = new UIEventSource(value, "preference: " + key) + if (value) { this.setPreferencesAll(key, value) } pref.addCallback(v => { - this.UploadPreference(key, v) - if(!excludeFromAll){ - this.setPreferencesAll(key, v) - } - }) - return pref - } - - private initLongPreference(key: string, initialValue: string): UIEventSource { - if (this.longPreferences[key] !== undefined) { - return this.longPreferences[key] - } - const pref = this.longPreferences[key] = new UIEventSource(initialValue, "long-preference-"+key) - const maxLength = 255 - const length = UIEventSource.asInt(this.initPreference(key + "-length", "0", true)) - if(initialValue){ - this.setPreferencesAll(key, initialValue) - } - pref.addCallback(v => { - if(v === null || v === undefined || v === ""){ - length.set(null) - return - } - length.set(Math.ceil((v?.length ?? 1) / maxLength)) - let i = 0 - while (v.length > 0) { - this.UploadPreference(key + "-" + i, v.substring(0, maxLength)) - i++ - v = v.substring(maxLength) - } + this.uploadKvSplit(key, v) this.setPreferencesAll(key, v) }) return pref } private async loadBulkPreferences() { - const prefs = await this.getPreferencesDict() - const isCombined = /-combined-/ - for (const key in prefs) { - if (key.endsWith("-combined-length")) { - const v = this.getLongValue(prefs, key.substring(0, key.length - "-length".length)) - this.initLongPreference(key, v) - } - if (key.match(isCombined)) { - continue - } + const prefs = await this.getPreferencesDictDirectly() + this.seenKeys = Object.keys(prefs) + const legacy = OsmPreferences.getLegacyCombinedItems(prefs) + const merged = OsmPreferences.mergeDict(prefs) + for (const key in merged) { this.initPreference(key, prefs[key]) } + for (const key in legacy) { + this.initPreference(key, legacy[key]) + } } - /** - * OSM preferences can be at most 255 chars. - * This method chains multiple together. - * Values written into this key will be erased when the user logs in - */ - public GetLongPreference(key: string, prefix: string = "mapcomplete-"): UIEventSource { - return this.getPreferenceSeedFromlocal(key, true, undefined, { prefix }) - } - - public GetPreference( + public getPreference( key: string, defaultValue: string = undefined, - options?: { - documentation?: string - prefix?: string - }, + prefix?: string, ) { - return this.getPreferenceSeedFromlocal(key, false, defaultValue, options) + return this.getPreferenceSeedFromlocal(key, defaultValue, { prefix }) } /** * Gets a OSM-preference. * The OSM-preference is cached in local storage and updated from the OSM.org as soon as those values come in. * THis means that values written before being logged in might be erased by the cloud settings - * @param key - * @param defaultValue - * @param options - * @constructor */ private getPreferenceSeedFromlocal( key: string, - long: boolean, defaultValue: string = undefined, options?: { prefix?: string, @@ -146,17 +95,11 @@ export class OsmPreferences { key = key.replace(/[:/"' {}.%\\]/g, "") - let pref : UIEventSource const localStorage = LocalStorageSource.Get(key) - if(localStorage.data === "null" || localStorage.data === "undefined"){ + if (localStorage.data === "null" || localStorage.data === "undefined") { localStorage.set(undefined) } - if(long){ - pref = this.initLongPreference(key, localStorage.data ?? defaultValue) - }else{ - pref = this.initPreference(key, localStorage.data ?? defaultValue) - } - + let pref: UIEventSource = this.initPreference(key, localStorage.data ?? defaultValue) if (this.localStorageInited.has(key)) { return pref } @@ -173,12 +116,67 @@ export class OsmPreferences { this.removeAllWithPrefix("") } + /** + * + * OsmPreferences.mergeDict({abc: "123", def: "123", "def:0": "456", "def:1":"789"}) // => {abc: "123", def: "123456789"} + */ + private static mergeDict(dict: Record): Record { + const newDict = {} + + const allKeys: string[] = Object.keys(dict) + const normalKeys = allKeys.filter(k => !k.match(/[a-z-_0-9A-Z]*:[0-9]+/)) + for (const normalKey of normalKeys) { + if(normalKey.match(/-combined-[0-9]*$/) || normalKey.match(/-combined-length$/)){ + // Ignore legacy keys + continue + } + const partKeys = OsmPreferences.keysStartingWith(allKeys, normalKey) + const parts = partKeys.map(k => dict[k]) + newDict[normalKey] = parts.join("") + } + return newDict + } + + /** + * Gets all items which have a 'combined'-string, the legacy long preferences + * + * const input = { + * "extra-noncombined-key":"xyz", + * "mapcomplete-unofficial-theme-httpsrawgithubusercontentcomosm-catalanwikidataimgmainwikidataimgjson-combined-0": + * "{\"id\":\"https://raw.githubusercontent.com/osm-catalan/wikidataimg/main/wikidataimg.json\",\"icon\":\"https://upload.wikimedia.org/wikipedia/commons/5/50/Yes_Check_Circle.svg\",\"title\":{\"ca\":\"wikidataimg\",\"_context\":\"themes:wikidataimg.title\"},\"shortDescription\"", + * "mapcomplete-unofficial-theme-httpsrawgithubusercontentcomosm-catalanwikidataimgmainwikidataimgjson-combined-1": + * ":{\"ca\":\"Afegeix imatges d'articles de wikimedia\",\"_context\":\"themes:wikidataimg\"}}", + * } + * const merged = OsmPreferences.getLegacyCombinedItems(input) + * const data = merged["mapcomplete-unofficial-theme-httpsrawgithubusercontentcomosm-catalanwikidataimgmainwikidataimgjson"] + * JSON.parse(data) // => {"id": "https://raw.githubusercontent.com/osm-catalan/wikidataimg/main/wikidataimg.json", "icon": "https://upload.wikimedia.org/wikipedia/commons/5/50/Yes_Check_Circle.svg","title": { "ca": "wikidataimg", "_context": "themes:wikidataimg.title" }, "shortDescription": {"ca": "Afegeix imatges d'articles de wikimedia","_context": "themes:wikidataimg"}} + * merged["extra-noncombined-key"] // => undefined + */ + public static getLegacyCombinedItems(dict: Record): Record { + const merged: Record = {} + const keys = Object.keys(dict) + const toCheck =Utils.NoNullInplace( Utils.Dedup(keys.map(k => k.match(/(.*)-combined-[0-9]+$/)?.[1]))) + for (const key of toCheck) { + let i = 0 + let str = "" + let v: string + do { + v = dict[key + "-combined-" + i] + str += v ?? "" + i++ + } while (v !== undefined) + merged[key] = str + } + + + return merged + } /** * Bulk-downloads all preferences * @private */ - private getPreferencesDict(): Promise> { + private getPreferencesDictDirectly(): Promise> { return new Promise>((resolve, reject) => { this.auth.xhr( { @@ -206,10 +204,86 @@ export class OsmPreferences { } /** - * UPloads the given k=v to the OSM-server + * Returns all keys matching `k:[number]` + * Split separately for test + * + * const keys = ["abc", "def", "ghi", "ghi:0", "ghi:1"] + * OsmPreferences.keysStartingWith(keys, "xyz") // => [] + * OsmPreferences.keysStartingWith(keys, "abc") // => ["abc"] + * OsmPreferences.keysStartingWith(keys, "ghi") // => ["ghi", "ghi:0", "ghi:1"] + * + */ + private static keysStartingWith(allKeys: string[], key: string): string[] { + const keys = allKeys.filter(k => k === key || k.match(new RegExp(key + ":[0-9]+"))) + keys.sort() + return keys + } + + /** + * Smart 'upload', which splits the value into `k`, `k:0`, `k:1` if needed. + * If `v` is null, undefined, empty, "undefined" (literal string) or "null" (literal string), will delete `k` and `k:[number]` + * + */ + private async uploadKvSplit(k: string, v: string) { + + if (v === null || v === undefined || v === "" || v === "undefined" || v === "null") { + const keysToDelete = OsmPreferences.keysStartingWith(this.seenKeys, k) + await Promise.all(keysToDelete.map(k => this.deleteKeyDirectly(k))) + return + } + + + await this.uploadKeyDirectly(k, v.slice(0, 255)) + v = v.slice(255) + let i = 0 + while (v.length > 0) { + await this.uploadKeyDirectly(`${k}:${i}`, v.slice(0, 255)) + v = v.slice(255) + } + + } + + /** + * Directly deletes this key + * @param k + * @private + */ + private deleteKeyDirectly(k: string) { + if (!this.osmConnection.userDetails.data.loggedIn) { + console.debug(`Not saving preference ${k}: user not logged in`) + return + } + + if (this._fakeUser) { + return + } + return new Promise((resolve, reject) => { + + this.auth.xhr( + { + method: "DELETE", + path: "/api/0.6/user/preferences/" + encodeURIComponent(k), + headers: { "Content-Type": "text/plain" }, + }, + (error) => { + if (error) { + console.warn("Could not remove preference", error) + reject(error) + return + } + console.debug("Preference ", k, "removed!") + resolve() + }, + ) + }, + ) + } + + /** + * Uploads the given k=v to the OSM-server * Deletes it if 'v' is undefined, null or empty */ - private UploadPreference(k: string, v: string) { + private async uploadKeyDirectly(k: string, v: string) { if (!this.osmConnection.userDetails.data.loggedIn) { console.debug(`Not saving preference ${k}: user not logged in`) return @@ -219,62 +293,49 @@ export class OsmPreferences { return } if (v === undefined || v === "" || v === null) { - this.auth.xhr( - { - method: "DELETE", - path: "/api/0.6/user/preferences/" + encodeURIComponent(k), - headers: { "Content-Type": "text/plain" }, - }, - (error) => { - if (error) { - console.warn("Could not remove preference", error) - return - } - console.debug("Preference ", k, "removed!") - }, - ) + await this.deleteKeyDirectly(k) return } - this.auth.xhr( - { - method: "PUT", - path: "/api/0.6/user/preferences/" + encodeURIComponent(k), - headers: { "Content-Type": "text/plain" }, - content: v, - }, - (error) => { - if (error) { - console.warn(`Could not set preference "${k}"'`, error) - return - } - }, - ) + if (v.length > 255) { + console.error("Preference too long, max 255 chars", { k, v }) + throw "Preference too long, at most 255 characters are supported" + } + + return new Promise((resolve, reject) => { + + this.auth.xhr( + { + method: "PUT", + path: "/api/0.6/user/preferences/" + encodeURIComponent(k), + headers: { "Content-Type": "text/plain" }, + content: v, + }, + (error) => { + if (error) { + console.warn(`Could not set preference "${k}"'`, error) + reject(error) + return + } + resolve() + }, + ) + }) } - removeAllWithPrefix(prefix: string) { - for (const key in this.normalPreferences) { - if(key.startsWith(prefix)){ - this.normalPreferences[key].set(null) - } - } - for (const key in this.longPreferences) { - if(key.startsWith(prefix)){ - this.longPreferences[key].set(null) - } + async removeAllWithPrefix(prefix: string) { + const keys = this.seenKeys + for (const key in keys) { + await this.deleteKeyDirectly(key) } } - getExistingPreference(key: string, defaultValue: undefined, prefix: string ): UIEventSource { + getExistingPreference(key: string, defaultValue: undefined, prefix: string): UIEventSource { if (prefix) { key = prefix + key } key = key.replace(/[:/"' {}.%\\]/g, "") - - if(this.normalPreferences[key]){ - return this.normalPreferences[key] - } - return this.longPreferences[key] + return this.preferences[key] } } diff --git a/src/Logic/State/FeatureSwitchState.ts b/src/Logic/State/FeatureSwitchState.ts index ad3d0458c..054d9c841 100644 --- a/src/Logic/State/FeatureSwitchState.ts +++ b/src/Logic/State/FeatureSwitchState.ts @@ -6,7 +6,6 @@ import { UIEventSource } from "../UIEventSource" import { QueryParameters } from "../Web/QueryParameters" import Constants from "../../Models/Constants" import { Utils } from "../../Utils" -import { Query } from "pg" import { eliCategory } from "../../Models/RasterLayerProperties" import { AvailableRasterLayers } from "../../Models/RasterLayers" import MarkdownUtils from "../../Utils/MarkdownUtils" diff --git a/src/Logic/State/UserRelatedState.ts b/src/Logic/State/UserRelatedState.ts index 69a977a5e..c2e78ffbb 100644 --- a/src/Logic/State/UserRelatedState.ts +++ b/src/Logic/State/UserRelatedState.ts @@ -38,7 +38,7 @@ export class OptionallySyncedHistory { this.osmconnection = osmconnection this._maxHistory = maxHistory this._isSame = isSame - this.syncPreference = osmconnection.GetPreference( + this.syncPreference = osmconnection.getPreference( "preference-" + key + "-history", "sync", ) @@ -189,28 +189,28 @@ export default class UserRelatedState { this._mapProperties = mapProperties this.showAllQuestionsAtOnce = UIEventSource.asBoolean( - this.osmConnection.GetPreference("show-all-questions", "false"), + this.osmConnection.getPreference("show-all-questions", "false"), ) - this.language = this.osmConnection.GetPreference("language") - this.showTags = this.osmConnection.GetPreference("show_tags") - this.showCrosshair = this.osmConnection.GetPreference("show_crosshair") - this.fixateNorth = this.osmConnection.GetPreference("fixate-north") - this.morePrivacy = this.osmConnection.GetPreference("more_privacy", "no") + this.language = this.osmConnection.getPreference("language") + this.showTags = this.osmConnection.getPreference("show_tags") + this.showCrosshair = this.osmConnection.getPreference("show_crosshair") + this.fixateNorth = this.osmConnection.getPreference("fixate-north") + this.morePrivacy = this.osmConnection.getPreference("more_privacy", "no") - this.a11y = this.osmConnection.GetPreference("a11y") + this.a11y = this.osmConnection.getPreference("a11y") this.mangroveIdentity = new MangroveIdentity( - this.osmConnection.GetLongPreference("identity", "mangrove"), - this.osmConnection.GetPreference("identity-creation-date", "mangrove"), + this.osmConnection.getPreference("identity", undefined,"mangrove"), + this.osmConnection.getPreference("identity-creation-date", undefined,"mangrove"), ) - this.preferredBackgroundLayer = this.osmConnection.GetPreference("preferred-background-layer") + this.preferredBackgroundLayer = this.osmConnection.getPreference("preferred-background-layer") - this.addNewFeatureMode = this.osmConnection.GetPreference( + this.addNewFeatureMode = this.osmConnection.getPreference( "preferences-add-new-mode", "button_click_right", ) - this.imageLicense = this.osmConnection.GetPreference("pictures-license", "CC0") + this.imageLicense = this.osmConnection.getPreference("pictures-license", "CC0") this.installedUserThemes = UserRelatedState.initInstalledUserThemes(osmConnection) this.translationMode = this.initTranslationMode() this.homeLocation = this.initHomeLocation() @@ -242,7 +242,7 @@ export default class UserRelatedState { private initTranslationMode(): UIEventSource<"false" | "true" | "mobile" | undefined | string> { const translationMode: UIEventSource = - this.osmConnection.GetPreference("translation-mode", "false") + this.osmConnection.getPreference("translation-mode", "false") translationMode.addCallbackAndRunD((mode) => { mode = mode.toLowerCase() if (mode === "true" || mode === "yes") { @@ -301,7 +301,7 @@ export default class UserRelatedState { this.osmConnection.isLoggedIn.addCallbackAndRunD((loggedIn) => { if (loggedIn) { this.osmConnection - .GetPreference("hidden-theme-" + layout?.id + "-enabled") + .getPreference("hidden-theme-" + layout?.id + "-enabled") .setData("true") return true } @@ -516,10 +516,8 @@ export default class UserRelatedState { if(tags[key] === null){ continue } - let pref = this.osmConnection.preferencesHandler.getExistingPreference(key, undefined, "") - if (!pref) { - pref = this.osmConnection.GetPreference(key, undefined, {prefix: ""}) - } + let pref = this.osmConnection.GetPreference(key, undefined, {prefix: ""}) + pref.set(tags[key]) } }) diff --git a/src/Logic/Web/MangroveReviews.ts b/src/Logic/Web/MangroveReviews.ts index bee5f9788..6765d8204 100644 --- a/src/Logic/Web/MangroveReviews.ts +++ b/src/Logic/Web/MangroveReviews.ts @@ -22,6 +22,9 @@ export class MangroveIdentity { this.mangroveIdentity = mangroveIdentity this._mangroveIdentityCreationDate = mangroveIdentityCreationDate mangroveIdentity.addCallbackAndRunD(async (data) => { + if(data === ""){ + return + } await this.setKeypair(data) }) } diff --git a/src/UI/AllThemesGui.svelte b/src/UI/AllThemesGui.svelte index 1642305ce..0ea38fccf 100644 --- a/src/UI/AllThemesGui.svelte +++ b/src/UI/AllThemesGui.svelte @@ -51,17 +51,17 @@ knownIds.indexOf(theme.id) >= 0 || state.osmConnection.userDetails.data.name === "Pieter Vander Vennet" )) - const customThemes: Store = Stores.ListStabilized(state.installedUserThemes) - .mapD(stableIds => stableIds.map(id => state.getUnofficialTheme(id))) - + .mapD(stableIds => Utils.NoNullInplace(stableIds.map(id => state.getUnofficialTheme(id)))) function filtered(themes: MinimalLayoutInformation[]): Store { + const prefiltered = themes.filter(th => th.id !== "personal") return searchStable.map(search => { if (!search) { return themes } - const scores = ThemeSearch.sortedByLowestScores(search, themes) + + const scores = ThemeSearch.sortedByLowestScores(search, prefiltered) const strict = scores.filter(sc => sc.lowest < 2) if (strict.length > 0) { return strict.map(sc => sc.theme) diff --git a/test/Logic/OSM/OsmObject.spec.ts b/test/Logic/OSM/OsmObject.spec.ts index 01c962ec6..812967ddb 100644 --- a/test/Logic/OSM/OsmObject.spec.ts +++ b/test/Logic/OSM/OsmObject.spec.ts @@ -1,4 +1,3 @@ -import { OsmObject } from "../../../src/Logic/Osm/OsmObject" import { Utils } from "../../../src/Utils" import ScriptUtils from "../../../scripts/ScriptUtils" import { readFileSync } from "fs"