From 164b02c8ffefee023d4a52f32081054434d9939c Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Sun, 27 Apr 2025 02:37:47 +0200 Subject: [PATCH] Performance: load less data for branded items, use caching --- Docs/ServerConfig/hetzner/Caddyfile | 1 + src/Logic/Web/NameSuggestionIndex.ts | 332 +++++++++++++++------------ 2 files changed, 186 insertions(+), 147 deletions(-) diff --git a/Docs/ServerConfig/hetzner/Caddyfile b/Docs/ServerConfig/hetzner/Caddyfile index 38960417ea..8db6d1203a 100644 --- a/Docs/ServerConfig/hetzner/Caddyfile +++ b/Docs/ServerConfig/hetzner/Caddyfile @@ -82,6 +82,7 @@ data.mapcomplete.org { header { +Permissions-Policy "interest-cohort=()" +Access-Control-Allow-Origin * + Cache-Control: max-age=86400, public, stale-while-revalidate=86400,stale-if-error=86400 } } diff --git a/src/Logic/Web/NameSuggestionIndex.ts b/src/Logic/Web/NameSuggestionIndex.ts index 10229e8daf..e16fefd304 100644 --- a/src/Logic/Web/NameSuggestionIndex.ts +++ b/src/Logic/Web/NameSuggestionIndex.ts @@ -55,77 +55,22 @@ export interface NSIItem { ext?: string } -export default class NameSuggestionIndex { +export class NameSuggestionIndexLight { public static readonly supportedTypes = ["brand", "flag", "operator", "transit"] as const - private readonly nsiFile: Readonly - private readonly nsiWdFile: Readonly< - Record< - string, - { - logos: { wikidata?: string; facebook?: string } - } - > - > + protected readonly _serverLocation: string + protected readonly nsiFile: Readonly + private readonly loco: LocationConflation // Some additional boundaries + private static initedLight: NameSuggestionIndexLight = undefined - private loco: LocationConflation // Some additional boundaries - - private _supportedTypes: string[] - private _serverLocation: string - - private constructor( + protected constructor( serverLocation: string, nsiFile: Readonly, - nsiWdFile: Readonly< - Record< - string, - { - logos: { wikidata?: string; facebook?: string } - } - > - >, features: Readonly ) { this._serverLocation = serverLocation this.nsiFile = nsiFile - this.nsiWdFile = nsiWdFile this.loco = new LocationConflation(features) - } - - private static inited: NameSuggestionIndex = undefined - - public static async getNsiIndex(): Promise { - if (NameSuggestionIndex.inited) { - return NameSuggestionIndex.inited - } - const [nsi, nsiWd, features] = await Promise.all( - [ - "./assets/data/nsi/nsi.min.json", - "./assets/data/nsi/wikidata.min.json", - "./assets/data/nsi/featureCollection.min.json", - ].map((url) => Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30)) - ) - NameSuggestionIndex.inited = new NameSuggestionIndex( - Constants.nsiLogosEndpoint, - nsi, - nsiWd["wikidata"], - features - ) - return NameSuggestionIndex.inited - } - - public supportedTypes(): string[] { - if (this._supportedTypes) { - return this._supportedTypes - } - const keys = Object.keys(this.nsiFile.nsi) - const all = keys.map((k) => this.nsiFile.nsi[k].properties.path.split("/")[0]) - this._supportedTypes = Utils.Dedup(all).map((s) => { - if (s.endsWith("s")) { - s = s.substring(0, s.length - 1) - } - return s - }) - return this._supportedTypes + NameSuggestionIndexLight.initedLight = this } /** @@ -165,25 +110,6 @@ export default class NameSuggestionIndex { return merged } - public isSvg(nsiItem: NSIItem, type: string): boolean | undefined { - if (this.nsiWdFile === undefined) { - throw ( - "nsiWdi file is not loaded, cannot determine if " + nsiItem.id + " has an SVG image" - ) - } - const logos = this.nsiWdFile[nsiItem?.tags?.[type + ":wikidata"]]?.logos - if (!logos) { - return undefined - } - if (logos.facebook) { - return false - } - const url: string = logos.wikidata - if (url.toLowerCase().endsWith(".svg")) { - return true - } - return false - } public async generateMappings( type: string, @@ -236,7 +162,7 @@ export default class NameSuggestionIndex { // As such, it should be "true" but this is not supported priorityIf: frequency > 0 ? new RegexTag("id", /.*/) : undefined, searchTerms: { "*": [nsiItem.displayName, nsiItem.id] }, - frequency: frequency ?? -1, + frequency: frequency ?? -1 }) } } @@ -247,68 +173,21 @@ export default class NameSuggestionIndex { return mappings } - public supportedTags( - type: "operator" | "brand" | "flag" | "transit" | string - ): Record { - const tags: Record = {} - const keys = Object.keys(this.nsiFile.nsi) - for (const key of keys) { - const nsiItem = this.nsiFile.nsi[key] - const path = nsiItem.properties.path - const [osmType, osmkey, osmvalue] = path.split("/") - if (type !== osmType && type + "s" !== osmType) { - continue - } - if (!tags[osmkey]) { - tags[osmkey] = [] - } - tags[osmkey].push(osmvalue) + public getIconUrl(nsiItem: NSIItem): string | undefined { + const baseUrl = this._serverLocation + if (!nsiItem.ext || baseUrl === null) { + // No extension -> there is no logo + return undefined } - return tags + return baseUrl + "/logos/" + nsiItem.id + "." + nsiItem.ext } - /** - * Returns a list of all brands/operators - * @param type - */ - public allPossible(type: string): NSIItem[] { - const options: NSIItem[] = [] - const tags = this.supportedTags(type) - for (const osmKey in tags) { - const values = tags[osmKey] - for (const osmValue of values) { - const suggestions = this.getSuggestionsForKV(type, osmKey, osmValue) - if (!suggestions) { - console.warn("No suggestions found for", type, osmKey, osmValue) - continue - } - options.push(...suggestions) - } - } - return options - } - - /** - * - * @param country: a string containing one or more country codes, separated by ";" - * @param location: center point of the feature, should be [lon, lat] - */ - public getSuggestionsFor( - type: string, - tags: { key: string; value: string }[], - country: string = undefined, - location: [number, number] = undefined - ): NSIItem[] { - return tags.flatMap((tag) => - this.getSuggestionsForKV(type, tag.key, tag.value, country, location) - ) - } /** * Caching for the resolved sets, as they can take a while * @private */ - private static resolvedSets: Record = {} + private static resolvedSets: Record = {} /** * Returns all suggestions for the given type (brand|operator) and main tag. @@ -379,6 +258,173 @@ export default class NameSuggestionIndex { }) } + public static async singleton(): Promise { + if (NameSuggestionIndexLight.initedLight) { + return NameSuggestionIndexLight.initedLight + } + const [nsi, features] = await Promise.all( + [ + "./assets/data/nsi/nsi.min.json", + "./assets/data/nsi/featureCollection.min.json" + ].map((url) => Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30)) + ) + return new NameSuggestionIndexLight( + Constants.nsiLogosEndpoint, + nsi, + features + ) + } + +} + +export default class NameSuggestionIndex extends NameSuggestionIndexLight { + private readonly nsiWdFile: Readonly< + Record< + string, + { + logos: { wikidata?: string; facebook?: string } + } + > + > + + + private _supportedTypes: string[] + private static inited: NameSuggestionIndex = undefined + + private constructor( + serverLocation: string, + nsiFile: Readonly, + nsiWdFile: Readonly< + Record< + string, + { + logos: { wikidata?: string; facebook?: string } + } + > + >, + features: Readonly + ) { + super(serverLocation, nsiFile, features) + this.nsiWdFile = nsiWdFile + NameSuggestionIndex.inited = this + } + + + public static async getNsiIndex(): Promise { + if (NameSuggestionIndex.inited) { + return NameSuggestionIndex.inited + } + const [nsi, nsiWd, features] = await Promise.all( + [ + "./assets/data/nsi/nsi.min.json", + "./assets/data/nsi/wikidata.min.json", + "./assets/data/nsi/featureCollection.min.json" + ].map((url) => Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30)) + ) + return new NameSuggestionIndex( + Constants.nsiLogosEndpoint, + nsi, + nsiWd["wikidata"], + features + ) + } + + + public supportedTypes(): string[] { + if (this._supportedTypes) { + return this._supportedTypes + } + const keys = Object.keys(this.nsiFile.nsi) + const all = keys.map((k) => this.nsiFile.nsi[k].properties.path.split("/")[0]) + this._supportedTypes = Utils.Dedup(all).map((s) => { + if (s.endsWith("s")) { + s = s.substring(0, s.length - 1) + } + return s + }) + return this._supportedTypes + } + + + public isSvg(nsiItem: NSIItem, type: string): boolean | undefined { + if (this.nsiWdFile === undefined) { + throw ( + "nsiWdi file is not loaded, cannot determine if " + nsiItem.id + " has an SVG image" + ) + } + const logos = this.nsiWdFile[nsiItem?.tags?.[type + ":wikidata"]]?.logos + if (!logos) { + return undefined + } + if (logos.facebook) { + return false + } + const url: string = logos.wikidata + if (url.toLowerCase().endsWith(".svg")) { + return true + } + return false + } + + + public supportedTags( + type: "operator" | "brand" | "flag" | "transit" | string + ): Record { + const tags: Record = {} + const keys = Object.keys(this.nsiFile.nsi) + for (const key of keys) { + const nsiItem = this.nsiFile.nsi[key] + const path = nsiItem.properties.path + const [osmType, osmkey, osmvalue] = path.split("/") + if (type !== osmType && type + "s" !== osmType) { + continue + } + if (!tags[osmkey]) { + tags[osmkey] = [] + } + tags[osmkey].push(osmvalue) + } + return tags + } + + /** + * Returns a list of all brands/operators + * @param type + */ + public allPossible(type: string): NSIItem[] { + const options: NSIItem[] = [] + const tags = this.supportedTags(type) + for (const osmKey in tags) { + const values = tags[osmKey] + for (const osmValue of values) { + const suggestions = this.getSuggestionsForKV(type, osmKey, osmValue) + if (!suggestions) { + console.warn("No suggestions found for", type, osmKey, osmValue) + continue + } + options.push(...suggestions) + } + } + return options + } + + /** + * + * @param country: a string containing one or more country codes, separated by ";" + * @param location: center point of the feature, should be [lon, lat] + */ + public getSuggestionsFor( + type: string, + tags: { key: string; value: string }[], + country: string = undefined, + location: [number, number] = undefined + ): NSIItem[] { + return tags.flatMap((tag) => + this.getSuggestionsForKV(type, tag.key, tag.value, country, location) + ) + } + + public static async generateMappings( key: string, tags: Exclude, undefined | null>, @@ -388,7 +434,7 @@ export default class NameSuggestionIndex { sortByFrequency: boolean } ): Promise { - const nsi = await NameSuggestionIndex.getNsiIndex() + const nsi = await NameSuggestionIndexLight.singleton() return nsi.generateMappings(key, tags, country, center, options) } @@ -403,14 +449,6 @@ export default class NameSuggestionIndex { return logos?.facebook ?? logos?.wikidata } - public getIconUrl(nsiItem: NSIItem): string | undefined { - const baseUrl = this._serverLocation - if (!nsiItem.ext || baseUrl === null) { - // No extension -> there is no logo - return undefined - } - return baseUrl +"/logos/"+ nsiItem.id + "." + nsiItem.ext - } private static readonly brandPrefix = ["name", "alt_name", "operator", "brand"] as const @@ -425,8 +463,8 @@ export default class NameSuggestionIndex { static asFilterTags( item: NSIItem ): string | { and: TagConfigJson[] } | { or: TagConfigJson[] } { - let brandDetection: string[] = [] - let required: string[] = [] + const brandDetection: string[] = [] + const required: string[] = [] const tags: Record = item.tags for (const k in tags) { if (NameSuggestionIndex.brandPrefix.some((br) => k === br || k.startsWith(br + ":"))) {