From 1723f268c03fdb7b49df99c5c0ae587892eebc7d Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Sat, 15 Feb 2025 23:42:07 +0100 Subject: [PATCH] Feature: faster theme search with indexing by lunr --- package-lock.json | 15 ++ package.json | 1 + scripts/generateLayerOverview.ts | 96 ++++++------- src/Logic/Search/ThemeSearch.ts | 229 +++++++++++++++---------------- src/Logic/State/SearchState.ts | 6 +- src/UI/AllThemesGui.svelte | 31 ++--- 6 files changed, 194 insertions(+), 184 deletions(-) diff --git a/package-lock.json b/package-lock.json index ee7c5bddb7..039afe98e5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "fake-dom": "^1.0.4", "flowbite-svelte": "^0.47.4", "follow-redirects": "^1.15.9", + "fuse.js": "^7.1.0", "geojson2svg": "^2.0.2", "html-to-image": "^1.11.11", "i18next-client": "^1.11.4", @@ -15781,6 +15782,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/fuse.js": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/fuse.js/-/fuse.js-7.1.0.tgz", + "integrity": "sha512-trLf4SzuuUxfusZADLINj+dE8clK1frKdmqiJNb1Es75fmI5oY6X2mxLVUciLLjxqw/xr72Dhy+lER6dGd02FQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=10" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "dev": true, @@ -41193,6 +41203,11 @@ "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==" }, + "fuse.js": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/fuse.js/-/fuse.js-7.1.0.tgz", + "integrity": "sha512-trLf4SzuuUxfusZADLINj+dE8clK1frKdmqiJNb1Es75fmI5oY6X2mxLVUciLLjxqw/xr72Dhy+lER6dGd02FQ==" + }, "gensync": { "version": "1.0.0-beta.2", "dev": true, diff --git a/package.json b/package.json index 87869a2070..5be1d84e88 100644 --- a/package.json +++ b/package.json @@ -208,6 +208,7 @@ "fake-dom": "^1.0.4", "flowbite-svelte": "^0.47.4", "follow-redirects": "^1.15.9", + "fuse.js": "^7.1.0", "geojson2svg": "^2.0.2", "html-to-image": "^1.11.11", "i18next-client": "^1.11.4", diff --git a/scripts/generateLayerOverview.ts b/scripts/generateLayerOverview.ts index fb727f7d0c..1b76b52233 100644 --- a/scripts/generateLayerOverview.ts +++ b/scripts/generateLayerOverview.ts @@ -9,16 +9,12 @@ import { DoesImageExist, PrevalidateTheme, ValidateLayer, - ValidateThemeEnsemble, + ValidateThemeEnsemble } from "../src/Models/ThemeConfig/Conversion/Validation" import { Translation } from "../src/UI/i18n/Translation" import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer" import { PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme" -import { - Conversion, - DesugaringContext, - DesugaringStep, -} from "../src/Models/ThemeConfig/Conversion/Conversion" +import { Conversion, DesugaringContext, DesugaringStep } from "../src/Models/ThemeConfig/Conversion/Conversion" import { Utils } from "../src/Utils" import Script from "./Script" import { AllSharedLayers } from "../src/Customizations/AllSharedLayers" @@ -267,6 +263,7 @@ class LayerOverviewUtils extends Script { addWord(lang, tr[lang]) } } + addWord("*", l.id) addWords(l.title) addWords(l.description) @@ -286,9 +283,9 @@ class LayerOverviewUtils extends Script { | LayerConfigJson | string | { - builtin - } - )[] + builtin + } + )[] }[], sharedLayers: Map ) { @@ -317,7 +314,10 @@ class LayerOverviewUtils extends Script { hideFromOverview: theme.hideFromOverview, mustHaveLanguage: theme.mustHaveLanguage, keywords, - layers: theme.layers.filter((l) => sharedLayers.has(l["id"])).map((l) => l["id"]), + layers: (theme.layers) + .filter((l) => sharedLayers.has(l.id)) + .filter(l => l.minzoom < 17) + .map((l) => l.id) } perId.set(data.id, data) } @@ -392,10 +392,10 @@ class LayerOverviewUtils extends Script { tagRenderings: bootstrapTagRenderings, tagRenderingOrder: bootstrapTagRenderingsOrder, sharedLayers: null, - publicLayers: null, + publicLayers: null }, { - addTagRenderingsToContext: true, + addTagRenderingsToContext: true } ) @@ -431,7 +431,7 @@ class LayerOverviewUtils extends Script { "src/assets/SocialImageBanner.svg", "src/assets/SocialImageRepo.svg", "src/assets/svg/osm-logo.svg", - "src/assets/templates/*", + "src/assets/templates/*" ] for (const path of allSvgs) { if ( @@ -456,8 +456,8 @@ class LayerOverviewUtils extends Script { if (contents.indexOf(" 0) { console.warn( "The SVG at " + - path + - " contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path" + path + + " contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path" ) errCount++ } @@ -529,7 +529,7 @@ class LayerOverviewUtils extends Script { JSON.stringify({ layers: Array.from(sharedLayers.values()).filter( (l) => !(l["#no-index"] === "yes") - ), + ) }) ) } @@ -546,11 +546,11 @@ class LayerOverviewUtils extends Script { // mapcomplete-changes shows an icon for each corresponding mapcomplete-theme const iconsPerTheme = Array.from(sharedThemes.values()).map((th) => ({ if: "theme=" + th.id, - then: th.icon, + then: th.icon })) const proto: ThemeConfigJson = JSON.parse( readFileSync("./assets/themes/mapcomplete-changes/mapcomplete-changes.proto.json", { - encoding: "utf8", + encoding: "utf8" }) ) const protolayer = ( @@ -566,7 +566,7 @@ class LayerOverviewUtils extends Script { new DetectDuplicateFilters().convertStrict( { layers: ScriptUtils.getLayerFiles().map((f) => f.parsed), - themes: ScriptUtils.getThemeFiles().map((f) => f.parsed), + themes: ScriptUtils.getThemeFiles().map((f) => f.parsed) }, ConversionContext.construct([], []) ) @@ -614,7 +614,7 @@ class LayerOverviewUtils extends Script { const state: DesugaringContext = { tagRenderings: LayerOverviewUtils.asDict(sharedTagRenderings), tagRenderingOrder: sharedTagRenderings.map((tr) => tr.id), - sharedLayers: AllSharedLayers.getSharedLayersConfigs(), + sharedLayers: AllSharedLayers.getSharedLayersConfigs() } const sharedLayers = new Map() const prepLayer = new PrepareLayer(state) @@ -659,12 +659,12 @@ class LayerOverviewUtils extends Script { console.log( "Recompiled layers " + - recompiledLayers.join(", ") + - " and skipped " + - skippedLayers.length + - " layers. Detected " + - warningCount + - " warnings" + recompiledLayers.join(", ") + + " and skipped " + + skippedLayers.length + + " layers. Detected " + + warningCount + + " warnings" ) // We always need the calculated tags of 'usersettings', so we export them separately this.extractJavascriptCodeForLayer( @@ -686,11 +686,11 @@ class LayerOverviewUtils extends Script { private extractJavascriptCode(themeFile: ThemeConfigJson) { const allCode = [ "import {Feature} from 'geojson'", - 'import { ExtraFuncType } from "../../../Logic/ExtraFunctions";', - 'import { Utils } from "../../../Utils"', + "import { ExtraFuncType } from \"../../../Logic/ExtraFunctions\";", + "import { Utils } from \"../../../Utils\"", "export class ThemeMetaTagging {", " public static readonly themeName = " + JSON.stringify(themeFile.id), - "", + "" ] for (const layer of themeFile.layers) { const l = layer @@ -699,8 +699,8 @@ class LayerOverviewUtils extends Script { allCode.push( " public metaTaggging_for_" + - id + - "(feat: Feature, helperFunctions: Record Function>) {" + id + + "(feat: Feature, helperFunctions: Record Function>) {" ) allCode.push(" const {" + ExtraFunctions.types.join(", ") + "} = helperFunctions") for (const line of code) { @@ -711,10 +711,10 @@ class LayerOverviewUtils extends Script { if (!isStrict) { allCode.push( " Utils.AddLazyProperty(feat.properties, '" + - attributeName + - "', () => " + - expression + - " ) " + attributeName + + "', () => " + + expression + + " ) " ) } else { attributeName = attributeName.substring(0, attributeName.length - 1).trim() @@ -754,7 +754,7 @@ class LayerOverviewUtils extends Script { `/** This code is autogenerated - do not edit. Edit ./assets/layers/${l?.id}/${l?.id}.json instead */`, "export class ThemeMetaTagging {", " public static readonly themeName = " + JSON.stringify(l.id), - "", + "" ] const code = l.calculatedTags ?? [] @@ -769,10 +769,10 @@ class LayerOverviewUtils extends Script { if (!isStrict) { allCode.push( " Utils.AddLazyProperty(feat.properties, '" + - attributeName + - "', () => " + - expression + - " ) " + attributeName + + "', () => " + + expression + + " ) " ) } else { attributeName = attributeName.substring(0, attributeName.length - 2).trim() @@ -813,7 +813,7 @@ class LayerOverviewUtils extends Script { sharedLayers, tagRenderings: LayerOverviewUtils.asDict(trs), tagRenderingOrder: trs.map((tr) => tr.id), - publicLayers, + publicLayers } const knownTagRenderings = new Set() convertState.tagRenderings.forEach((_, key) => knownTagRenderings.add(key)) @@ -870,7 +870,7 @@ class LayerOverviewUtils extends Script { ) try { themeFile = new PrepareTheme(convertState, { - skipDefaultLayers: true, + skipDefaultLayers: true }).convertStrict( themeFile, ConversionContext.construct([themePath], ["PrepareLayer"]) @@ -919,7 +919,7 @@ class LayerOverviewUtils extends Script { const e: string = [ `the icon for theme ${themeFile.id} is too small. Please rescale the icon at ${themeFile.icon}`, `Even though an SVG is 'infinitely scaleable', the icon should be dimensioned bigger. One of the build steps of the theme does convert the image to a PNG (to serve as PWA-icon) and having a small dimension will cause blurry images.`, - ` Width = ${width} height = ${height}; we recommend a size of at least 500px * 500px and to use a square aspect ratio.`, + ` Width = ${width} height = ${height}; we recommend a size of at least 500px * 500px and to use a square aspect ratio.` ].join("\n") err(e) } @@ -956,7 +956,7 @@ class LayerOverviewUtils extends Script { hideFromOverview: t.hideFromOverview ?? false, shortDescription: t.shortDescription ?? new Translation(t.description).FirstSentence(), - mustHaveLanguage: t.mustHaveLanguage?.length > 0, + mustHaveLanguage: t.mustHaveLanguage?.length > 0 } }), sharedLayers @@ -965,10 +965,10 @@ class LayerOverviewUtils extends Script { console.log( "Recompiled themes " + - recompiledThemes.join(", ") + - " and skipped " + - skippedThemes.length + - " themes" + recompiledThemes.join(", ") + + " and skipped " + + skippedThemes.length + + " themes" ) return fixed diff --git a/src/Logic/Search/ThemeSearch.ts b/src/Logic/Search/ThemeSearch.ts index 13f8d3e510..b1b6585b2c 100644 --- a/src/Logic/Search/ThemeSearch.ts +++ b/src/Logic/Search/ThemeSearch.ts @@ -1,19 +1,109 @@ import ThemeConfig, { MinimalThemeInformation } from "../../Models/ThemeConfig/ThemeConfig" import { Store } from "../UIEventSource" import UserRelatedState from "../State/UserRelatedState" -import { Utils } from "../../Utils" -import Locale from "../../UI/i18n/Locale" import themeOverview from "../../assets/generated/theme_overview.json" -import LayerSearch from "./LayerSearch" -import SearchUtils from "./SearchUtils" import { OsmConnection } from "../Osm/OsmConnection" import { AndroidPolyfill } from "../Web/AndroidPolyfill" +import Fuse from "fuse.js" +import Constants from "../../Models/Constants" +import Locale from "../../UI/i18n/Locale" +import { Utils } from "../../Utils" -type ThemeSearchScore = { - theme: MinimalThemeInformation - lowest: number - perLayer?: Record - other: number + +export class ThemeSearchIndex { + + private readonly themeIndex: Fuse + private readonly layerIndex: Fuse<{ id: string, description }> + + constructor(language: string, themesToSearch?: MinimalThemeInformation[], layersToIgnore: string[] = []) { + const themes = themesToSearch ?? ThemeSearch.officialThemes?.themes + if (!themes) { + throw "No themes loaded. Did generate:layeroverview fail?" + } + const fuseOptions = { + ignoreLocation: true, + threshold: 0.2, + keys: [ + { name: "id", weight: 2 }, + "title." + language, + "keywords." + language, + "shortDescription." + language + ] + } + + this.themeIndex = new Fuse(themes.filter(th => th.id !== "personal"), fuseOptions) + + const toIgnore = new Set(layersToIgnore) + const layersAsList: { id: string, description: Record }[] = [] + for (const id in ThemeSearch.officialThemes.layers) { + if (Constants.isPriviliged(id)) { + continue + } + if (toIgnore.has(id)) { + continue + } + const l: Record = ThemeSearch.officialThemes.layers[id] + layersAsList.push({ id, description: l }) + } + this.layerIndex = new Fuse(layersAsList, { + includeScore: true, + minMatchCharLength: 3, + ignoreLocation: true, + threshold: 0.02, + keys: ["id", "description." + language] + }) + } + + public search(text: string, limit?: number): MinimalThemeInformation[] { + const scored = this.searchWithScores(text) + let result = Array.from(scored.entries()) + result.sort((a, b) => b[0] - a[0]) + if (limit) { + result = result.slice(0, limit) + } + return result.map(e => ThemeSearch.officialThemesById.get(e[0])) + } + + public searchWithScores(text: string): Map { + const result = new Map() + const themeResults = this.themeIndex.search(text) + for (const themeResult of themeResults) { + result.set(themeResult.item.id, themeResult.score) + } + + const layerResults = this.layerIndex.search(text) + + for (const layer of layerResults) { + const matchingThemes = ThemeSearch.layersToThemes.get(layer.item.id) + const score = layer.score + matchingThemes?.forEach(th => { + const previous = result.get(th.id) ?? 10000 + result.set(th.id, Math.min(previous, score * 5)) + }) + } + + + return result + } + + /** + * Builds a search index containing all public and visited themes, but ignoring the layers loaded by the current theme + */ + public static fromState(state: { osmConnection: OsmConnection; theme: ThemeConfig }): Store { + const layersToIgnore = state.theme.layers.filter((l) => l.isNormal()).map((l) => l.id) + const knownHidden: Store = UserRelatedState.initDiscoveredHiddenThemes( + state.osmConnection + ).map((list) => Utils.Dedup(list)) + const otherThemes: MinimalThemeInformation[] = ThemeSearch.officialThemes.themes.filter( + (th) => th.id !== state.theme.id + ) + return Locale.language.map(language => { + const themes = otherThemes.concat(...knownHidden.data.map(id => ThemeSearch.officialThemesById.get(id))) + return new ThemeSearchIndex(language, themes, layersToIgnore) + }, + [knownHidden] + ) + } } export default class ThemeSearch { @@ -25,42 +115,26 @@ export default class ThemeSearch { string, MinimalThemeInformation >() + + + /* + * For every layer id, states which themes use the layer + */ + public static readonly layersToThemes: Map = new Map() static { for (const th of ThemeSearch.officialThemes.themes ?? []) { ThemeSearch.officialThemesById.set(th.id, th) + for (const layer of th.layers) { + let list = ThemeSearch.layersToThemes.get(layer) + if (!list) { + list = [] + ThemeSearch.layersToThemes.set(layer, list) + } + list.push(th) + } } } - private readonly _knownHiddenThemes: Store> - private readonly _layersToIgnore: string[] - private readonly _otherThemes: MinimalThemeInformation[] - - constructor(state: { osmConnection: OsmConnection; theme: ThemeConfig }) { - this._layersToIgnore = state.theme.layers.filter((l) => l.isNormal()).map((l) => l.id) - this._knownHiddenThemes = UserRelatedState.initDiscoveredHiddenThemes( - state.osmConnection - ).map((list) => new Set(list)) - this._otherThemes = ThemeSearch.officialThemes.themes.filter( - (th) => th.id !== state.theme.id - ) - } - - public search(query: string, limit: number, threshold: number = 3): MinimalThemeInformation[] { - if (query.length < 1) { - return [] - } - const sorted = ThemeSearch.sortedByLowestScores( - query, - this._otherThemes, - this._layersToIgnore - ) - return sorted - .filter((sorted) => sorted.lowest < threshold) - .map((th) => th.theme) - .filter((th) => !th.hideFromOverview || this._knownHiddenThemes.data.has(th.id)) - .slice(0, limit) - } - public static createUrlFor(layout: { id: string }, state?: { layoutToUse?: { id } }): string { if (layout === undefined) { return undefined @@ -97,82 +171,5 @@ export default class ThemeSearch { return `${linkPrefix}` } - /** - * Returns a score based on textual search - * - * Note that, if `query.length < 3`, layers are _not_ searched because this takes too much time - * @param query - * @param themes - * @param ignoreLayers - * @private - */ - private static scoreThemes( - query: string, - themes: MinimalThemeInformation[], - ignoreLayers: string[] = undefined - ): Record { - if (query?.length < 1) { - return undefined - } - themes = Utils.NoNullInplace(themes) - let options: { blacklist: Set } = undefined - if (ignoreLayers?.length > 0) { - options = { blacklist: new Set(ignoreLayers) } - } - const layerScores = query.length < 3 ? {} : LayerSearch.scoreLayers(query, options) - const results: Record = {} - for (const layoutInfo of themes) { - const theme = layoutInfo.id - if (theme === "personal") { - continue - } - if (Utils.simplifyStringForSearch(theme) === query) { - results[theme] = { - theme: layoutInfo, - lowest: -1, - other: 0, - } - continue - } - const perLayer = Utils.asRecord(layoutInfo.layers ?? [], (layer) => layerScores[layer]) - const language = Locale.language.data - - const keywords = Utils.NoNullInplace([ - layoutInfo.shortDescription, - layoutInfo.title, - ]).map((item) => (typeof item === "string" ? item : item[language] ?? item["*"])) - - const other = Math.min( - SearchUtils.scoreKeywords(query, keywords), - SearchUtils.scoreKeywords(query, layoutInfo.keywords) - ) - const lowest = Math.min(other, ...Object.values(perLayer)) - results[theme] = { - theme: layoutInfo, - perLayer, - other, - lowest, - } - } - return results - } - - public static sortedByLowestScores( - search: string, - themes: MinimalThemeInformation[], - ignoreLayers: string[] = [] - ): ThemeSearchScore[] { - const scored = Object.values(this.scoreThemes(search, themes, ignoreLayers)) - scored.sort((a, b) => a.lowest - b.lowest) - return scored - } - - public static sortedByLowest( - search: string, - themes: MinimalThemeInformation[], - ignoreLayers: string[] = [] - ): MinimalThemeInformation[] { - return this.sortedByLowestScores(search, themes, ignoreLayers).map((th) => th.theme) - } } diff --git a/src/Logic/State/SearchState.ts b/src/Logic/State/SearchState.ts index 1d01737957..e4bf83bba1 100644 --- a/src/Logic/State/SearchState.ts +++ b/src/Logic/State/SearchState.ts @@ -4,7 +4,7 @@ import CombinedSearcher from "../Search/CombinedSearcher" import FilterSearch, { FilterSearchResult } from "../Search/FilterSearch" import LocalElementSearch from "../Search/LocalElementSearch" import CoordinateSearch from "../Search/CoordinateSearch" -import ThemeSearch from "../Search/ThemeSearch" +import { ThemeSearchIndex } from "../Search/ThemeSearch" import OpenStreetMapIdSearch from "../Search/OpenStreetMapIdSearch" import PhotonSearch from "../Search/PhotonSearch" import ThemeViewState from "../../Models/ThemeViewState" @@ -67,8 +67,8 @@ export default class SearchState { Stores.concat(suggestions).map((suggestions) => CombinedSearcher.merge(suggestions)) ) - const themeSearch = new ThemeSearch(state) - this.themeSuggestions = this.searchTerm.mapD((query) => themeSearch.search(query, 3)) + const themeSearch = ThemeSearchIndex.fromState(state) + this.themeSuggestions = this.searchTerm.mapD((query) => themeSearch.data.search(query, 3), [themeSearch]) const layerSearch = new LayerSearch(state.theme) this.layerSuggestions = this.searchTerm.mapD((query) => layerSearch.search(query, 5)) diff --git a/src/UI/AllThemesGui.svelte b/src/UI/AllThemesGui.svelte index ca745216b1..c9bed2eaa4 100644 --- a/src/UI/AllThemesGui.svelte +++ b/src/UI/AllThemesGui.svelte @@ -18,15 +18,15 @@ import Mastodon from "../assets/svg/Mastodon.svelte" import Liberapay from "../assets/svg/Liberapay.svelte" import Bug from "../assets/svg/Bug.svelte" - import Github from "../assets/svg/Github.svelte" import { Utils } from "../Utils" import { ArrowTrendingUp } from "@babeard/svelte-heroicons/solid/ArrowTrendingUp" import Searchbar from "./Base/Searchbar.svelte" - import ThemeSearch from "../Logic/Search/ThemeSearch" + import ThemeSearch, { ThemeSearchIndex } from "../Logic/Search/ThemeSearch" import SearchUtils from "../Logic/Search/SearchUtils" import ChevronDoubleRight from "@babeard/svelte-heroicons/mini/ChevronDoubleRight" import { AndroidPolyfill } from "../Logic/Web/AndroidPolyfill" import Forgejo from "../assets/svg/Forgejo.svelte" + import Locale from "./i18n/Locale" AndroidPolyfill.init().then(() => console.log("Android polyfill setup completed")) const featureSwitches = new OsmConnectionFeatureSwitches() const osmConnection = new OsmConnection({ @@ -65,29 +65,26 @@ state.installedUserThemes ).mapD((stableIds) => Utils.NoNullInplace(stableIds.map((id) => state.getUnofficialTheme(id)))) function filtered(themes: Store): Store { + const searchIndex = Locale.language.map(language => { + return new ThemeSearchIndex(language, themes.data) + }, [themes]) + + return searchStable.map( - (search) => { + (searchTerm) => { if (!themes.data) { return [] } - if (!search) { + if (!searchTerm) { return themes.data } - const start = new Date().getTime() - const scores = ThemeSearch.sortedByLowestScores(search, themes.data) - const end = new Date().getTime() - console.trace("Scores for", search, "are", scores, "searching took", end - start, "ms") - const strict = scores.filter((sc) => sc.lowest < 2) - if (strict.length > 0) { - return strict.map((sc) => sc.theme) - } - return scores - .filter((sc) => sc.lowest < 4) - .slice(0, 6) - .map((sc) => sc.theme) + const index = searchIndex.data + + return index.search(searchTerm) + }, - [themes] + [searchIndex] ) }