Feature: faster theme search with indexing by lunr

package-lock.json

@@ -47,6 +47,7 @@
         "fake-dom": "^1.0.4",
         "flowbite-svelte": "^0.47.4",
         "follow-redirects": "^1.15.9",
+        "fuse.js": "^7.1.0",
         "geojson2svg": "^2.0.2",
         "html-to-image": "^1.11.11",
         "i18next-client": "^1.11.4",
@@ -15781,6 +15782,15 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/fuse.js": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/fuse.js/-/fuse.js-7.1.0.tgz",
+      "integrity": "sha512-trLf4SzuuUxfusZADLINj+dE8clK1frKdmqiJNb1Es75fmI5oY6X2mxLVUciLLjxqw/xr72Dhy+lER6dGd02FQ==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/gensync": {
       "version": "1.0.0-beta.2",
       "dev": true,
@@ -41193,6 +41203,11 @@
       "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz",
       "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ=="
     },
+    "fuse.js": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/fuse.js/-/fuse.js-7.1.0.tgz",
+      "integrity": "sha512-trLf4SzuuUxfusZADLINj+dE8clK1frKdmqiJNb1Es75fmI5oY6X2mxLVUciLLjxqw/xr72Dhy+lER6dGd02FQ=="
+    },
     "gensync": {
       "version": "1.0.0-beta.2",
       "dev": true,

package.json

@@ -208,6 +208,7 @@
     "fake-dom": "^1.0.4",
     "flowbite-svelte": "^0.47.4",
     "follow-redirects": "^1.15.9",
+    "fuse.js": "^7.1.0",
     "geojson2svg": "^2.0.2",
     "html-to-image": "^1.11.11",
     "i18next-client": "^1.11.4",

scripts/generateLayerOverview.ts

@@ -9,16 +9,12 @@ import {
     DoesImageExist,
     PrevalidateTheme,
     ValidateLayer,
-    ValidateThemeEnsemble,
+    ValidateThemeEnsemble
 } from "../src/Models/ThemeConfig/Conversion/Validation"
 import { Translation } from "../src/UI/i18n/Translation"
 import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
 import { PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme"
-import {
+import { Conversion, DesugaringContext, DesugaringStep } from "../src/Models/ThemeConfig/Conversion/Conversion"
-    Conversion,
-    DesugaringContext,
-    DesugaringStep,
-} from "../src/Models/ThemeConfig/Conversion/Conversion"
 import { Utils } from "../src/Utils"
 import Script from "./Script"
 import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
@@ -267,6 +263,7 @@ class LayerOverviewUtils extends Script {
                 addWord(lang, tr[lang])
             }
         }
+
         addWord("*", l.id)
         addWords(l.title)
         addWords(l.description)
@@ -317,7 +314,10 @@ class LayerOverviewUtils extends Script {
                 hideFromOverview: theme.hideFromOverview,
                 mustHaveLanguage: theme.mustHaveLanguage,
                 keywords,
-                layers: theme.layers.filter((l) => sharedLayers.has(l["id"])).map((l) => l["id"]),
+                layers: (<LayerConfigJson[]>theme.layers)
+                    .filter((l) => sharedLayers.has(l.id))
+                    .filter(l => l.minzoom < 17)
+                    .map((l) => l.id)
             }
             perId.set(data.id, data)
         }
@@ -392,10 +392,10 @@ class LayerOverviewUtils extends Script {
                 tagRenderings: bootstrapTagRenderings,
                 tagRenderingOrder: bootstrapTagRenderingsOrder,
                 sharedLayers: null,
-                publicLayers: null,
+                publicLayers: null
             },
             {
-                addTagRenderingsToContext: true,
+                addTagRenderingsToContext: true
             }
         )
 
@@ -431,7 +431,7 @@ class LayerOverviewUtils extends Script {
             "src/assets/SocialImageBanner.svg",
             "src/assets/SocialImageRepo.svg",
             "src/assets/svg/osm-logo.svg",
-            "src/assets/templates/*",
+            "src/assets/templates/*"
         ]
         for (const path of allSvgs) {
             if (
@@ -529,7 +529,7 @@ class LayerOverviewUtils extends Script {
                 JSON.stringify({
                     layers: Array.from(sharedLayers.values()).filter(
                         (l) => !(l["#no-index"] === "yes")
-                    ),
+                    )
                 })
             )
         }
@@ -546,11 +546,11 @@ class LayerOverviewUtils extends Script {
             // mapcomplete-changes shows an icon for each corresponding mapcomplete-theme
             const iconsPerTheme = Array.from(sharedThemes.values()).map((th) => ({
                 if: "theme=" + th.id,
-                then: th.icon,
+                then: th.icon
             }))
             const proto: ThemeConfigJson = JSON.parse(
                 readFileSync("./assets/themes/mapcomplete-changes/mapcomplete-changes.proto.json", {
-                    encoding: "utf8",
+                    encoding: "utf8"
                 })
             )
             const protolayer = <LayerConfigJson>(
@@ -566,7 +566,7 @@ class LayerOverviewUtils extends Script {
         new DetectDuplicateFilters().convertStrict(
             {
                 layers: ScriptUtils.getLayerFiles().map((f) => f.parsed),
-                themes: ScriptUtils.getThemeFiles().map((f) => f.parsed),
+                themes: ScriptUtils.getThemeFiles().map((f) => f.parsed)
             },
             ConversionContext.construct([], [])
         )
@@ -614,7 +614,7 @@ class LayerOverviewUtils extends Script {
         const state: DesugaringContext = {
             tagRenderings: LayerOverviewUtils.asDict(sharedTagRenderings),
             tagRenderingOrder: sharedTagRenderings.map((tr) => tr.id),
-            sharedLayers: AllSharedLayers.getSharedLayersConfigs(),
+            sharedLayers: AllSharedLayers.getSharedLayersConfigs()
         }
         const sharedLayers = new Map<string, LayerConfigJson>()
         const prepLayer = new PrepareLayer(state)
@@ -686,11 +686,11 @@ class LayerOverviewUtils extends Script {
     private extractJavascriptCode(themeFile: ThemeConfigJson) {
         const allCode = [
             "import {Feature} from 'geojson'",
-            'import { ExtraFuncType } from "../../../Logic/ExtraFunctions";',
+            "import { ExtraFuncType } from \"../../../Logic/ExtraFunctions\";",
-            'import { Utils } from "../../../Utils"',
+            "import { Utils } from \"../../../Utils\"",
             "export class ThemeMetaTagging {",
             "   public static readonly themeName = " + JSON.stringify(themeFile.id),
-            "",
+            ""
         ]
         for (const layer of themeFile.layers) {
             const l = <LayerConfigJson>layer
@@ -754,7 +754,7 @@ class LayerOverviewUtils extends Script {
             `/** This code is autogenerated - do not edit. Edit ./assets/layers/${l?.id}/${l?.id}.json instead */`,
             "export class ThemeMetaTagging {",
             "   public static readonly themeName = " + JSON.stringify(l.id),
-            "",
+            ""
         ]
         const code = l.calculatedTags ?? []
 
@@ -813,7 +813,7 @@ class LayerOverviewUtils extends Script {
             sharedLayers,
             tagRenderings: LayerOverviewUtils.asDict(trs),
             tagRenderingOrder: trs.map((tr) => tr.id),
-            publicLayers,
+            publicLayers
         }
         const knownTagRenderings = new Set<string>()
         convertState.tagRenderings.forEach((_, key) => knownTagRenderings.add(key))
@@ -870,7 +870,7 @@ class LayerOverviewUtils extends Script {
             )
             try {
                 themeFile = new PrepareTheme(convertState, {
-                    skipDefaultLayers: true,
+                    skipDefaultLayers: true
                 }).convertStrict(
                     themeFile,
                     ConversionContext.construct([themePath], ["PrepareLayer"])
@@ -919,7 +919,7 @@ class LayerOverviewUtils extends Script {
                                 const e: string = [
                                     `the icon for theme ${themeFile.id} is too small. Please rescale the icon at ${themeFile.icon}`,
                                     `Even though an SVG is 'infinitely scaleable', the icon should be dimensioned bigger. One of the build steps of the theme does convert the image to a PNG (to serve as PWA-icon) and having a small dimension will cause blurry images.`,
-                                    ` Width = ${width} height = ${height}; we recommend a size of at least 500px * 500px and to use a square aspect ratio.`,
+                                    ` Width = ${width} height = ${height}; we recommend a size of at least 500px * 500px and to use a square aspect ratio.`
                                 ].join("\n")
                                 err(e)
                             }
@@ -956,7 +956,7 @@ class LayerOverviewUtils extends Script {
                         hideFromOverview: t.hideFromOverview ?? false,
                         shortDescription:
                             t.shortDescription ?? new Translation(t.description).FirstSentence(),
-                        mustHaveLanguage: t.mustHaveLanguage?.length > 0,
+                        mustHaveLanguage: t.mustHaveLanguage?.length > 0
                     }
                 }),
                 sharedLayers

src/Logic/Search/ThemeSearch.ts

@@ -1,19 +1,109 @@
 import ThemeConfig, { MinimalThemeInformation } from "../../Models/ThemeConfig/ThemeConfig"
 import { Store } from "../UIEventSource"
 import UserRelatedState from "../State/UserRelatedState"
-import { Utils } from "../../Utils"
-import Locale from "../../UI/i18n/Locale"
 import themeOverview from "../../assets/generated/theme_overview.json"
-import LayerSearch from "./LayerSearch"
-import SearchUtils from "./SearchUtils"
 import { OsmConnection } from "../Osm/OsmConnection"
 import { AndroidPolyfill } from "../Web/AndroidPolyfill"
+import Fuse from "fuse.js"
+import Constants from "../../Models/Constants"
+import Locale from "../../UI/i18n/Locale"
+import { Utils } from "../../Utils"
 
-type ThemeSearchScore = {
+
-    theme: MinimalThemeInformation
+export class ThemeSearchIndex {
-    lowest: number
+
-    perLayer?: Record<string, number>
+    private readonly themeIndex: Fuse<MinimalThemeInformation>
-    other: number
+    private readonly layerIndex: Fuse<{ id: string, description }>
+
+    constructor(language: string, themesToSearch?: MinimalThemeInformation[], layersToIgnore: string[] = []) {
+        const themes = themesToSearch ?? ThemeSearch.officialThemes?.themes
+        if (!themes) {
+            throw "No themes loaded. Did generate:layeroverview fail?"
+        }
+        const fuseOptions = {
+            ignoreLocation: true,
+            threshold: 0.2,
+            keys: [
+                { name: "id", weight: 2 },
+                "title." + language,
+                "keywords." + language,
+                "shortDescription." + language
+            ]
+        }
+
+        this.themeIndex = new Fuse(themes.filter(th => th.id !== "personal"), fuseOptions)
+
+        const toIgnore = new Set(layersToIgnore)
+        const layersAsList: { id: string, description: Record<string, string[]> }[] = []
+        for (const id in ThemeSearch.officialThemes.layers) {
+            if (Constants.isPriviliged(id)) {
+                continue
+            }
+            if (toIgnore.has(id)) {
+                continue
+            }
+            const l: Record<string, string[]> = ThemeSearch.officialThemes.layers[id]
+            layersAsList.push({ id, description: l })
+        }
+        this.layerIndex = new Fuse(layersAsList, {
+            includeScore: true,
+            minMatchCharLength: 3,
+            ignoreLocation: true,
+            threshold: 0.02,
+            keys: ["id", "description." + language]
+        })
+    }
+
+    public search(text: string, limit?: number): MinimalThemeInformation[] {
+        const scored = this.searchWithScores(text)
+        let result = Array.from(scored.entries())
+        result.sort((a, b) => b[0] - a[0])
+        if (limit) {
+            result = result.slice(0, limit)
+        }
+        return result.map(e => ThemeSearch.officialThemesById.get(e[0]))
+    }
+
+    public searchWithScores(text: string): Map<string, number> {
+        const result = new Map<string, number>()
+        const themeResults = this.themeIndex.search(text)
+        for (const themeResult of themeResults) {
+            result.set(themeResult.item.id, themeResult.score)
+        }
+
+        const layerResults = this.layerIndex.search(text)
+
+        for (const layer of layerResults) {
+            const matchingThemes = ThemeSearch.layersToThemes.get(layer.item.id)
+            const score = layer.score
+            matchingThemes?.forEach(th => {
+                const previous = result.get(th.id) ?? 10000
+                result.set(th.id, Math.min(previous, score * 5))
+            })
+        }
+
+
+        return result
+    }
+
+    /**
+     * Builds a search index containing all public and visited themes, but ignoring the layers loaded by the current theme
+     */
+    public static fromState(state: { osmConnection: OsmConnection; theme: ThemeConfig }): Store<ThemeSearchIndex> {
+        const layersToIgnore = state.theme.layers.filter((l) => l.isNormal()).map((l) => l.id)
+        const knownHidden: Store<string[]> = UserRelatedState.initDiscoveredHiddenThemes(
+            state.osmConnection
+        ).map((list) => Utils.Dedup(list))
+        const otherThemes: MinimalThemeInformation[] = ThemeSearch.officialThemes.themes.filter(
+            (th) => th.id !== state.theme.id
+        )
+        return Locale.language.map(language => {
+                const themes = otherThemes.concat(...knownHidden.data.map(id => ThemeSearch.officialThemesById.get(id)))
+                return new ThemeSearchIndex(language, themes, layersToIgnore)
+            },
+            [knownHidden]
+        )
+    }
 }
 
 export default class ThemeSearch {
@@ -25,40 +115,24 @@ export default class ThemeSearch {
         string,
         MinimalThemeInformation
     >()
+
+
+    /*
+    * For every layer id, states which themes use the layer
+     */
+    public static readonly layersToThemes: Map<string, MinimalThemeInformation[]> = new Map()
     static {
         for (const th of ThemeSearch.officialThemes.themes ?? []) {
             ThemeSearch.officialThemesById.set(th.id, th)
+            for (const layer of th.layers) {
+                let list = ThemeSearch.layersToThemes.get(layer)
+                if (!list) {
+                    list = []
+                    ThemeSearch.layersToThemes.set(layer, list)
+                }
+                list.push(th)
             }
         }
-
-    private readonly _knownHiddenThemes: Store<Set<string>>
-    private readonly _layersToIgnore: string[]
-    private readonly _otherThemes: MinimalThemeInformation[]
-
-    constructor(state: { osmConnection: OsmConnection; theme: ThemeConfig }) {
-        this._layersToIgnore = state.theme.layers.filter((l) => l.isNormal()).map((l) => l.id)
-        this._knownHiddenThemes = UserRelatedState.initDiscoveredHiddenThemes(
-            state.osmConnection
-        ).map((list) => new Set(list))
-        this._otherThemes = ThemeSearch.officialThemes.themes.filter(
-            (th) => th.id !== state.theme.id
-        )
-    }
-
-    public search(query: string, limit: number, threshold: number = 3): MinimalThemeInformation[] {
-        if (query.length < 1) {
-            return []
-        }
-        const sorted = ThemeSearch.sortedByLowestScores(
-            query,
-            this._otherThemes,
-            this._layersToIgnore
-        )
-        return sorted
-            .filter((sorted) => sorted.lowest < threshold)
-            .map((th) => th.theme)
-            .filter((th) => !th.hideFromOverview || this._knownHiddenThemes.data.has(th.id))
-            .slice(0, limit)
     }
 
     public static createUrlFor(layout: { id: string }, state?: { layoutToUse?: { id } }): string {
@@ -97,82 +171,5 @@ export default class ThemeSearch {
         return `${linkPrefix}`
     }
 
-    /**
-     * Returns a score based on textual search
-     *
-     * Note that, if `query.length < 3`, layers are _not_ searched because this takes too much time
-     * @param query
-     * @param themes
-     * @param ignoreLayers
-     * @private
-     */
-    private static scoreThemes(
-        query: string,
-        themes: MinimalThemeInformation[],
-        ignoreLayers: string[] = undefined
-    ): Record<string, ThemeSearchScore> {
-        if (query?.length < 1) {
-            return undefined
-        }
-        themes = Utils.NoNullInplace(themes)
 
-        let options: { blacklist: Set<string> } = undefined
-        if (ignoreLayers?.length > 0) {
-            options = { blacklist: new Set(ignoreLayers) }
-        }
-        const layerScores = query.length < 3 ? {} : LayerSearch.scoreLayers(query, options)
-        const results: Record<string, ThemeSearchScore> = {}
-        for (const layoutInfo of themes) {
-            const theme = layoutInfo.id
-            if (theme === "personal") {
-                continue
-            }
-            if (Utils.simplifyStringForSearch(theme) === query) {
-                results[theme] = {
-                    theme: layoutInfo,
-                    lowest: -1,
-                    other: 0,
-                }
-                continue
-            }
-            const perLayer = Utils.asRecord(layoutInfo.layers ?? [], (layer) => layerScores[layer])
-            const language = Locale.language.data
-
-            const keywords = Utils.NoNullInplace([
-                layoutInfo.shortDescription,
-                layoutInfo.title,
-            ]).map((item) => (typeof item === "string" ? item : item[language] ?? item["*"]))
-
-            const other = Math.min(
-                SearchUtils.scoreKeywords(query, keywords),
-                SearchUtils.scoreKeywords(query, layoutInfo.keywords)
-            )
-            const lowest = Math.min(other, ...Object.values(perLayer))
-            results[theme] = {
-                theme: layoutInfo,
-                perLayer,
-                other,
-                lowest,
-            }
-        }
-        return results
-    }
-
-    public static sortedByLowestScores(
-        search: string,
-        themes: MinimalThemeInformation[],
-        ignoreLayers: string[] = []
-    ): ThemeSearchScore[] {
-        const scored = Object.values(this.scoreThemes(search, themes, ignoreLayers))
-        scored.sort((a, b) => a.lowest - b.lowest)
-        return scored
-    }
-
-    public static sortedByLowest(
-        search: string,
-        themes: MinimalThemeInformation[],
-        ignoreLayers: string[] = []
-    ): MinimalThemeInformation[] {
-        return this.sortedByLowestScores(search, themes, ignoreLayers).map((th) => th.theme)
-    }
 }

src/Logic/State/SearchState.ts

@@ -4,7 +4,7 @@ import CombinedSearcher from "../Search/CombinedSearcher"
 import FilterSearch, { FilterSearchResult } from "../Search/FilterSearch"
 import LocalElementSearch from "../Search/LocalElementSearch"
 import CoordinateSearch from "../Search/CoordinateSearch"
-import ThemeSearch from "../Search/ThemeSearch"
+import { ThemeSearchIndex } from "../Search/ThemeSearch"
 import OpenStreetMapIdSearch from "../Search/OpenStreetMapIdSearch"
 import PhotonSearch from "../Search/PhotonSearch"
 import ThemeViewState from "../../Models/ThemeViewState"
@@ -67,8 +67,8 @@ export default class SearchState {
             Stores.concat(suggestions).map((suggestions) => CombinedSearcher.merge(suggestions))
         )
 
-        const themeSearch = new ThemeSearch(state)
+        const themeSearch = ThemeSearchIndex.fromState(state)
-        this.themeSuggestions = this.searchTerm.mapD((query) => themeSearch.search(query, 3))
+        this.themeSuggestions = this.searchTerm.mapD((query) => themeSearch.data.search(query, 3), [themeSearch])
 
         const layerSearch = new LayerSearch(state.theme)
         this.layerSuggestions = this.searchTerm.mapD((query) => layerSearch.search(query, 5))

src/UI/AllThemesGui.svelte

@@ -18,15 +18,15 @@
   import Mastodon from "../assets/svg/Mastodon.svelte"
   import Liberapay from "../assets/svg/Liberapay.svelte"
   import Bug from "../assets/svg/Bug.svelte"
-  import Github from "../assets/svg/Github.svelte"
   import { Utils } from "../Utils"
   import { ArrowTrendingUp } from "@babeard/svelte-heroicons/solid/ArrowTrendingUp"
   import Searchbar from "./Base/Searchbar.svelte"
-  import ThemeSearch from "../Logic/Search/ThemeSearch"
+  import ThemeSearch, { ThemeSearchIndex } from "../Logic/Search/ThemeSearch"
   import SearchUtils from "../Logic/Search/SearchUtils"
   import ChevronDoubleRight from "@babeard/svelte-heroicons/mini/ChevronDoubleRight"
   import { AndroidPolyfill } from "../Logic/Web/AndroidPolyfill"
   import Forgejo from "../assets/svg/Forgejo.svelte"
+  import Locale from "./i18n/Locale"
   AndroidPolyfill.init().then(() => console.log("Android polyfill setup completed"))
   const featureSwitches = new OsmConnectionFeatureSwitches()
   const osmConnection = new OsmConnection({
@@ -65,29 +65,26 @@
     state.installedUserThemes
   ).mapD((stableIds) => Utils.NoNullInplace(stableIds.map((id) => state.getUnofficialTheme(id))))
   function filtered(themes: Store<MinimalThemeInformation[]>): Store<MinimalThemeInformation[]> {
+    const searchIndex = Locale.language.map(language => {
+      return new ThemeSearchIndex(language, themes.data)
+    }, [themes])
+
+
     return searchStable.map(
-      (search) => {
+      (searchTerm) => {
         if (!themes.data) {
           return []
         }
-        if (!search) {
+        if (!searchTerm) {
           return themes.data
         }
 
-        const start = new Date().getTime()
+        const index = searchIndex.data
-        const scores = ThemeSearch.sortedByLowestScores(search, themes.data)
+
-        const end = new Date().getTime()
+        return index.search(searchTerm)
-        console.trace("Scores for", search, "are", scores, "searching took", end - start, "ms")
+
-        const strict = scores.filter((sc) => sc.lowest < 2)
-        if (strict.length > 0) {
-          return strict.map((sc) => sc.theme)
-        }
-        return scores
-          .filter((sc) => sc.lowest < 4)
-          .slice(0, 6)
-          .map((sc) => sc.theme)
       },
-      [themes]
+      [searchIndex]
     )
   }