midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Feature: faster theme search with indexing by lunr

Browse source
This commit is contained in:
Pieter Vander Vennet 2025-02-15 23:42:07 +01:00
parent 7226c82009
commit 1723f268c0
6 changed files with 194 additions and 184 deletions
Download patch file Download diff file Expand all files Collapse all files

15
package-lock.json generated
View file

@ -47,6 +47,7 @@
"fake-dom": "^1.0.4",
"flowbite-svelte": "^0.47.4",
"follow-redirects": "^1.15.9",
"fuse.js": "^7.1.0",
"geojson2svg": "^2.0.2",
"html-to-image": "^1.11.11",
"i18next-client": "^1.11.4",
@ -15781,6 +15782,15 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/fuse.js": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/fuse.js/-/fuse.js-7.1.0.tgz",
"integrity": "sha512-trLf4SzuuUxfusZADLINj+dE8clK1frKdmqiJNb1Es75fmI5oY6X2mxLVUciLLjxqw/xr72Dhy+lER6dGd02FQ==",
"license": "Apache-2.0",
"engines": {
"node": ">=10"
}
},
"node_modules/gensync": {
"version": "1.0.0-beta.2",
"dev": true,
@ -41193,6 +41203,11 @@
"resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz",
"integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ=="
},
"fuse.js": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/fuse.js/-/fuse.js-7.1.0.tgz",
"integrity": "sha512-trLf4SzuuUxfusZADLINj+dE8clK1frKdmqiJNb1Es75fmI5oY6X2mxLVUciLLjxqw/xr72Dhy+lER6dGd02FQ=="
},
"gensync": {
"version": "1.0.0-beta.2",
"dev": true,

1
package.json
View file

@ -208,6 +208,7 @@
"fake-dom": "^1.0.4",
"flowbite-svelte": "^0.47.4",
"follow-redirects": "^1.15.9",
"fuse.js": "^7.1.0",
"geojson2svg": "^2.0.2",
"html-to-image": "^1.11.11",
"i18next-client": "^1.11.4",

46
scripts/generateLayerOverview.ts
View file

@ -9,16 +9,12 @@ import {
DoesImageExist,
PrevalidateTheme,
ValidateLayer,
ValidateThemeEnsemble,
ValidateThemeEnsemble
} from "../src/Models/ThemeConfig/Conversion/Validation"
import { Translation } from "../src/UI/i18n/Translation"
import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
import { PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme"
import {
Conversion,
DesugaringContext,
DesugaringStep,
} from "../src/Models/ThemeConfig/Conversion/Conversion"
import { Conversion, DesugaringContext, DesugaringStep } from "../src/Models/ThemeConfig/Conversion/Conversion"
import { Utils } from "../src/Utils"
import Script from "./Script"
import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
@ -267,6 +263,7 @@ class LayerOverviewUtils extends Script {
addWord(lang, tr[lang])
}
}
addWord("*", l.id)
addWords(l.title)
addWords(l.description)
@ -317,7 +314,10 @@ class LayerOverviewUtils extends Script {
hideFromOverview: theme.hideFromOverview,
mustHaveLanguage: theme.mustHaveLanguage,
keywords,
layers: theme.layers.filter((l) => sharedLayers.has(l["id"])).map((l) => l["id"]),
layers: (<LayerConfigJson[]>theme.layers)
.filter((l) => sharedLayers.has(l.id))
.filter(l => l.minzoom < 17)
.map((l) => l.id)
}
perId.set(data.id, data)
}
@ -392,10 +392,10 @@ class LayerOverviewUtils extends Script {
tagRenderings: bootstrapTagRenderings,
tagRenderingOrder: bootstrapTagRenderingsOrder,
sharedLayers: null,
publicLayers: null,
publicLayers: null
},
{
addTagRenderingsToContext: true,
addTagRenderingsToContext: true
}
)
@ -431,7 +431,7 @@ class LayerOverviewUtils extends Script {
"src/assets/SocialImageBanner.svg",
"src/assets/SocialImageRepo.svg",
"src/assets/svg/osm-logo.svg",
"src/assets/templates/*",
"src/assets/templates/*"
]
for (const path of allSvgs) {
if (
@ -529,7 +529,7 @@ class LayerOverviewUtils extends Script {
JSON.stringify({
layers: Array.from(sharedLayers.values()).filter(
(l) => !(l["#no-index"] === "yes")
),
)
})
)
}
@ -546,11 +546,11 @@ class LayerOverviewUtils extends Script {
// mapcomplete-changes shows an icon for each corresponding mapcomplete-theme
const iconsPerTheme = Array.from(sharedThemes.values()).map((th) => ({
if: "theme=" + th.id,
then: th.icon,
then: th.icon
}))
const proto: ThemeConfigJson = JSON.parse(
readFileSync("./assets/themes/mapcomplete-changes/mapcomplete-changes.proto.json", {
encoding: "utf8",
encoding: "utf8"
})
)
const protolayer = <LayerConfigJson>(
@ -566,7 +566,7 @@ class LayerOverviewUtils extends Script {
new DetectDuplicateFilters().convertStrict(
{
layers: ScriptUtils.getLayerFiles().map((f) => f.parsed),
themes: ScriptUtils.getThemeFiles().map((f) => f.parsed),
themes: ScriptUtils.getThemeFiles().map((f) => f.parsed)
},
ConversionContext.construct([], [])
)
@ -614,7 +614,7 @@ class LayerOverviewUtils extends Script {
const state: DesugaringContext = {
tagRenderings: LayerOverviewUtils.asDict(sharedTagRenderings),
tagRenderingOrder: sharedTagRenderings.map((tr) => tr.id),
sharedLayers: AllSharedLayers.getSharedLayersConfigs(),
sharedLayers: AllSharedLayers.getSharedLayersConfigs()
}
const sharedLayers = new Map<string, LayerConfigJson>()
const prepLayer = new PrepareLayer(state)
@ -686,11 +686,11 @@ class LayerOverviewUtils extends Script {
private extractJavascriptCode(themeFile: ThemeConfigJson) {
const allCode = [
"import {Feature} from 'geojson'",
'import { ExtraFuncType } from "../../../Logic/ExtraFunctions";',
'import { Utils } from "../../../Utils"',
"import { ExtraFuncType } from \"../../../Logic/ExtraFunctions\";",
"import { Utils } from \"../../../Utils\"",
"export class ThemeMetaTagging {",
" public static readonly themeName = " + JSON.stringify(themeFile.id),
"",
""
]
for (const layer of themeFile.layers) {
const l = <LayerConfigJson>layer
@ -754,7 +754,7 @@ class LayerOverviewUtils extends Script {
`/** This code is autogenerated - do not edit. Edit ./assets/layers/${l?.id}/${l?.id}.json instead */`,
"export class ThemeMetaTagging {",
" public static readonly themeName = " + JSON.stringify(l.id),
"",
""
]
const code = l.calculatedTags ?? []
@ -813,7 +813,7 @@ class LayerOverviewUtils extends Script {
sharedLayers,
tagRenderings: LayerOverviewUtils.asDict(trs),
tagRenderingOrder: trs.map((tr) => tr.id),
publicLayers,
publicLayers
}
const knownTagRenderings = new Set<string>()
convertState.tagRenderings.forEach((_, key) => knownTagRenderings.add(key))
@ -870,7 +870,7 @@ class LayerOverviewUtils extends Script {
)
try {
themeFile = new PrepareTheme(convertState, {
skipDefaultLayers: true,
skipDefaultLayers: true
}).convertStrict(
themeFile,
ConversionContext.construct([themePath], ["PrepareLayer"])
@ -919,7 +919,7 @@ class LayerOverviewUtils extends Script {
const e: string = [
`the icon for theme ${themeFile.id} is too small. Please rescale the icon at ${themeFile.icon}`,
`Even though an SVG is 'infinitely scaleable', the icon should be dimensioned bigger. One of the build steps of the theme does convert the image to a PNG (to serve as PWA-icon) and having a small dimension will cause blurry images.`,
` Width = ${width} height = ${height}; we recommend a size of at least 500px * 500px and to use a square aspect ratio.`,
` Width = ${width} height = ${height}; we recommend a size of at least 500px * 500px and to use a square aspect ratio.`
].join("\n")
err(e)
}
@ -956,7 +956,7 @@ class LayerOverviewUtils extends Script {
hideFromOverview: t.hideFromOverview ?? false,
shortDescription:
t.shortDescription ?? new Translation(t.description).FirstSentence(),
mustHaveLanguage: t.mustHaveLanguage?.length > 0,
mustHaveLanguage: t.mustHaveLanguage?.length > 0
}
}),
sharedLayers

227
src/Logic/Search/ThemeSearch.ts
View file

@ -1,19 +1,109 @@
import ThemeConfig, { MinimalThemeInformation } from "../../Models/ThemeConfig/ThemeConfig"
import { Store } from "../UIEventSource"
import UserRelatedState from "../State/UserRelatedState"
import { Utils } from "../../Utils"
import Locale from "../../UI/i18n/Locale"
import themeOverview from "../../assets/generated/theme_overview.json"
import LayerSearch from "./LayerSearch"
import SearchUtils from "./SearchUtils"
import { OsmConnection } from "../Osm/OsmConnection"
import { AndroidPolyfill } from "../Web/AndroidPolyfill"
import Fuse from "fuse.js"
import Constants from "../../Models/Constants"
import Locale from "../../UI/i18n/Locale"
import { Utils } from "../../Utils"
type ThemeSearchScore = {
theme: MinimalThemeInformation
lowest: number
perLayer?: Record<string, number>
other: number
export class ThemeSearchIndex {
private readonly themeIndex: Fuse<MinimalThemeInformation>
private readonly layerIndex: Fuse<{ id: string, description }>
constructor(language: string, themesToSearch?: MinimalThemeInformation[], layersToIgnore: string[] = []) {
const themes = themesToSearch ?? ThemeSearch.officialThemes?.themes
if (!themes) {
throw "No themes loaded. Did generate:layeroverview fail?"
}
const fuseOptions = {
ignoreLocation: true,
threshold: 0.2,
keys: [
{ name: "id", weight: 2 },
"title." + language,
"keywords." + language,
"shortDescription." + language
]
}
this.themeIndex = new Fuse(themes.filter(th => th.id !== "personal"), fuseOptions)
const toIgnore = new Set(layersToIgnore)
const layersAsList: { id: string, description: Record<string, string[]> }[] = []
for (const id in ThemeSearch.officialThemes.layers) {
if (Constants.isPriviliged(id)) {
continue
}
if (toIgnore.has(id)) {
continue
}
const l: Record<string, string[]> = ThemeSearch.officialThemes.layers[id]
layersAsList.push({ id, description: l })
}
this.layerIndex = new Fuse(layersAsList, {
includeScore: true,
minMatchCharLength: 3,
ignoreLocation: true,
threshold: 0.02,
keys: ["id", "description." + language]
})
}
public search(text: string, limit?: number): MinimalThemeInformation[] {
const scored = this.searchWithScores(text)
let result = Array.from(scored.entries())
result.sort((a, b) => b[0] - a[0])
if (limit) {
result = result.slice(0, limit)
}
return result.map(e => ThemeSearch.officialThemesById.get(e[0]))
}
public searchWithScores(text: string): Map<string, number> {
const result = new Map<string, number>()
const themeResults = this.themeIndex.search(text)
for (const themeResult of themeResults) {
result.set(themeResult.item.id, themeResult.score)
}
const layerResults = this.layerIndex.search(text)
for (const layer of layerResults) {
const matchingThemes = ThemeSearch.layersToThemes.get(layer.item.id)
const score = layer.score
matchingThemes?.forEach(th => {
const previous = result.get(th.id) ?? 10000
result.set(th.id, Math.min(previous, score * 5))
})
}
return result
}
/**
* Builds a search index containing all public and visited themes, but ignoring the layers loaded by the current theme
*/
public static fromState(state: { osmConnection: OsmConnection; theme: ThemeConfig }): Store<ThemeSearchIndex> {
const layersToIgnore = state.theme.layers.filter((l) => l.isNormal()).map((l) => l.id)
const knownHidden: Store<string[]> = UserRelatedState.initDiscoveredHiddenThemes(
state.osmConnection
).map((list) => Utils.Dedup(list))
const otherThemes: MinimalThemeInformation[] = ThemeSearch.officialThemes.themes.filter(
(th) => th.id !== state.theme.id
)
return Locale.language.map(language => {
const themes = otherThemes.concat(...knownHidden.data.map(id => ThemeSearch.officialThemesById.get(id)))
return new ThemeSearchIndex(language, themes, layersToIgnore)
},
[knownHidden]
)
}
}
export default class ThemeSearch {
@ -25,40 +115,24 @@ export default class ThemeSearch {
string,
MinimalThemeInformation
>()
/*
* For every layer id, states which themes use the layer
*/
public static readonly layersToThemes: Map<string, MinimalThemeInformation[]> = new Map()
static {
for (const th of ThemeSearch.officialThemes.themes ?? []) {
ThemeSearch.officialThemesById.set(th.id, th)
for (const layer of th.layers) {
let list = ThemeSearch.layersToThemes.get(layer)
if (!list) {
list = []
ThemeSearch.layersToThemes.set(layer, list)
}
list.push(th)
}
}
private readonly _knownHiddenThemes: Store<Set<string>>
private readonly _layersToIgnore: string[]
private readonly _otherThemes: MinimalThemeInformation[]
constructor(state: { osmConnection: OsmConnection; theme: ThemeConfig }) {
this._layersToIgnore = state.theme.layers.filter((l) => l.isNormal()).map((l) => l.id)
this._knownHiddenThemes = UserRelatedState.initDiscoveredHiddenThemes(
state.osmConnection
).map((list) => new Set(list))
this._otherThemes = ThemeSearch.officialThemes.themes.filter(
(th) => th.id !== state.theme.id
)
}
public search(query: string, limit: number, threshold: number = 3): MinimalThemeInformation[] {
if (query.length < 1) {
return []
}
const sorted = ThemeSearch.sortedByLowestScores(
query,
this._otherThemes,
this._layersToIgnore
)
return sorted
.filter((sorted) => sorted.lowest < threshold)
.map((th) => th.theme)
.filter((th) => !th.hideFromOverview || this._knownHiddenThemes.data.has(th.id))
.slice(0, limit)
}
public static createUrlFor(layout: { id: string }, state?: { layoutToUse?: { id } }): string {
@ -97,82 +171,5 @@ export default class ThemeSearch {
return `${linkPrefix}`
}
/**
* Returns a score based on textual search
*
* Note that, if `query.length < 3`, layers are _not_ searched because this takes too much time
* @param query
* @param themes
* @param ignoreLayers
* @private
*/
private static scoreThemes(
query: string,
themes: MinimalThemeInformation[],
ignoreLayers: string[] = undefined
): Record<string, ThemeSearchScore> {
if (query?.length < 1) {
return undefined
}
themes = Utils.NoNullInplace(themes)
let options: { blacklist: Set<string> } = undefined
if (ignoreLayers?.length > 0) {
options = { blacklist: new Set(ignoreLayers) }
}
const layerScores = query.length < 3 ? {} : LayerSearch.scoreLayers(query, options)
const results: Record<string, ThemeSearchScore> = {}
for (const layoutInfo of themes) {
const theme = layoutInfo.id
if (theme === "personal") {
continue
}
if (Utils.simplifyStringForSearch(theme) === query) {
results[theme] = {
theme: layoutInfo,
lowest: -1,
other: 0,
}
continue
}
const perLayer = Utils.asRecord(layoutInfo.layers ?? [], (layer) => layerScores[layer])
const language = Locale.language.data
const keywords = Utils.NoNullInplace([
layoutInfo.shortDescription,
layoutInfo.title,
]).map((item) => (typeof item === "string" ? item : item[language] ?? item["*"]))
const other = Math.min(
SearchUtils.scoreKeywords(query, keywords),
SearchUtils.scoreKeywords(query, layoutInfo.keywords)
)
const lowest = Math.min(other, ...Object.values(perLayer))
results[theme] = {
theme: layoutInfo,
perLayer,
other,
lowest,
}
}
return results
}
public static sortedByLowestScores(
search: string,
themes: MinimalThemeInformation[],
ignoreLayers: string[] = []
): ThemeSearchScore[] {
const scored = Object.values(this.scoreThemes(search, themes, ignoreLayers))
scored.sort((a, b) => a.lowest - b.lowest)
return scored
}
public static sortedByLowest(
search: string,
themes: MinimalThemeInformation[],
ignoreLayers: string[] = []
): MinimalThemeInformation[] {
return this.sortedByLowestScores(search, themes, ignoreLayers).map((th) => th.theme)
}
}

6
src/Logic/State/SearchState.ts
View file

@ -4,7 +4,7 @@ import CombinedSearcher from "../Search/CombinedSearcher"
import FilterSearch, { FilterSearchResult } from "../Search/FilterSearch"
import LocalElementSearch from "../Search/LocalElementSearch"
import CoordinateSearch from "../Search/CoordinateSearch"
import ThemeSearch from "../Search/ThemeSearch"
import { ThemeSearchIndex } from "../Search/ThemeSearch"
import OpenStreetMapIdSearch from "../Search/OpenStreetMapIdSearch"
import PhotonSearch from "../Search/PhotonSearch"
import ThemeViewState from "../../Models/ThemeViewState"
@ -67,8 +67,8 @@ export default class SearchState {
Stores.concat(suggestions).map((suggestions) => CombinedSearcher.merge(suggestions))
)
const themeSearch = new ThemeSearch(state)
this.themeSuggestions = this.searchTerm.mapD((query) => themeSearch.search(query, 3))
const themeSearch = ThemeSearchIndex.fromState(state)
this.themeSuggestions = this.searchTerm.mapD((query) => themeSearch.data.search(query, 3), [themeSearch])
const layerSearch = new LayerSearch(state.theme)
this.layerSuggestions = this.searchTerm.mapD((query) => layerSearch.search(query, 5))

31
src/UI/AllThemesGui.svelte
View file

@ -18,15 +18,15 @@
import Mastodon from "../assets/svg/Mastodon.svelte"
import Liberapay from "../assets/svg/Liberapay.svelte"
import Bug from "../assets/svg/Bug.svelte"
import Github from "../assets/svg/Github.svelte"
import { Utils } from "../Utils"
import { ArrowTrendingUp } from "@babeard/svelte-heroicons/solid/ArrowTrendingUp"
import Searchbar from "./Base/Searchbar.svelte"
import ThemeSearch from "../Logic/Search/ThemeSearch"
import ThemeSearch, { ThemeSearchIndex } from "../Logic/Search/ThemeSearch"
import SearchUtils from "../Logic/Search/SearchUtils"
import ChevronDoubleRight from "@babeard/svelte-heroicons/mini/ChevronDoubleRight"
import { AndroidPolyfill } from "../Logic/Web/AndroidPolyfill"
import Forgejo from "../assets/svg/Forgejo.svelte"
import Locale from "./i18n/Locale"
AndroidPolyfill.init().then(() => console.log("Android polyfill setup completed"))
const featureSwitches = new OsmConnectionFeatureSwitches()
const osmConnection = new OsmConnection({
@ -65,29 +65,26 @@
state.installedUserThemes
).mapD((stableIds) => Utils.NoNullInplace(stableIds.map((id) => state.getUnofficialTheme(id))))
function filtered(themes: Store<MinimalThemeInformation[]>): Store<MinimalThemeInformation[]> {
const searchIndex = Locale.language.map(language => {
return new ThemeSearchIndex(language, themes.data)
}, [themes])
return searchStable.map(
(search) => {
(searchTerm) => {
if (!themes.data) {
return []
}
if (!search) {
if (!searchTerm) {
return themes.data
}
const start = new Date().getTime()
const scores = ThemeSearch.sortedByLowestScores(search, themes.data)
const end = new Date().getTime()
console.trace("Scores for", search, "are", scores, "searching took", end - start, "ms")
const strict = scores.filter((sc) => sc.lowest < 2)
if (strict.length > 0) {
return strict.map((sc) => sc.theme)
}
return scores
.filter((sc) => sc.lowest < 4)
.slice(0, 6)
.map((sc) => sc.theme)
const index = searchIndex.data
return index.search(searchTerm)
},
[themes]
[searchIndex]
)
}