Robin-van-der-Linde/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Performance: load NSI when needed, should decrease bundle size

Browse source
This commit is contained in:
Pieter Vander Vennet 2025-01-02 03:38:15 +01:00
parent 4fcdd8ba5a
commit a79557c87c
7 changed files with 122 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
scripts/build.sh
View file

@ -24,6 +24,8 @@ else
exit 1
fi
cp node_modules/name-suggestion-index/dist/nsi.json public/assets/data/nsi
cp node_modules/name-suggestion-index/dist/wikidata.min.json public/assets/data/nsi
export NODE_OPTIONS=--max-old-space-size=16000
which vite

93
src/Logic/Web/NameSuggestionIndex.ts
View file

@ -1,6 +1,3 @@
import * as nsi from "../../../node_modules/name-suggestion-index/dist/nsi.json"
import * as nsiWD from "../../../node_modules/name-suggestion-index/dist/wikidata.min.json"
import * as nsiFeatures from "../../../node_modules/name-suggestion-index/dist/featureCollection.json"
import { LocationConflation } from "@rapideditor/location-conflation"
import type { Feature, MultiPolygon } from "geojson"
@ -56,27 +53,53 @@ export interface NSIItem {
}
export default class NameSuggestionIndex {
private static readonly nsiFile: Readonly<NSIFile> = <any>nsi
private static readonly nsiWdFile: Readonly<
public static readonly supportedTypes = ["brand",
"flag",
"operator",
"transit"] as const
private readonly nsiFile: Readonly<NSIFile>
private readonly nsiWdFile: Readonly<
Record<
string,
{
logos: { wikidata?: string; facebook?: string }
}
>
> = <any>nsiWD["wikidata"]
>
private static loco = new LocationConflation(nsiFeatures) // Some additional boundaries
private static _supportedTypes: string[]
private _supportedTypes: string[]
public static supportedTypes(): string[] {
constructor(nsiFile: Readonly<NSIFile>, nsiWdFile: Readonly<
Record<
string,
{
logos: { wikidata?: string; facebook?: string }
}
>>) {
this.nsiFile = nsiFile
this.nsiWdFile = nsiWdFile
}
private static inited: NameSuggestionIndex = undefined
public static async getNsiIndex(): Promise<NameSuggestionIndex> {
if (NameSuggestionIndex.inited) {
return NameSuggestionIndex.inited
}
const [nsi, nsiWd] = await Promise.all(["assets/data/nsi/nsi.json", "assets/data/nsi/wikidata.min.json"].map(url => Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30)))
NameSuggestionIndex.inited = new NameSuggestionIndex(<any>nsi, <any>nsiWd["wikidata"])
return NameSuggestionIndex.inited
}
public supportedTypes(): string[] {
if (this._supportedTypes) {
return this._supportedTypes
}
const keys = Object.keys(NameSuggestionIndex.nsiFile.nsi)
const keys = Object.keys(this.nsiFile.nsi)
const all = keys.map(
(k) => NameSuggestionIndex.nsiFile.nsi[k].properties.path.split("/")[0]
(k) => this.nsiFile.nsi[k].properties.path.split("/")[0],
)
this._supportedTypes = Utils.Dedup(all).map((s) => {
if (s.endsWith("s")) {
@ -99,13 +122,13 @@ export default class NameSuggestionIndex {
try {
return Utils.downloadJsonCached<Record<string, number>>(
`./assets/data/nsi/stats/${type}.${c.toUpperCase()}.json`,
24 * 60 * 60 * 1000
24 * 60 * 60 * 1000,
)
} catch (e) {
console.error("Could not fetch " + type + " statistics due to", e)
return undefined
}
})
}),
)
stats = Utils.NoNull(stats)
if (stats.length === 1) {
@ -123,7 +146,10 @@ export default class NameSuggestionIndex {
return merged
}
public static isSvg(nsiItem: NSIItem, type: string): boolean | undefined {
public isSvg(nsiItem: NSIItem, type: string): boolean | undefined {
if (this.nsiWdFile === undefined) {
throw "nsiWdi file is not loaded, cannot determine if " + nsiItem.id + " has an SVG image"
}
const logos = this.nsiWdFile[nsiItem?.tags?.[type + ":wikidata"]]?.logos
if (!logos) {
return undefined
@ -138,7 +164,7 @@ export default class NameSuggestionIndex {
return false
}
public static async generateMappings(
public async generateMappings(
type: string,
tags: Record<string, string>,
country: string[],
@ -148,7 +174,7 @@ export default class NameSuggestionIndex {
* If set, sort by frequency instead of alphabetically
*/
sortByFrequency: boolean
}
},
): Promise<Mapping[]> {
const mappings: (Mapping & { frequency: number })[] = []
const frequencies = await NameSuggestionIndex.fetchFrequenciesFor(type, country)
@ -157,12 +183,12 @@ export default class NameSuggestionIndex {
continue
}
const value = tags[key]
const actualBrands = NameSuggestionIndex.getSuggestionsForKV(
const actualBrands = this.getSuggestionsForKV(
type,
key,
value,
country.join(";"),
location
location,
)
if (!actualBrands) {
continue
@ -177,7 +203,7 @@ export default class NameSuggestionIndex {
if (hasIcon) {
// Using <img src=...> works fine without an extension for JPG and PNG, but _not_ svg :(
icon = "./assets/data/nsi/logos/" + nsiItem.id
if (NameSuggestionIndex.isSvg(nsiItem, type)) {
if (this.isSvg(nsiItem, type)) {
icon = icon + ".svg"
}
}
@ -207,13 +233,13 @@ export default class NameSuggestionIndex {
return mappings
}
public static supportedTags(
type: "operator" | "brand" | "flag" | "transit" | string
public supportedTags(
type: "operator" | "brand" | "flag" | "transit" | string,
): Record<string, string[]> {
const tags: Record<string, string[]> = {}
const keys = Object.keys(NameSuggestionIndex.nsiFile.nsi)
const keys = Object.keys(this.nsiFile.nsi)
for (const key of keys) {
const nsiItem = NameSuggestionIndex.nsiFile.nsi[key]
const nsiItem = this.nsiFile.nsi[key]
const path = nsiItem.properties.path
const [osmType, osmkey, osmvalue] = path.split("/")
if (type !== osmType && type + "s" !== osmType) {
@ -231,9 +257,9 @@ export default class NameSuggestionIndex {
* Returns a list of all brands/operators
* @param type
*/
public static allPossible(type: "brand" | "operator"): NSIItem[] {
public allPossible(type: "brand" | "operator"): NSIItem[] {
const options: NSIItem[] = []
const tags = NameSuggestionIndex.supportedTags(type)
const tags = this.supportedTags(type)
for (const osmKey in tags) {
const values = tags[osmKey]
for (const osmValue of values) {
@ -249,14 +275,14 @@ export default class NameSuggestionIndex {
* @param country: a string containing one or more country codes, separated by ";"
* @param location: center point of the feature, should be [lon, lat]
*/
public static getSuggestionsFor(
public getSuggestionsFor(
type: string,
tags: { key: string; value: string }[],
country: string = undefined,
location: [number, number] = undefined
location: [number, number] = undefined,
): NSIItem[] {
return tags.flatMap((tag) =>
this.getSuggestionsForKV(type, tag.key, tag.value, country, location)
this.getSuggestionsForKV(type, tag.key, tag.value, country, location),
)
}
@ -274,15 +300,15 @@ export default class NameSuggestionIndex {
* @param country: a string containing one or more country codes, separated by ";"
* @param location: center point of the feature, should be [lon, lat]
*/
public static getSuggestionsForKV(
public getSuggestionsForKV(
type: string,
key: string,
value: string,
country: string = undefined,
location: [number, number] = undefined
location: [number, number] = undefined,
): NSIItem[] {
const path = `${type}s/${key}/${value}`
const entry = NameSuggestionIndex.nsiFile.nsi[path]
const entry = this.nsiFile.nsi[path]
const countries = country?.split(";") ?? []
return entry?.items?.filter((i) => {
if (i.locationSet.include.indexOf("001") >= 0) {
@ -335,4 +361,11 @@ export default class NameSuggestionIndex {
return false
})
}
public static async generateMappings(key: string, tags: Exclude<Record<string, string>, undefined | null>, country: string[], center: [number, number], options: {
sortByFrequency: boolean
}): Promise<Mapping[]> {
const nsi = await NameSuggestionIndex.getNsiIndex()
return nsi.generateMappings(key, tags, country, center, options)
}
}

20
src/Models/ThemeConfig/Conversion/MiscTagRenderingChecks.ts
View file

@ -1,17 +1,13 @@
import { DesugaringStep } from "./Conversion"
import { TagRenderingConfigJson } from "../Json/TagRenderingConfigJson"
import { LayerConfigJson } from "../Json/LayerConfigJson"
import {
MappingConfigJson,
QuestionableTagRenderingConfigJson,
} from "../Json/QuestionableTagRenderingConfigJson"
import { MappingConfigJson, QuestionableTagRenderingConfigJson } from "../Json/QuestionableTagRenderingConfigJson"
import { ConversionContext } from "./ConversionContext"
import { Translation } from "../../../UI/i18n/Translation"
import NameSuggestionIndex from "../../../Logic/Web/NameSuggestionIndex"
import { TagUtils } from "../../../Logic/Tags/TagUtils"
import { Tag } from "../../../Logic/Tags/Tag"
import Validators from "../../../UI/InputElement/Validators"
import { CheckTranslation } from "./Validation"
import NameSuggestionIndex from "../../../Logic/Web/NameSuggestionIndex"
export class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
private readonly _layerConfig: LayerConfigJson
@ -197,11 +193,11 @@ export class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJso
}
}
if (
this._layerConfig?.source?.osmTags &&
NameSuggestionIndex.supportedTypes().indexOf(json.freeform.key) >= 0
this._layerConfig?.source?.["osmTags"] &&
NameSuggestionIndex.supportedTypes.indexOf(<any> json.freeform.key) >= 0
) {
const tags = TagUtils.TagD(this._layerConfig?.source?.osmTags)?.usedTags()
const suggestions = NameSuggestionIndex.getSuggestionsFor(json.freeform.key, tags)
const tags = TagUtils.TagD(this._layerConfig?.source?.["osmTags"])?.usedTags()
/* const suggestions = nameSuggestionIndexBundled.getSuggestionsFor(json.freeform.key, tags)
if (suggestions === undefined) {
context
.enters("freeform", "type")
@ -209,8 +205,8 @@ export class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJso
"No entry found in the 'Name Suggestion Index'. None of the 'osmSource'-tags match an entry in the NSI.\n\tOsmSource-tags are " +
tags.map((t) => new Tag(t.key, t.value).asHumanString()).join(" ; ")
)
}
} else if (json.freeform.type === "nsi") {
}*/
} else if (json.freeform["type"] === "nsi") {
context
.enters("freeform", "type")
.warn(

6
src/Models/ThemeConfig/TagRenderingConfig.ts
View file

@ -999,7 +999,7 @@ export class TagRenderingConfigUtils {
tags: UIEventSource<Record<string, string>>,
feature?: Feature
): Store<TagRenderingConfig> {
const isNSI = NameSuggestionIndex.supportedTypes().indexOf(config.freeform?.key) >= 0
const isNSI = NameSuggestionIndex.supportedTypes.indexOf(<any> config.freeform?.key) >= 0
if (!isNSI) {
return new ImmutableStore(config)
}
@ -1019,8 +1019,8 @@ export class TagRenderingConfigUtils {
)
)
})
return extraMappings.map((extraMappings) => {
if (!extraMappings || extraMappings.length == 0) {
return extraMappings.mapD((extraMappings) => {
if (extraMappings.length == 0) {
return config
}
const clone: TagRenderingConfig = Object.create(config)

5
src/UI/Popup/TagRendering/TagRenderingAnswerDynamic.svelte
View file

@ -7,6 +7,7 @@
import { UIEventSource } from "../../../Logic/UIEventSource"
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
import TagRenderingAnswer from "./TagRenderingAnswer.svelte"
import Loading from "../../Base/Loading.svelte"
export let tags: UIEventSource<Record<string, string> | undefined>
@ -20,6 +21,9 @@
let dynamicConfig = TagRenderingConfigUtils.withNameSuggestionIndex(config, tags, selectedElement)
</script>
{#if $dynamicConfig === undefined}
<Loading />
{:else}
<TagRenderingAnswer
{selectedElement}
{layer}
@ -29,3 +33,4 @@
{tags}
{state}
/>
{/if}

5
src/UI/Popup/TagRendering/TagRenderingEditableDynamic.svelte
View file

@ -7,6 +7,7 @@
import type { SpecialVisualizationState } from "../../SpecialVisualization"
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
import TagRenderingEditable from "./TagRenderingEditable.svelte"
import Loading from "../../Base/Loading.svelte"
export let config: TagRenderingConfig
export let tags: UIEventSource<Record<string, string>>
@ -23,6 +24,7 @@
let dynamicConfig = TagRenderingConfigUtils.withNameSuggestionIndex(config, tags, selectedElement)
</script>
{#if $dynamicConfig}
<TagRenderingEditable
config={$dynamicConfig}
{editMode}
@ -34,3 +36,6 @@
{selectedElement}
{tags}
/>
{:else}
<Loading />
{/if}

5
src/UI/Popup/TagRendering/TagRenderingQuestionDynamic.svelte
View file

@ -18,6 +18,7 @@
import { twJoin } from "tailwind-merge"
import Tr from "../../Base/Tr.svelte"
import { TrashIcon } from "@rgossiaux/svelte-heroicons/solid"
import Loading from "../../Base/Loading.svelte"
export let config: TagRenderingConfig
export let tags: UIEventSource<Record<string, string>>
@ -31,6 +32,7 @@
let dynamicConfig = TagRenderingConfigUtils.withNameSuggestionIndex(config, tags, selectedElement)
</script>
{#if $dynamicConfig }
<TagRenderingQuestion
{tags}
config={$dynamicConfig}
@ -42,3 +44,6 @@
>
<slot name="cancel" slot="cancel" />
</TagRenderingQuestion>
{:else}
<Loading />
{/if}