forked from MapComplete/MapComplete
Performance: load less data for branded items, use caching
This commit is contained in:
parent
2a3ee4c4f6
commit
164b02c8ff
2 changed files with 186 additions and 147 deletions
|
|
@ -82,6 +82,7 @@ data.mapcomplete.org {
|
|||
header {
|
||||
+Permissions-Policy "interest-cohort=()"
|
||||
+Access-Control-Allow-Origin *
|
||||
Cache-Control: max-age=86400, public, stale-while-revalidate=86400,stale-if-error=86400
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -55,77 +55,22 @@ export interface NSIItem {
|
|||
ext?: string
|
||||
}
|
||||
|
||||
export default class NameSuggestionIndex {
|
||||
export class NameSuggestionIndexLight {
|
||||
public static readonly supportedTypes = ["brand", "flag", "operator", "transit"] as const
|
||||
private readonly nsiFile: Readonly<NSIFile>
|
||||
private readonly nsiWdFile: Readonly<
|
||||
Record<
|
||||
string,
|
||||
{
|
||||
logos: { wikidata?: string; facebook?: string }
|
||||
}
|
||||
>
|
||||
>
|
||||
protected readonly _serverLocation: string
|
||||
protected readonly nsiFile: Readonly<NSIFile>
|
||||
private readonly loco: LocationConflation // Some additional boundaries
|
||||
private static initedLight: NameSuggestionIndexLight = undefined
|
||||
|
||||
private loco: LocationConflation // Some additional boundaries
|
||||
|
||||
private _supportedTypes: string[]
|
||||
private _serverLocation: string
|
||||
|
||||
private constructor(
|
||||
protected constructor(
|
||||
serverLocation: string,
|
||||
nsiFile: Readonly<NSIFile>,
|
||||
nsiWdFile: Readonly<
|
||||
Record<
|
||||
string,
|
||||
{
|
||||
logos: { wikidata?: string; facebook?: string }
|
||||
}
|
||||
>
|
||||
>,
|
||||
features: Readonly<FeatureCollection>
|
||||
) {
|
||||
this._serverLocation = serverLocation
|
||||
this.nsiFile = nsiFile
|
||||
this.nsiWdFile = nsiWdFile
|
||||
this.loco = new LocationConflation(features)
|
||||
}
|
||||
|
||||
private static inited: NameSuggestionIndex = undefined
|
||||
|
||||
public static async getNsiIndex(): Promise<NameSuggestionIndex> {
|
||||
if (NameSuggestionIndex.inited) {
|
||||
return NameSuggestionIndex.inited
|
||||
}
|
||||
const [nsi, nsiWd, features] = await Promise.all(
|
||||
[
|
||||
"./assets/data/nsi/nsi.min.json",
|
||||
"./assets/data/nsi/wikidata.min.json",
|
||||
"./assets/data/nsi/featureCollection.min.json",
|
||||
].map((url) => Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30))
|
||||
)
|
||||
NameSuggestionIndex.inited = new NameSuggestionIndex(
|
||||
Constants.nsiLogosEndpoint,
|
||||
<any>nsi,
|
||||
<any>nsiWd["wikidata"],
|
||||
<any>features
|
||||
)
|
||||
return NameSuggestionIndex.inited
|
||||
}
|
||||
|
||||
public supportedTypes(): string[] {
|
||||
if (this._supportedTypes) {
|
||||
return this._supportedTypes
|
||||
}
|
||||
const keys = Object.keys(this.nsiFile.nsi)
|
||||
const all = keys.map((k) => this.nsiFile.nsi[k].properties.path.split("/")[0])
|
||||
this._supportedTypes = Utils.Dedup(all).map((s) => {
|
||||
if (s.endsWith("s")) {
|
||||
s = s.substring(0, s.length - 1)
|
||||
}
|
||||
return s
|
||||
})
|
||||
return this._supportedTypes
|
||||
NameSuggestionIndexLight.initedLight = this
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -165,25 +110,6 @@ export default class NameSuggestionIndex {
|
|||
return merged
|
||||
}
|
||||
|
||||
public isSvg(nsiItem: NSIItem, type: string): boolean | undefined {
|
||||
if (this.nsiWdFile === undefined) {
|
||||
throw (
|
||||
"nsiWdi file is not loaded, cannot determine if " + nsiItem.id + " has an SVG image"
|
||||
)
|
||||
}
|
||||
const logos = this.nsiWdFile[nsiItem?.tags?.[type + ":wikidata"]]?.logos
|
||||
if (!logos) {
|
||||
return undefined
|
||||
}
|
||||
if (logos.facebook) {
|
||||
return false
|
||||
}
|
||||
const url: string = logos.wikidata
|
||||
if (url.toLowerCase().endsWith(".svg")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
public async generateMappings(
|
||||
type: string,
|
||||
|
|
@ -236,7 +162,7 @@ export default class NameSuggestionIndex {
|
|||
// As such, it should be "true" but this is not supported
|
||||
priorityIf: frequency > 0 ? new RegexTag("id", /.*/) : undefined,
|
||||
searchTerms: { "*": [nsiItem.displayName, nsiItem.id] },
|
||||
frequency: frequency ?? -1,
|
||||
frequency: frequency ?? -1
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -247,68 +173,21 @@ export default class NameSuggestionIndex {
|
|||
return mappings
|
||||
}
|
||||
|
||||
public supportedTags(
|
||||
type: "operator" | "brand" | "flag" | "transit" | string
|
||||
): Record<string, string[]> {
|
||||
const tags: Record<string, string[]> = {}
|
||||
const keys = Object.keys(this.nsiFile.nsi)
|
||||
for (const key of keys) {
|
||||
const nsiItem = this.nsiFile.nsi[key]
|
||||
const path = nsiItem.properties.path
|
||||
const [osmType, osmkey, osmvalue] = path.split("/")
|
||||
if (type !== osmType && type + "s" !== osmType) {
|
||||
continue
|
||||
}
|
||||
if (!tags[osmkey]) {
|
||||
tags[osmkey] = []
|
||||
}
|
||||
tags[osmkey].push(osmvalue)
|
||||
public getIconUrl(nsiItem: NSIItem): string | undefined {
|
||||
const baseUrl = this._serverLocation
|
||||
if (!nsiItem.ext || baseUrl === null) {
|
||||
// No extension -> there is no logo
|
||||
return undefined
|
||||
}
|
||||
return tags
|
||||
return baseUrl + "/logos/" + nsiItem.id + "." + nsiItem.ext
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a list of all brands/operators
|
||||
* @param type
|
||||
*/
|
||||
public allPossible(type: string): NSIItem[] {
|
||||
const options: NSIItem[] = []
|
||||
const tags = this.supportedTags(type)
|
||||
for (const osmKey in tags) {
|
||||
const values = tags[osmKey]
|
||||
for (const osmValue of values) {
|
||||
const suggestions = this.getSuggestionsForKV(type, osmKey, osmValue)
|
||||
if (!suggestions) {
|
||||
console.warn("No suggestions found for", type, osmKey, osmValue)
|
||||
continue
|
||||
}
|
||||
options.push(...suggestions)
|
||||
}
|
||||
}
|
||||
return options
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param country: a string containing one or more country codes, separated by ";"
|
||||
* @param location: center point of the feature, should be [lon, lat]
|
||||
*/
|
||||
public getSuggestionsFor(
|
||||
type: string,
|
||||
tags: { key: string; value: string }[],
|
||||
country: string = undefined,
|
||||
location: [number, number] = undefined
|
||||
): NSIItem[] {
|
||||
return tags.flatMap((tag) =>
|
||||
this.getSuggestionsForKV(type, tag.key, tag.value, country, location)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Caching for the resolved sets, as they can take a while
|
||||
* @private
|
||||
*/
|
||||
private static resolvedSets: Record<string, any> = {}
|
||||
private static resolvedSets: Record<string, { type, location, id, feature }> = {}
|
||||
|
||||
/**
|
||||
* Returns all suggestions for the given type (brand|operator) and main tag.
|
||||
|
|
@ -379,6 +258,173 @@ export default class NameSuggestionIndex {
|
|||
})
|
||||
}
|
||||
|
||||
public static async singleton(): Promise<NameSuggestionIndexLight> {
|
||||
if (NameSuggestionIndexLight.initedLight) {
|
||||
return NameSuggestionIndexLight.initedLight
|
||||
}
|
||||
const [nsi, features] = await Promise.all(
|
||||
[
|
||||
"./assets/data/nsi/nsi.min.json",
|
||||
"./assets/data/nsi/featureCollection.min.json"
|
||||
].map((url) => Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30))
|
||||
)
|
||||
return new NameSuggestionIndexLight(
|
||||
Constants.nsiLogosEndpoint,
|
||||
<any>nsi,
|
||||
<any>features
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export default class NameSuggestionIndex extends NameSuggestionIndexLight {
|
||||
private readonly nsiWdFile: Readonly<
|
||||
Record<
|
||||
string,
|
||||
{
|
||||
logos: { wikidata?: string; facebook?: string }
|
||||
}
|
||||
>
|
||||
>
|
||||
|
||||
|
||||
private _supportedTypes: string[]
|
||||
private static inited: NameSuggestionIndex = undefined
|
||||
|
||||
private constructor(
|
||||
serverLocation: string,
|
||||
nsiFile: Readonly<NSIFile>,
|
||||
nsiWdFile: Readonly<
|
||||
Record<
|
||||
string,
|
||||
{
|
||||
logos: { wikidata?: string; facebook?: string }
|
||||
}
|
||||
>
|
||||
>,
|
||||
features: Readonly<FeatureCollection>
|
||||
) {
|
||||
super(serverLocation, nsiFile, features)
|
||||
this.nsiWdFile = nsiWdFile
|
||||
NameSuggestionIndex.inited = this
|
||||
}
|
||||
|
||||
|
||||
public static async getNsiIndex(): Promise<NameSuggestionIndex> {
|
||||
if (NameSuggestionIndex.inited) {
|
||||
return NameSuggestionIndex.inited
|
||||
}
|
||||
const [nsi, nsiWd, features] = await Promise.all(
|
||||
[
|
||||
"./assets/data/nsi/nsi.min.json",
|
||||
"./assets/data/nsi/wikidata.min.json",
|
||||
"./assets/data/nsi/featureCollection.min.json"
|
||||
].map((url) => Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30))
|
||||
)
|
||||
return new NameSuggestionIndex(
|
||||
Constants.nsiLogosEndpoint,
|
||||
<any>nsi,
|
||||
<any>nsiWd["wikidata"],
|
||||
<any>features
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
public supportedTypes(): string[] {
|
||||
if (this._supportedTypes) {
|
||||
return this._supportedTypes
|
||||
}
|
||||
const keys = Object.keys(this.nsiFile.nsi)
|
||||
const all = keys.map((k) => this.nsiFile.nsi[k].properties.path.split("/")[0])
|
||||
this._supportedTypes = Utils.Dedup(all).map((s) => {
|
||||
if (s.endsWith("s")) {
|
||||
s = s.substring(0, s.length - 1)
|
||||
}
|
||||
return s
|
||||
})
|
||||
return this._supportedTypes
|
||||
}
|
||||
|
||||
|
||||
public isSvg(nsiItem: NSIItem, type: string): boolean | undefined {
|
||||
if (this.nsiWdFile === undefined) {
|
||||
throw (
|
||||
"nsiWdi file is not loaded, cannot determine if " + nsiItem.id + " has an SVG image"
|
||||
)
|
||||
}
|
||||
const logos = this.nsiWdFile[nsiItem?.tags?.[type + ":wikidata"]]?.logos
|
||||
if (!logos) {
|
||||
return undefined
|
||||
}
|
||||
if (logos.facebook) {
|
||||
return false
|
||||
}
|
||||
const url: string = logos.wikidata
|
||||
if (url.toLowerCase().endsWith(".svg")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
public supportedTags(
|
||||
type: "operator" | "brand" | "flag" | "transit" | string
|
||||
): Record<string, string[]> {
|
||||
const tags: Record<string, string[]> = {}
|
||||
const keys = Object.keys(this.nsiFile.nsi)
|
||||
for (const key of keys) {
|
||||
const nsiItem = this.nsiFile.nsi[key]
|
||||
const path = nsiItem.properties.path
|
||||
const [osmType, osmkey, osmvalue] = path.split("/")
|
||||
if (type !== osmType && type + "s" !== osmType) {
|
||||
continue
|
||||
}
|
||||
if (!tags[osmkey]) {
|
||||
tags[osmkey] = []
|
||||
}
|
||||
tags[osmkey].push(osmvalue)
|
||||
}
|
||||
return tags
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a list of all brands/operators
|
||||
* @param type
|
||||
*/
|
||||
public allPossible(type: string): NSIItem[] {
|
||||
const options: NSIItem[] = []
|
||||
const tags = this.supportedTags(type)
|
||||
for (const osmKey in tags) {
|
||||
const values = tags[osmKey]
|
||||
for (const osmValue of values) {
|
||||
const suggestions = this.getSuggestionsForKV(type, osmKey, osmValue)
|
||||
if (!suggestions) {
|
||||
console.warn("No suggestions found for", type, osmKey, osmValue)
|
||||
continue
|
||||
}
|
||||
options.push(...suggestions)
|
||||
}
|
||||
}
|
||||
return options
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param country: a string containing one or more country codes, separated by ";"
|
||||
* @param location: center point of the feature, should be [lon, lat]
|
||||
*/
|
||||
public getSuggestionsFor(
|
||||
type: string,
|
||||
tags: { key: string; value: string }[],
|
||||
country: string = undefined,
|
||||
location: [number, number] = undefined
|
||||
): NSIItem[] {
|
||||
return tags.flatMap((tag) =>
|
||||
this.getSuggestionsForKV(type, tag.key, tag.value, country, location)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
public static async generateMappings(
|
||||
key: string,
|
||||
tags: Exclude<Record<string, string>, undefined | null>,
|
||||
|
|
@ -388,7 +434,7 @@ export default class NameSuggestionIndex {
|
|||
sortByFrequency: boolean
|
||||
}
|
||||
): Promise<Mapping[]> {
|
||||
const nsi = await NameSuggestionIndex.getNsiIndex()
|
||||
const nsi = await NameSuggestionIndexLight.singleton()
|
||||
return nsi.generateMappings(key, tags, country, center, options)
|
||||
}
|
||||
|
||||
|
|
@ -403,14 +449,6 @@ export default class NameSuggestionIndex {
|
|||
return logos?.facebook ?? logos?.wikidata
|
||||
}
|
||||
|
||||
public getIconUrl(nsiItem: NSIItem): string | undefined {
|
||||
const baseUrl = this._serverLocation
|
||||
if (!nsiItem.ext || baseUrl === null) {
|
||||
// No extension -> there is no logo
|
||||
return undefined
|
||||
}
|
||||
return baseUrl +"/logos/"+ nsiItem.id + "." + nsiItem.ext
|
||||
}
|
||||
|
||||
private static readonly brandPrefix = ["name", "alt_name", "operator", "brand"] as const
|
||||
|
||||
|
|
@ -425,8 +463,8 @@ export default class NameSuggestionIndex {
|
|||
static asFilterTags(
|
||||
item: NSIItem
|
||||
): string | { and: TagConfigJson[] } | { or: TagConfigJson[] } {
|
||||
let brandDetection: string[] = []
|
||||
let required: string[] = []
|
||||
const brandDetection: string[] = []
|
||||
const required: string[] = []
|
||||
const tags: Record<string, string> = item.tags
|
||||
for (const k in tags) {
|
||||
if (NameSuggestionIndex.brandPrefix.some((br) => k === br || k.startsWith(br + ":"))) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue