forked from MapComplete/MapComplete
Merge develop
This commit is contained in:
commit
baa7379fbf
7880 changed files with 2079327 additions and 39792 deletions
|
|
@ -1,12 +1,13 @@
|
|||
import * as nsiFeatures from "../../../node_modules/name-suggestion-index/dist/featureCollection.json"
|
||||
import { LocationConflation } from "@rapideditor/location-conflation"
|
||||
import type { Feature, MultiPolygon } from "geojson"
|
||||
import type { Feature, FeatureCollection, MultiPolygon } from "geojson"
|
||||
import { Utils } from "../../Utils"
|
||||
import * as turf from "@turf/turf"
|
||||
import { Mapping } from "../../Models/ThemeConfig/TagRenderingConfig"
|
||||
import { Tag } from "../Tags/Tag"
|
||||
import { TypedTranslation } from "../../UI/i18n/Translation"
|
||||
import { RegexTag } from "../Tags/RegexTag"
|
||||
import { TagConfigJson } from "../../Models/ThemeConfig/Json/TagConfigJson"
|
||||
import { TagUtils } from "../Tags/TagUtils"
|
||||
|
||||
/**
|
||||
* Main name suggestion index file
|
||||
|
|
@ -42,17 +43,18 @@ interface NSIEntry {
|
|||
* Represents a single brand/operator/flagpole/...
|
||||
*/
|
||||
export interface NSIItem {
|
||||
displayName: string
|
||||
id: string
|
||||
readonly displayName: string
|
||||
readonly id: string
|
||||
locationSet: {
|
||||
include: string[]
|
||||
exclude: string[]
|
||||
}
|
||||
tags: Record<string, string>
|
||||
fromTemplate?: boolean
|
||||
readonly tags: Readonly<Record<string, string>>
|
||||
readonly fromTemplate?: boolean
|
||||
}
|
||||
|
||||
export default class NameSuggestionIndex {
|
||||
|
||||
public static readonly supportedTypes = ["brand", "flag", "operator", "transit"] as const
|
||||
private readonly nsiFile: Readonly<NSIFile>
|
||||
private readonly nsiWdFile: Readonly<
|
||||
|
|
@ -64,7 +66,7 @@ export default class NameSuggestionIndex {
|
|||
>
|
||||
>
|
||||
|
||||
private static loco = new LocationConflation(nsiFeatures) // Some additional boundaries
|
||||
private loco: LocationConflation // Some additional boundaries
|
||||
|
||||
private _supportedTypes: string[]
|
||||
|
||||
|
|
@ -77,10 +79,12 @@ export default class NameSuggestionIndex {
|
|||
logos: { wikidata?: string; facebook?: string }
|
||||
}
|
||||
>
|
||||
>
|
||||
>,
|
||||
features: Readonly<FeatureCollection>,
|
||||
) {
|
||||
this.nsiFile = nsiFile
|
||||
this.nsiWdFile = nsiWdFile
|
||||
this.loco = new LocationConflation(features)
|
||||
}
|
||||
|
||||
private static inited: NameSuggestionIndex = undefined
|
||||
|
|
@ -89,12 +93,12 @@ export default class NameSuggestionIndex {
|
|||
if (NameSuggestionIndex.inited) {
|
||||
return NameSuggestionIndex.inited
|
||||
}
|
||||
const [nsi, nsiWd] = await Promise.all(
|
||||
["assets/data/nsi/nsi.json", "assets/data/nsi/wikidata.min.json"].map((url) =>
|
||||
Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30)
|
||||
)
|
||||
const [nsi, nsiWd, features] = await Promise.all(
|
||||
["./assets/data/nsi/nsi.min.json", "./assets/data/nsi/wikidata.min.json", "./assets/data/nsi/featureCollection.min.json"].map((url) =>
|
||||
Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30),
|
||||
),
|
||||
)
|
||||
NameSuggestionIndex.inited = new NameSuggestionIndex(<any>nsi, <any>nsiWd["wikidata"])
|
||||
NameSuggestionIndex.inited = new NameSuggestionIndex(<any>nsi, <any>nsiWd["wikidata"], <any>features)
|
||||
return NameSuggestionIndex.inited
|
||||
}
|
||||
|
||||
|
|
@ -125,13 +129,13 @@ export default class NameSuggestionIndex {
|
|||
try {
|
||||
return Utils.downloadJsonCached<Record<string, number>>(
|
||||
`./assets/data/nsi/stats/${type}.${c.toUpperCase()}.json`,
|
||||
24 * 60 * 60 * 1000
|
||||
24 * 60 * 60 * 1000,
|
||||
)
|
||||
} catch (e) {
|
||||
console.error("Could not fetch " + type + " statistics due to", e)
|
||||
return undefined
|
||||
}
|
||||
})
|
||||
}),
|
||||
)
|
||||
stats = Utils.NoNull(stats)
|
||||
if (stats.length === 1) {
|
||||
|
|
@ -172,17 +176,17 @@ export default class NameSuggestionIndex {
|
|||
public async generateMappings(
|
||||
type: string,
|
||||
tags: Record<string, string>,
|
||||
country: string[],
|
||||
country?: string[],
|
||||
location?: [number, number],
|
||||
options?: {
|
||||
/**
|
||||
* If set, sort by frequency instead of alphabetically
|
||||
*/
|
||||
sortByFrequency: boolean
|
||||
}
|
||||
},
|
||||
): Promise<Mapping[]> {
|
||||
const mappings: (Mapping & { frequency: number })[] = []
|
||||
const frequencies = await NameSuggestionIndex.fetchFrequenciesFor(type, country)
|
||||
const frequencies = country !== undefined ? await NameSuggestionIndex.fetchFrequenciesFor(type, country) : {}
|
||||
for (const key in tags) {
|
||||
if (key.startsWith("_")) {
|
||||
continue
|
||||
|
|
@ -193,7 +197,7 @@ export default class NameSuggestionIndex {
|
|||
key,
|
||||
value,
|
||||
country.join(";"),
|
||||
location
|
||||
location,
|
||||
)
|
||||
if (!actualBrands) {
|
||||
continue
|
||||
|
|
@ -201,8 +205,7 @@ export default class NameSuggestionIndex {
|
|||
for (const nsiItem of actualBrands) {
|
||||
const tags = nsiItem.tags
|
||||
const frequency = frequencies[nsiItem.displayName]
|
||||
const logos = this.nsiWdFile[nsiItem.tags[type + ":wikidata"]]?.logos
|
||||
const iconUrl = logos?.facebook ?? logos?.wikidata
|
||||
const iconUrl = this.getIconExternalUrl(nsiItem, type)
|
||||
const hasIcon = iconUrl !== undefined
|
||||
let icon = undefined
|
||||
if (hasIcon) {
|
||||
|
|
@ -239,7 +242,7 @@ export default class NameSuggestionIndex {
|
|||
}
|
||||
|
||||
public supportedTags(
|
||||
type: "operator" | "brand" | "flag" | "transit" | string
|
||||
type: "operator" | "brand" | "flag" | "transit" | string,
|
||||
): Record<string, string[]> {
|
||||
const tags: Record<string, string[]> = {}
|
||||
const keys = Object.keys(this.nsiFile.nsi)
|
||||
|
|
@ -262,7 +265,7 @@ export default class NameSuggestionIndex {
|
|||
* Returns a list of all brands/operators
|
||||
* @param type
|
||||
*/
|
||||
public allPossible(type: "brand" | "operator"): NSIItem[] {
|
||||
public allPossible(type: string): NSIItem[] {
|
||||
const options: NSIItem[] = []
|
||||
const tags = this.supportedTags(type)
|
||||
for (const osmKey in tags) {
|
||||
|
|
@ -284,10 +287,10 @@ export default class NameSuggestionIndex {
|
|||
type: string,
|
||||
tags: { key: string; value: string }[],
|
||||
country: string = undefined,
|
||||
location: [number, number] = undefined
|
||||
location: [number, number] = undefined,
|
||||
): NSIItem[] {
|
||||
return tags.flatMap((tag) =>
|
||||
this.getSuggestionsForKV(type, tag.key, tag.value, country, location)
|
||||
this.getSuggestionsForKV(type, tag.key, tag.value, country, location),
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -310,7 +313,7 @@ export default class NameSuggestionIndex {
|
|||
key: string,
|
||||
value: string,
|
||||
country: string = undefined,
|
||||
location: [number, number] = undefined
|
||||
location: [number, number] = undefined,
|
||||
): NSIItem[] {
|
||||
const path = `${type}s/${key}/${value}`
|
||||
const entry = this.nsiFile.nsi[path]
|
||||
|
|
@ -351,7 +354,7 @@ export default class NameSuggestionIndex {
|
|||
const key = i.locationSet.include?.join(";") + "-" + i.locationSet.exclude?.join(";")
|
||||
const fromCache = NameSuggestionIndex.resolvedSets[key]
|
||||
const resolvedSet =
|
||||
fromCache ?? NameSuggestionIndex.loco.resolveLocationSet(i.locationSet)
|
||||
fromCache ?? this.loco.resolveLocationSet(i.locationSet)
|
||||
if (!fromCache) {
|
||||
NameSuggestionIndex.resolvedSets[key] = resolvedSet
|
||||
}
|
||||
|
|
@ -374,9 +377,52 @@ export default class NameSuggestionIndex {
|
|||
center: [number, number],
|
||||
options: {
|
||||
sortByFrequency: boolean
|
||||
}
|
||||
},
|
||||
): Promise<Mapping[]> {
|
||||
const nsi = await NameSuggestionIndex.getNsiIndex()
|
||||
return nsi.generateMappings(key, tags, country, center, options)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Where can we find the URL on the world wide web?
|
||||
* Probably facebook! Don't use in the website, might expose people
|
||||
* @param nsiItem
|
||||
* @param type
|
||||
*/
|
||||
private getIconExternalUrl(nsiItem: NSIItem, type: string): string {
|
||||
const logos = this.nsiWdFile[nsiItem.tags[type + ":wikidata"]]?.logos
|
||||
return logos?.facebook ?? logos?.wikidata
|
||||
}
|
||||
|
||||
public getIconUrl(nsiItem: NSIItem, type: string) {
|
||||
let icon = "./assets/data/nsi/logos/" + nsiItem.id
|
||||
if (this.isSvg(nsiItem, type)) {
|
||||
icon = icon + ".svg"
|
||||
}
|
||||
return icon
|
||||
}
|
||||
private static readonly brandPrefix = ["name", "alt_name", "operator","brand"] as const
|
||||
|
||||
/**
|
||||
* An NSI-item might have tags such as `name=X`, `alt_name=brand X`, `brand=X`, `brand:wikidata`, `shop=Y`, `service:abc=yes`
|
||||
* Many of those tags are all added, but having only one of them is a good indication that it should match this item.
|
||||
*
|
||||
* This method is a heuristic which attempts to move all the brand-related tags into an `or` but still requiring the `shop` and other tags
|
||||
*
|
||||
* (More of an extension method on NSIItem)
|
||||
*/
|
||||
static asFilterTags(item: NSIItem): string | { and: TagConfigJson[] } | { or: TagConfigJson[] } {
|
||||
let brandDetection: string[] = []
|
||||
let required: string[] = []
|
||||
const tags: Record<string, string> = item.tags
|
||||
for (const k in tags) {
|
||||
if (NameSuggestionIndex.brandPrefix.some(br => k === br || k.startsWith(br + ":"))) {
|
||||
brandDetection.push(k + "=" + tags[k])
|
||||
} else {
|
||||
required.push(k + "=" + tags[k])
|
||||
}
|
||||
}
|
||||
return <TagConfigJson>TagUtils.optimzeJson({ and: [...required, { or: brandDetection }] })
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { Utils } from "../../Utils"
|
||||
import type { FeatureCollection } from "geojson"
|
||||
import ScriptUtils from "../../../scripts/ScriptUtils"
|
||||
|
||||
export interface TagInfoStats {
|
||||
/**
|
||||
|
|
@ -39,12 +40,12 @@ export default class TagInfo {
|
|||
let url: string
|
||||
if (value) {
|
||||
url = `${this._backend}api/4/tag/stats?key=${encodeURIComponent(
|
||||
key
|
||||
key,
|
||||
)}&value=${encodeURIComponent(value)}`
|
||||
} else {
|
||||
url = `${this._backend}api/4/key/stats?key=${encodeURIComponent(key)}`
|
||||
}
|
||||
return await Utils.downloadJsonCached<TagInfoStats>(url, 1000 * 60 * 60)
|
||||
return await Utils.downloadJsonCached<TagInfoStats>(url, 1000 * 60 * 60 * 24)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -69,10 +70,10 @@ export default class TagInfo {
|
|||
}
|
||||
const countriesFC: FeatureCollection = await Utils.downloadJsonCached<FeatureCollection>(
|
||||
"https://download.geofabrik.de/index-v1-nogeom.json",
|
||||
24 * 1000 * 60 * 60
|
||||
24 * 1000 * 60 * 60,
|
||||
)
|
||||
TagInfo._geofabrikCountries = countriesFC.features.map(
|
||||
(f) => <GeofabrikCountryProperties>f.properties
|
||||
(f) => <GeofabrikCountryProperties>f.properties,
|
||||
)
|
||||
return TagInfo._geofabrikCountries
|
||||
}
|
||||
|
|
@ -98,7 +99,7 @@ export default class TagInfo {
|
|||
private static async getDistributionsFor(
|
||||
countryCode: string,
|
||||
key: string,
|
||||
value?: string
|
||||
value?: string,
|
||||
): Promise<TagInfoStats> {
|
||||
if (!countryCode) {
|
||||
return undefined
|
||||
|
|
@ -110,30 +111,43 @@ export default class TagInfo {
|
|||
try {
|
||||
return await ti.getStats(key, value)
|
||||
} catch (e) {
|
||||
console.warn("Could not fetch info for", countryCode, key, value, "due to", e)
|
||||
console.warn("Could not fetch info from taginfo for", countryCode, key, value, "due to", e, "Taginfo country specific instance is ", ti._backend)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
private static readonly blacklist = ["VI", "GF", "PR"]
|
||||
|
||||
public static async getGlobalDistributionsFor(
|
||||
/**
|
||||
* Get a taginfo object for every supportedCountry. This statistic is handled by 'f' and written into the passed in object
|
||||
* @param writeInto
|
||||
* @param f
|
||||
* @param key
|
||||
* @param value
|
||||
*/
|
||||
public static async getGlobalDistributionsFor<T>(
|
||||
writeInto: Record<string, T>,
|
||||
f: ((stats: TagInfoStats) => T),
|
||||
key: string,
|
||||
value?: string
|
||||
): Promise<Record<string, TagInfoStats>> {
|
||||
value?: string,
|
||||
): Promise<number> {
|
||||
const countriesAll = await this.geofabrikCountries()
|
||||
const countries = countriesAll
|
||||
.map((c) => c["iso3166-1:alpha2"]?.[0])
|
||||
.filter((c) => !!c && TagInfo.blacklist.indexOf(c) < 0)
|
||||
const perCountry: Record<string, TagInfoStats> = {}
|
||||
const results = await Promise.all(
|
||||
countries.map((country) => TagInfo.getDistributionsFor(country, key, value))
|
||||
)
|
||||
for (let i = 0; i < countries.length; i++) {
|
||||
const countryCode = countries[i]
|
||||
if (results[i]) {
|
||||
perCountry[countryCode] = results[i]
|
||||
|
||||
let downloaded = 0
|
||||
for (const country of countries) {
|
||||
if(writeInto[country] !== undefined){
|
||||
continue
|
||||
}
|
||||
const r = await TagInfo.getDistributionsFor(country, key, value)
|
||||
if(r === undefined){
|
||||
continue
|
||||
}
|
||||
downloaded ++
|
||||
writeInto[country] = f(r)
|
||||
}
|
||||
return perCountry
|
||||
return downloaded
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue