loviuz/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Refactoring: split NameSuggestionIndex into parts

Browse source
This commit is contained in:
Pieter Vander Vennet 2025-04-27 22:54:51 +02:00
parent 4899065fc4
commit cf7e005fd1
5 changed files with 103 additions and 139 deletions
Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -120,7 +120,7 @@
"velopark:download": "vite-node scripts/velopark/veloParkToGeojson.ts ",
"### 1": "CODE AND DATA MAINTAINENCE",
"download:nsi-logos": "mkdir -p ./public/assets/data/nsi && vite-node scripts/nsiLogos.ts -- download prune addExtensions patch || npm run download:nsi-logos # This script crashes often without the possibility to correct - hence the auto retry with OR",
"download:nsi": "npm i name-suggestion-index && vite-node scripts/generateStats.ts && npm run download:nsi-logos && vite-node scripts/nsiLogos.ts -- all",
"download:nsi": "npm i name-suggestion-index && vite-node scripts/generateNsiStats.ts && npm run download:nsi-logos && vite-node scripts/nsiLogos.ts -- all",
"download:editor-layer-index": "mkdir -p ./public/assets/data/ && vite-node scripts/downloadEli.ts",
"download:stats": "vite-node scripts/GenerateSeries.ts",
"download:community-index": "vite-node scripts/downloadCommunityIndex.ts ",

8
scripts/generateStats.ts → scripts/generateNsiStats.ts
View file

@ -12,7 +12,7 @@ import TagInfo from "../src/Logic/Web/TagInfo"
import { TagsFilter } from "../src/Logic/Tags/TagsFilter"
class GenerateStats extends Script {
class GenerateNsiStats extends Script {
async createOptimizationFile(includeTags = true) {
ScriptUtils.fixUtils()
const layers = <LayerConfigJson[]>known_layers["layers"]
@ -139,7 +139,7 @@ class GenerateStats extends Script {
path
)
}
const nsi = await NameSuggestionIndex.getNsiIndex("./assets/data/nsi/")
const nsi = await NameSuggestionIndex.singleton()
const allBrandNames: string[] = Utils.Dedup(
nsi.allPossible(<any>type).map((item) => item.tags[type])
)
@ -182,7 +182,7 @@ class GenerateStats extends Script {
)
}
async main(_: string[]) {
async main() {
const target = "./public/assets/data/nsi/"
const basepath = target + "stats/"
{
@ -202,4 +202,4 @@ class GenerateStats extends Script {
}
}
new GenerateStats().run()
new GenerateNsiStats().run()

18
scripts/nsiLogos.ts
View file

@ -1,5 +1,5 @@
import Script from "./Script"
import NameSuggestionIndex, { NSIItem } from "../src/Logic/Web/NameSuggestionIndex"
import NameSuggestionIndex, { NamgeSuggestionWikidata, NSIItem } from "../src/Logic/Web/NameSuggestionIndex"
import * as nsiWD from "../node_modules/name-suggestion-index/dist/wikidata.min.json"
import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "fs"
import ScriptUtils from "./ScriptUtils"
@ -42,8 +42,8 @@ class NsiLogos extends Script {
let path = basePath + nsiItem.id
const logos = nsiWD["wikidata"][nsiItem?.tags?.[type + ":wikidata"]]?.logos
const nsi = await NameSuggestionIndex.getNsiIndex("./assets/data/nsi/")
if (nsi.isSvg(nsiItem, type)) {
const nsiWd = new NamgeSuggestionWikidata(nsiWD)
if (nsiWd.isSvg(nsiItem, type)) {
path = path + ".svg"
}
@ -99,13 +99,17 @@ class NsiLogos extends Script {
return false
}
private async getAllPossibleNsiItems(type: string): Promise<NSIItem[]> {
const nsi = await NameSuggestionIndex.singleton()
return nsi.allPossible(type)
}
/**
* Returns
* @param type
*/
async downloadFor(type: string): Promise<{ downloadCount: number; errored: number }> {
const nsi = await NameSuggestionIndex.getNsiIndex("./assets/data/nsi/")
const items = nsi.allPossible(type)
const items = await this.getAllPossibleNsiItems(type)
const basePath = "./public/assets/data/nsi/logos/"
let downloadCount = 0
let errored = 0
@ -158,8 +162,8 @@ class NsiLogos extends Script {
}
private async generateRendering(type: string) {
const nsi = await NameSuggestionIndex.getNsiIndex("./assets/data/nsi/")
const items = nsi.allPossible(type)
const nsi = await NameSuggestionIndex.singleton()
const items = await this.getAllPossibleNsiItems(type)
const filterOptions: FilterConfigOptionJson[] = items.map((item) => {
return {
question: item.displayName,

191
src/Logic/Web/NameSuggestionIndex.ts
View file

@ -55,14 +55,18 @@ export interface NSIItem {
ext?: string
}
export class NameSuggestionIndexLight {
public static readonly supportedTypes = ["brand", "flag", "operator", "transit"] as const
const supportedTypes = ["brand", "flag", "operator", "transit"] as const
export type NsiSupportedType = typeof supportedTypes[number]
export default class NameSuggestionIndex {
public static readonly supportedTypes: ReadonlyArray<NsiSupportedType> = ["brand", "flag", "operator", "transit"] as const
protected readonly _serverLocation: string
protected readonly nsiFile: Readonly<NSIFile>
private readonly loco: LocationConflation // Some additional boundaries
private static initedLight: NameSuggestionIndexLight = undefined
private static inited: NameSuggestionIndex = undefined
private _supportedTypes: NsiSupportedType[]
protected constructor(
private constructor(
serverLocation: string,
nsiFile: Readonly<NSIFile>,
features: Readonly<FeatureCollection>
@ -70,7 +74,7 @@ export class NameSuggestionIndexLight {
this._serverLocation = serverLocation
this.nsiFile = nsiFile
this.loco = new LocationConflation(features)
NameSuggestionIndexLight.initedLight = this
NameSuggestionIndex.inited = this
}
/**
@ -258,9 +262,9 @@ export class NameSuggestionIndexLight {
})
}
public static async singleton(): Promise<NameSuggestionIndexLight> {
if (NameSuggestionIndexLight.initedLight) {
return NameSuggestionIndexLight.initedLight
public static async singleton(): Promise<NameSuggestionIndex> {
if (NameSuggestionIndex.inited) {
return NameSuggestionIndex.inited
}
const endpoint = Constants.nsiLogosEndpoint ?? "./assets/data/nsi/"
const [nsi, features] = await Promise.all(
@ -269,70 +273,14 @@ export class NameSuggestionIndexLight {
endpoint + "featureCollection.min.json"
].map((url) => Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30))
)
return new NameSuggestionIndexLight(
return new NameSuggestionIndex(
endpoint,
<any>nsi,
<any>features
)
}
}
export default class NameSuggestionIndex extends NameSuggestionIndexLight {
private readonly nsiWdFile: Readonly<
Record<
string,
{
logos: { wikidata?: string; facebook?: string }
}
>
>
private _supportedTypes: string[]
private static inited: NameSuggestionIndex = undefined
private constructor(
serverLocation: string,
nsiFile: Readonly<NSIFile>,
nsiWdFile: Readonly<
Record<
string,
{
logos: { wikidata?: string; facebook?: string }
}
>
>,
features: Readonly<FeatureCollection>
) {
super(serverLocation, nsiFile, features)
this.nsiWdFile = nsiWdFile
NameSuggestionIndex.inited = this
}
public static async getNsiIndex(endPoint ?: string): Promise<NameSuggestionIndex> {
if (NameSuggestionIndex.inited) {
return NameSuggestionIndex.inited
}
endPoint ??= Constants.nsiLogosEndpoint ?? "./assets/data/nsi/"
const [nsi, nsiWd, features] = await Promise.all(
[
endPoint + "nsi.min.json",
endPoint + "wikidata.min.json",
endPoint + "featureCollection.min.json"
].map((url) => Utils.downloadJsonCached(url, 1000 * 60 * 60 * 24 * 30))
)
return new NameSuggestionIndex(
Constants.nsiLogosEndpoint,
<any>nsi,
<any>nsiWd["wikidata"],
<any>features
)
}
public supportedTypes(): string[] {
public supportedTypes(): NsiSupportedType[] {
if (this._supportedTypes) {
return this._supportedTypes
}
@ -342,35 +290,13 @@ export default class NameSuggestionIndex extends NameSuggestionIndexLight {
if (s.endsWith("s")) {
s = s.substring(0, s.length - 1)
}
return s
return <NsiSupportedType>s
})
return this._supportedTypes
}
public isSvg(nsiItem: NSIItem, type: string): boolean | undefined {
if (this.nsiWdFile === undefined) {
throw (
"nsiWdi file is not loaded, cannot determine if " + nsiItem.id + " has an SVG image"
)
}
const logos = this.nsiWdFile[nsiItem?.tags?.[type + ":wikidata"]]?.logos
if (!logos) {
return undefined
}
if (logos.facebook) {
return false
}
const url: string = logos.wikidata
if (url.toLowerCase().endsWith(".svg")) {
return true
}
return false
}
public supportedTags(
type: "operator" | "brand" | "flag" | "transit" | string
type: NsiSupportedType
): Record<string, string[]> {
const tags: Record<string, string[]> = {}
const keys = Object.keys(this.nsiFile.nsi)
@ -389,11 +315,17 @@ export default class NameSuggestionIndex extends NameSuggestionIndexLight {
return tags
}
public static async getNsiIndex(endPoint ?: string): Promise<NameSuggestionIndex> {
}
/**
* Returns a list of all brands/operators
* @param type
* @param type One of the supported types, e.g. "brand" or "operator"
*/
public allPossible(type: string): NSIItem[] {
public allPossible(type: NsiSupportedType): NSIItem[] {
const options: NSIItem[] = []
const tags = this.supportedTags(type)
for (const osmKey in tags) {
@ -412,8 +344,8 @@ export default class NameSuggestionIndex extends NameSuggestionIndexLight {
/**
*
* @param country: a string containing one or more country codes, separated by ";"
* @param location: center point of the feature, should be [lon, lat]
* @param country a string containing one or more country codes, separated by ";"
* @param location center point of the feature, should be [lon, lat]
*/
public getSuggestionsFor(
type: string,
@ -436,21 +368,10 @@ export default class NameSuggestionIndex extends NameSuggestionIndexLight {
sortByFrequency: boolean
}
): Promise<Mapping[]> {
const nsi = await NameSuggestionIndexLight.singleton()
const nsi = await NameSuggestionIndex.singleton()
return nsi.generateMappings(key, tags, country, center, options)
}
/**
* Where can we find the URL on the world wide web?
* Probably facebook! Don't use in the website, might expose people
* @param nsiItem
* @param type
*/
private getIconExternalUrl(nsiItem: NSIItem, type: string): string {
const logos = this.nsiWdFile[nsiItem.tags[type + ":wikidata"]]?.logos
return logos?.facebook ?? logos?.wikidata
}
private static readonly brandPrefix = ["name", "alt_name", "operator", "brand"] as const
@ -478,3 +399,61 @@ export default class NameSuggestionIndex extends NameSuggestionIndexLight {
return <TagConfigJson>TagUtils.optimzeJson({ and: [...required, { or: brandDetection }] })
}
}
export class NamgeSuggestionWikidata {
private readonly nsiWdFile: Readonly<
Record<
string,
{
logos: { wikidata?: string; facebook?: string }
}
>
>
constructor(
nsiWdFile: Readonly<
Record<
string,
{
logos: { wikidata?: string; facebook?: string }
}
>
>
) {
this.nsiWdFile = nsiWdFile
}
public isSvg(nsiItem: NSIItem, type: string): boolean | undefined {
if (this.nsiWdFile === undefined) {
throw (
"nsiWdi file is not loaded, cannot determine if " + nsiItem.id + " has an SVG image"
)
}
const logos = this.nsiWdFile[nsiItem?.tags?.[type + ":wikidata"]]?.logos
if (!logos) {
return undefined
}
if (logos.facebook) {
return false
}
const url: string = logos.wikidata
if (url.toLowerCase().endsWith(".svg")) {
return true
}
return false
}
/**
* Where can we find the URL on the world wide web?
* Probably facebook! Don't use in the website, might expose people
* @param nsiItem
* @param type
*/
private getIconExternalUrl(nsiItem: NSIItem, type: string): string {
const logos = this.nsiWdFile[nsiItem.tags[type + ":wikidata"]]?.logos
return logos?.facebook ?? logos?.wikidata
}
}

23
src/Models/ThemeConfig/Conversion/MiscTagRenderingChecks.ts
View file

@ -1,16 +1,11 @@
import { DesugaringStep } from "./Conversion"
import { TagRenderingConfigJson } from "../Json/TagRenderingConfigJson"
import { LayerConfigJson } from "../Json/LayerConfigJson"
import {
MappingConfigJson,
QuestionableTagRenderingConfigJson,
} from "../Json/QuestionableTagRenderingConfigJson"
import { MappingConfigJson, QuestionableTagRenderingConfigJson } from "../Json/QuestionableTagRenderingConfigJson"
import { ConversionContext } from "./ConversionContext"
import { Translation } from "../../../UI/i18n/Translation"
import { TagUtils } from "../../../Logic/Tags/TagUtils"
import Validators from "../../../UI/InputElement/Validators"
import { CheckTranslation } from "./Validation"
import NameSuggestionIndex from "../../../Logic/Web/NameSuggestionIndex"
export class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
private readonly _layerConfig: LayerConfigJson
@ -195,21 +190,7 @@ export class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJso
)
}
}
if (
this._layerConfig?.source?.["osmTags"] &&
NameSuggestionIndex.supportedTypes.indexOf(<any>json.freeform.key) >= 0
) {
const tags = TagUtils.TagD(this._layerConfig?.source?.["osmTags"])?.usedTags()
/* const suggestions = nameSuggestionIndexBundled.getSuggestionsFor(json.freeform.key, tags)
if (suggestions === undefined) {
context
.enters("freeform", "type")
.err(
"No entry found in the 'Name Suggestion Index'. None of the 'osmSource'-tags match an entry in the NSI.\n\tOsmSource-tags are " +
tags.map((t) => new Tag(t.key, t.value).asHumanString()).join(" ; ")
)
}*/
} else if (json.freeform["type"] === "nsi") {
if (json.freeform["type"] === "nsi") {
context
.enters("freeform", "type")
.warn(