diff --git a/scripts/generateStats.ts b/scripts/generateStats.ts index 2931cbaf7..a387bd91e 100644 --- a/scripts/generateStats.ts +++ b/scripts/generateStats.ts @@ -2,88 +2,138 @@ import known_layers from "../src/assets/generated/known_layers.json" import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson" import { TagUtils } from "../src/Logic/Tags/TagUtils" import { Utils } from "../src/Utils" -import { writeFileSync } from "fs" +import { existsSync, readFileSync, writeFileSync } from "fs" import ScriptUtils from "./ScriptUtils" import TagRenderingConfig from "../src/Models/ThemeConfig/TagRenderingConfig" import { And } from "../src/Logic/Tags/And" +import Script from "./Script" +import NameSuggestionIndex, { NSIItem } from "../src/Logic/Web/NameSuggestionIndex" +import TagInfo, { TagInfoStats } from "../src/Logic/Web/TagInfo" -/* Downloads stats on osmSource-tags and keys from tagInfo */ - -async function main(includeTags = true) { - ScriptUtils.fixUtils() - const layers = known_layers.layers - - const keysAndTags = new Map>() - - for (const layer of layers) { - if (layer.source["geoJson"] !== undefined && !layer.source["isOsmCache"]) { - continue - } - if (layer.source == null || typeof layer.source === "string") { - continue +class Utilities { + static mapValues(record: Record, f: ((t: T) => TOut)): Record { + const newR = >{} + for (const x in record) { + newR[x] = f(record[x]) } + return newR + } +} +class GenerateStats extends Script { - const sourcesList = [TagUtils.Tag(layer.source["osmTags"])] - if (layer?.title) { - sourcesList.push(...new TagRenderingConfig(layer.title).usedTags()) - } + async createOptimizationFile(includeTags = true) { + ScriptUtils.fixUtils() + const layers = known_layers.layers - const sources = new And(sourcesList) - const allKeys = sources.usedKeys() - for (const key of allKeys) { - if (!keysAndTags.has(key)) { - keysAndTags.set(key, new Set()) + const keysAndTags = new Map>() + + for (const layer of layers) { + if (layer.source["geoJson"] !== undefined && !layer.source["isOsmCache"]) { + continue + } + if (layer.source == null || typeof layer.source === "string") { + continue + } + + const sourcesList = [TagUtils.Tag(layer.source["osmTags"])] + if (layer?.title) { + sourcesList.push(...new TagRenderingConfig(layer.title).usedTags()) + } + + const sources = new And(sourcesList) + const allKeys = sources.usedKeys() + for (const key of allKeys) { + if (!keysAndTags.has(key)) { + keysAndTags.set(key, new Set()) + } + } + const allTags = includeTags ? sources.usedTags() : [] + for (const tag of allTags) { + if (!keysAndTags.has(tag.key)) { + keysAndTags.set(tag.key, new Set()) + } + keysAndTags.get(tag.key).add(tag.value) } } - const allTags = includeTags ? sources.usedTags() : [] - for (const tag of allTags) { - if (!keysAndTags.has(tag.key)) { - keysAndTags.set(tag.key, new Set()) - } - keysAndTags.get(tag.key).add(tag.value) - } + + const keyTotal = new Map() + const tagTotal = new Map>() + await Promise.all( + Array.from(keysAndTags.keys()).map(async (key) => { + const values = keysAndTags.get(key) + const data = await Utils.downloadJson( + `https://taginfo.openstreetmap.org/api/4/key/stats?key=${key}` + ) + const count = data.data.find((item) => item.type === "all").count + keyTotal.set(key, count) + console.log(key, "-->", count) + + if (values.size > 0) { + tagTotal.set(key, new Map()) + await Promise.all( + Array.from(values).map(async (value) => { + const tagData = await Utils.downloadJson( + `https://taginfo.openstreetmap.org/api/4/tag/stats?key=${key}&value=${value}` + ) + const count = tagData.data.find((item) => item.type === "all").count + tagTotal.get(key).set(value, count) + console.log(key + "=" + value, "-->", count) + }) + ) + } + }) + ) + writeFileSync( + "./src/assets/key_totals.json", + JSON.stringify( + { + "#": "Generated with generateStats.ts", + date: new Date().toISOString(), + keys: Utils.MapToObj(keyTotal, (t) => t), + tags: Utils.MapToObj(tagTotal, (v) => Utils.MapToObj(v, (t) => t)) + }, + null, + " " + ) + ) } - const keyTotal = new Map() - const tagTotal = new Map>() - await Promise.all( - Array.from(keysAndTags.keys()).map(async (key) => { - const values = keysAndTags.get(key) - const data = await Utils.downloadJson( - `https://taginfo.openstreetmap.org/api/4/key/stats?key=${key}` - ) - const count = data.data.find((item) => item.type === "all").count - keyTotal.set(key, count) - console.log(key, "-->", count) - - if (values.size > 0) { - tagTotal.set(key, new Map()) - await Promise.all( - Array.from(values).map(async (value) => { - const tagData = await Utils.downloadJson( - `https://taginfo.openstreetmap.org/api/4/tag/stats?key=${key}&value=${value}` - ) - const count = tagData.data.find((item) => item.type === "all").count - tagTotal.get(key).set(value, count) - console.log(key + "=" + value, "-->", count) - }) - ) + async createNameSuggestionIndexFile() { + const type = "brand" + let allBrands = >>{} + const path = "./src/assets/generated/nsi_stats/" + type + ".json" + if (existsSync(path)) { + allBrands = JSON.parse(readFileSync(path, "utf8")) + console.log("Loaded",Object.keys(allBrands).length," previously loaded brands") + } + let lastWrite = new Date() + const allBrandNames: string[] = NameSuggestionIndex.allPossible(type) + for (const brand of allBrandNames) { + if(allBrands[brand] !== undefined){ + console.log("Skipping", brand,", already loaded") + continue } - }) - ) - writeFileSync( - "./src/assets/key_totals.json", - JSON.stringify( - { - "#": "Generated with generateStats.ts", - date: new Date().toISOString(), - keys: Utils.MapToObj(keyTotal, (t) => t), - tags: Utils.MapToObj(tagTotal, (v) => Utils.MapToObj(v, (t) => t)), - }, - null, - " " - ) - ) + const distribution: Record = Utilities.mapValues(await TagInfo.getGlobalDistributionsFor(type, brand), s => s.data.find(t => t.type === "all").count) + allBrands[brand] = distribution + if ((new Date().getTime() - lastWrite.getTime()) / 1000 >= 5) { + writeFileSync(path, JSON.stringify(allBrands), "utf8") + console.log("Checkpointed", path) + } + } + writeFileSync(path, JSON.stringify(allBrands), "utf8") + } + + constructor() { + super("Downloads stats on osmSource-tags and keys from tagInfo. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index ") + } + + async main(_: string[]) { + // this.createOptimizationFile() + await this.createNameSuggestionIndexFile() + + } + } -main().then(() => console.log("All done")) + +new GenerateStats().run() diff --git a/src/Logic/Web/NameSuggestionIndex.ts b/src/Logic/Web/NameSuggestionIndex.ts new file mode 100644 index 000000000..2ccbee5ba --- /dev/null +++ b/src/Logic/Web/NameSuggestionIndex.ts @@ -0,0 +1,153 @@ +import * as nsi from "../../../node_modules/name-suggestion-index/dist/nsi.json" +import * as nsiFeatures from "../../../node_modules/name-suggestion-index/dist/featureCollection.json" +import { LocationConflation } from "@rapideditor/location-conflation" +import type { Feature, FeatureCollection, MultiPolygon } from "geojson" +import * as turf from "@turf/turf" +import { Utils } from "../../Utils" +import TagInfo from "./TagInfo" + +/** + * Main name suggestion index file + */ +interface NSIFile { + _meta: { + version: string + generated: string + url: string + hash: string + } + nsi: { + [path: string]: NSIEntry + } +} + +/** + * A collection of brands/operators/flagpoles/... with common properties + * See https://github.com/osmlab/name-suggestion-index/wiki/Category-Files for an introduction and + * https://github.com/osmlab/name-suggestion-index/blob/main/schema/categories.json for a full breakdown + */ +interface NSIEntry { + properties: { + path: string + skipCollection?: boolean + preserveTags?: string[] + exclude: unknown + } + items: NSIItem[] +} + +/** + * Represents a single brand/operator/flagpole/... + */ +export interface NSIItem { + displayName: string + id: string + locationSet: { + include: string[], + exclude: string[] + } + tags: { + [key: string]: string + } + fromTemplate?: boolean +} + +export default class NameSuggestionIndex { + + private static readonly nsiFile: Readonly = nsi + private static loco = new LocationConflation(nsiFeatures) // Some additional boundaries + + private static _supportedTypes: string[] + + public static supportedTypes(): string[] { + if (this._supportedTypes) { + return this._supportedTypes + } + const keys = Object.keys(NameSuggestionIndex.nsiFile.nsi) + const all = keys.map(k => NameSuggestionIndex.nsiFile.nsi[k].properties.path.split("/")[0]) + this._supportedTypes = Utils.Dedup(all) + return this._supportedTypes + } + + public static async buildTaginfoCountsPerCountry(type = "brand", key: string, value: string) { + const allData: { nsi: NSIItem, stats }[] = [] + const brands = NameSuggestionIndex.getSuggestionsFor(type, key, value) + for (const brand of brands) { + const brandValue = brand.tags[type] + const allStats = await TagInfo.getGlobalDistributionsFor(type, brandValue) + allData.push({ nsi: brand, stats: allStats }) + } + return allData + } + + public static supportedTags(type: "operator" | "brand" | "flag" | "transit" | string): Record { + const tags: Record = {} + const keys = Object.keys(NameSuggestionIndex.nsiFile.nsi) + for (const key of keys) { + + const nsiItem = NameSuggestionIndex.nsiFile.nsi[key] + const path = nsiItem.properties.path + const [osmType, osmkey, osmvalue] = path.split("/") + if (type !== osmType && (type + "s" !== osmType)) { + continue + } + if (!tags[osmkey]) { + tags[osmkey] = [] + } + tags[osmkey].push(osmvalue) + } + return tags + } + + public static allPossible(type: "brand" | "operator"): string[] { + const options: string[] = [] + const tags = NameSuggestionIndex.supportedTags(type) + for (const osmKey in tags) { + const values = tags[osmKey] + for (const osmValue of values) { + const suggestions = this.getSuggestionsFor(type, osmKey, osmValue) + for (const suggestion of suggestions) { + const value = suggestion.tags[type] + options.push(value) + } + } + } + return Utils.Dedup(options) + } + + /** + * + * @param path + * @param country + * @param location: center point of the feature, should be [lon, lat] + */ + public static getSuggestionsFor(type: string, key: string, value: string, country: string = undefined, location: [number, number] = undefined): NSIItem[] { + const path = `${type}s/${key}/${value}` + const entry = NameSuggestionIndex.nsiFile.nsi[path] + return entry?.items?.filter(i => { + if (i.locationSet.include.indexOf("001") >= 0) { + return true + } + + if (country === undefined || + // We prefer the countries provided by lonlat2country, they are more precise + // Country might contain multiple countries, separated by ';' + i.locationSet.include.some(c => country.indexOf(c) >= 0)) { + return true + } + + if (location === undefined) { + return true + } + const resolvedSet = NameSuggestionIndex.loco.resolveLocationSet(i.locationSet) + if (resolvedSet) { + // We actually have a location set, so we can check if the feature is in it, by determining if our point is inside the MultiPolygon using @turf/boolean-point-in-polygon + // This might occur for some extra boundaries, such as counties, ... + const setFeature: Feature = resolvedSet.feature + return turf.booleanPointInPolygon(location, setFeature.geometry) + } + + return false + }) + } +} diff --git a/src/Models/ThemeConfig/Conversion/Validation.ts b/src/Models/ThemeConfig/Conversion/Validation.ts index 2e798962e..98a5f6ff6 100644 --- a/src/Models/ThemeConfig/Conversion/Validation.ts +++ b/src/Models/ThemeConfig/Conversion/Validation.ts @@ -26,6 +26,7 @@ import { Translatable } from "../Json/Translatable" import { ConversionContext } from "./ConversionContext" import { AvailableRasterLayers } from "../../RasterLayers" import PointRenderingConfigJson from "../Json/PointRenderingConfigJson" +import NameSuggestionIndex from "../../../Logic/Web/NameSuggestionIndex" class ValidateLanguageCompleteness extends DesugaringStep { private readonly _languages: string[] @@ -1032,6 +1033,14 @@ class MiscTagRenderingChecks extends DesugaringStep { ) } } + if(json.freeform.type === "nsi"){ + const [key, value] = json.freeform.helperArgs[0].split("=") + const path = `${json.freeform.key}s/${key}/${value}` + const suggestions = NameSuggestionIndex.getSuggestionsFor(path) + if(suggestions === undefined){ + context.enters("freeform","type").err("No entry found in the 'Name Suggestion Index' for "+path) + } + } } if (json.render && json["question"] && json.freeform === undefined) { context.err( diff --git a/src/UI/InputElement/Helpers/NameSuggestionIndexInput.svelte b/src/UI/InputElement/Helpers/NameSuggestionIndexInput.svelte new file mode 100644 index 000000000..f2f3313ee --- /dev/null +++ b/src/UI/InputElement/Helpers/NameSuggestionIndexInput.svelte @@ -0,0 +1,145 @@ + + + +{#if items?.length >= 0} +
+
+ +
+
+ {#each filteredItems as item} +
{ + select(item) + }} + on:keydown={(e) => { + if (e.key === "Enter") { + select(item) + } + }} + > + {item.displayName} +
+ {/each} +
+
+ +{/if}