midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Move NSI-code in separate file

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-05-13 17:21:40 +02:00
parent 037887fea0
commit 07edee550c
4 changed files with 428 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

192
scripts/generateStats.ts
View file

@ -2,88 +2,138 @@ import known_layers from "../src/assets/generated/known_layers.json"
import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
import { TagUtils } from "../src/Logic/Tags/TagUtils"
import { Utils } from "../src/Utils"
import { writeFileSync } from "fs"
import { existsSync, readFileSync, writeFileSync } from "fs"
import ScriptUtils from "./ScriptUtils"
import TagRenderingConfig from "../src/Models/ThemeConfig/TagRenderingConfig"
import { And } from "../src/Logic/Tags/And"
import Script from "./Script"
import NameSuggestionIndex, { NSIItem } from "../src/Logic/Web/NameSuggestionIndex"
import TagInfo, { TagInfoStats } from "../src/Logic/Web/TagInfo"
/* Downloads stats on osmSource-tags and keys from tagInfo */
async function main(includeTags = true) {
ScriptUtils.fixUtils()
const layers = <LayerConfigJson[]>known_layers.layers
const keysAndTags = new Map<string, Set<string>>()
for (const layer of layers) {
if (layer.source["geoJson"] !== undefined && !layer.source["isOsmCache"]) {
continue
}
if (layer.source == null || typeof layer.source === "string") {
continue
class Utilities {
static mapValues<X extends string | number, T, TOut>(record: Record<X, T>, f: ((t: T) => TOut)): Record<X, TOut> {
const newR = <Record<X, TOut>>{}
for (const x in record) {
newR[x] = f(record[x])
}
return newR
}
}
class GenerateStats extends Script {
const sourcesList = [TagUtils.Tag(layer.source["osmTags"])]
if (layer?.title) {
sourcesList.push(...new TagRenderingConfig(layer.title).usedTags())
}
async createOptimizationFile(includeTags = true) {
ScriptUtils.fixUtils()
const layers = <LayerConfigJson[]>known_layers.layers
const sources = new And(sourcesList)
const allKeys = sources.usedKeys()
for (const key of allKeys) {
if (!keysAndTags.has(key)) {
keysAndTags.set(key, new Set<string>())
const keysAndTags = new Map<string, Set<string>>()
for (const layer of layers) {
if (layer.source["geoJson"] !== undefined && !layer.source["isOsmCache"]) {
continue
}
if (layer.source == null || typeof layer.source === "string") {
continue
}
const sourcesList = [TagUtils.Tag(layer.source["osmTags"])]
if (layer?.title) {
sourcesList.push(...new TagRenderingConfig(layer.title).usedTags())
}
const sources = new And(sourcesList)
const allKeys = sources.usedKeys()
for (const key of allKeys) {
if (!keysAndTags.has(key)) {
keysAndTags.set(key, new Set<string>())
}
}
const allTags = includeTags ? sources.usedTags() : []
for (const tag of allTags) {
if (!keysAndTags.has(tag.key)) {
keysAndTags.set(tag.key, new Set<string>())
}
keysAndTags.get(tag.key).add(tag.value)
}
}
const allTags = includeTags ? sources.usedTags() : []
for (const tag of allTags) {
if (!keysAndTags.has(tag.key)) {
keysAndTags.set(tag.key, new Set<string>())
}
keysAndTags.get(tag.key).add(tag.value)
}
const keyTotal = new Map<string, number>()
const tagTotal = new Map<string, Map<string, number>>()
await Promise.all(
Array.from(keysAndTags.keys()).map(async (key) => {
const values = keysAndTags.get(key)
const data = await Utils.downloadJson(
`https://taginfo.openstreetmap.org/api/4/key/stats?key=${key}`
)
const count = data.data.find((item) => item.type === "all").count
keyTotal.set(key, count)
console.log(key, "-->", count)
if (values.size > 0) {
tagTotal.set(key, new Map<string, number>())
await Promise.all(
Array.from(values).map(async (value) => {
const tagData = await Utils.downloadJson(
`https://taginfo.openstreetmap.org/api/4/tag/stats?key=${key}&value=${value}`
)
const count = tagData.data.find((item) => item.type === "all").count
tagTotal.get(key).set(value, count)
console.log(key + "=" + value, "-->", count)
})
)
}
})
)
writeFileSync(
"./src/assets/key_totals.json",
JSON.stringify(
{
"#": "Generated with generateStats.ts",
date: new Date().toISOString(),
keys: Utils.MapToObj(keyTotal, (t) => t),
tags: Utils.MapToObj(tagTotal, (v) => Utils.MapToObj(v, (t) => t))
},
null,
" "
)
)
}
const keyTotal = new Map<string, number>()
const tagTotal = new Map<string, Map<string, number>>()
await Promise.all(
Array.from(keysAndTags.keys()).map(async (key) => {
const values = keysAndTags.get(key)
const data = await Utils.downloadJson(
`https://taginfo.openstreetmap.org/api/4/key/stats?key=${key}`
)
const count = data.data.find((item) => item.type === "all").count
keyTotal.set(key, count)
console.log(key, "-->", count)
if (values.size > 0) {
tagTotal.set(key, new Map<string, number>())
await Promise.all(
Array.from(values).map(async (value) => {
const tagData = await Utils.downloadJson(
`https://taginfo.openstreetmap.org/api/4/tag/stats?key=${key}&value=${value}`
)
const count = tagData.data.find((item) => item.type === "all").count
tagTotal.get(key).set(value, count)
console.log(key + "=" + value, "-->", count)
})
)
async createNameSuggestionIndexFile() {
const type = "brand"
let allBrands = <Record<string, Record<string, number>>>{}
const path = "./src/assets/generated/nsi_stats/" + type + ".json"
if (existsSync(path)) {
allBrands = JSON.parse(readFileSync(path, "utf8"))
console.log("Loaded",Object.keys(allBrands).length," previously loaded brands")
}
let lastWrite = new Date()
const allBrandNames: string[] = NameSuggestionIndex.allPossible(type)
for (const brand of allBrandNames) {
if(allBrands[brand] !== undefined){
console.log("Skipping", brand,", already loaded")
continue
}
})
)
writeFileSync(
"./src/assets/key_totals.json",
JSON.stringify(
{
"#": "Generated with generateStats.ts",
date: new Date().toISOString(),
keys: Utils.MapToObj(keyTotal, (t) => t),
tags: Utils.MapToObj(tagTotal, (v) => Utils.MapToObj(v, (t) => t)),
},
null,
" "
)
)
const distribution: Record<string, number> = Utilities.mapValues(await TagInfo.getGlobalDistributionsFor(type, brand), s => s.data.find(t => t.type === "all").count)
allBrands[brand] = distribution
if ((new Date().getTime() - lastWrite.getTime()) / 1000 >= 5) {
writeFileSync(path, JSON.stringify(allBrands), "utf8")
console.log("Checkpointed", path)
}
}
writeFileSync(path, JSON.stringify(allBrands), "utf8")
}
constructor() {
super("Downloads stats on osmSource-tags and keys from tagInfo. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index ")
}
async main(_: string[]) {
// this.createOptimizationFile()
await this.createNameSuggestionIndexFile()
}
}
main().then(() => console.log("All done"))
new GenerateStats().run()

153
src/Logic/Web/NameSuggestionIndex.ts Normal file
View file

@ -0,0 +1,153 @@
import * as nsi from "../../../node_modules/name-suggestion-index/dist/nsi.json"
import * as nsiFeatures from "../../../node_modules/name-suggestion-index/dist/featureCollection.json"
import { LocationConflation } from "@rapideditor/location-conflation"
import type { Feature, FeatureCollection, MultiPolygon } from "geojson"
import * as turf from "@turf/turf"
import { Utils } from "../../Utils"
import TagInfo from "./TagInfo"
/**
* Main name suggestion index file
*/
interface NSIFile {
_meta: {
version: string
generated: string
url: string
hash: string
}
nsi: {
[path: string]: NSIEntry
}
}
/**
* A collection of brands/operators/flagpoles/... with common properties
* See https://github.com/osmlab/name-suggestion-index/wiki/Category-Files for an introduction and
* https://github.com/osmlab/name-suggestion-index/blob/main/schema/categories.json for a full breakdown
*/
interface NSIEntry {
properties: {
path: string
skipCollection?: boolean
preserveTags?: string[]
exclude: unknown
}
items: NSIItem[]
}
/**
* Represents a single brand/operator/flagpole/...
*/
export interface NSIItem {
displayName: string
id: string
locationSet: {
include: string[],
exclude: string[]
}
tags: {
[key: string]: string
}
fromTemplate?: boolean
}
export default class NameSuggestionIndex {
private static readonly nsiFile: Readonly<NSIFile> = <any>nsi
private static loco = new LocationConflation(nsiFeatures) // Some additional boundaries
private static _supportedTypes: string[]
public static supportedTypes(): string[] {
if (this._supportedTypes) {
return this._supportedTypes
}
const keys = Object.keys(NameSuggestionIndex.nsiFile.nsi)
const all = keys.map(k => NameSuggestionIndex.nsiFile.nsi[k].properties.path.split("/")[0])
this._supportedTypes = Utils.Dedup(all)
return this._supportedTypes
}
public static async buildTaginfoCountsPerCountry(type = "brand", key: string, value: string) {
const allData: { nsi: NSIItem, stats }[] = []
const brands = NameSuggestionIndex.getSuggestionsFor(type, key, value)
for (const brand of brands) {
const brandValue = brand.tags[type]
const allStats = await TagInfo.getGlobalDistributionsFor(type, brandValue)
allData.push({ nsi: brand, stats: allStats })
}
return allData
}
public static supportedTags(type: "operator" | "brand" | "flag" | "transit" | string): Record<string, string[]> {
const tags: Record<string, string []> = {}
const keys = Object.keys(NameSuggestionIndex.nsiFile.nsi)
for (const key of keys) {
const nsiItem = NameSuggestionIndex.nsiFile.nsi[key]
const path = nsiItem.properties.path
const [osmType, osmkey, osmvalue] = path.split("/")
if (type !== osmType && (type + "s" !== osmType)) {
continue
}
if (!tags[osmkey]) {
tags[osmkey] = []
}
tags[osmkey].push(osmvalue)
}
return tags
}
public static allPossible(type: "brand" | "operator"): string[] {
const options: string[] = []
const tags = NameSuggestionIndex.supportedTags(type)
for (const osmKey in tags) {
const values = tags[osmKey]
for (const osmValue of values) {
const suggestions = this.getSuggestionsFor(type, osmKey, osmValue)
for (const suggestion of suggestions) {
const value = suggestion.tags[type]
options.push(value)
}
}
}
return Utils.Dedup(options)
}
/**
*
* @param path
* @param country
* @param location: center point of the feature, should be [lon, lat]
*/
public static getSuggestionsFor(type: string, key: string, value: string, country: string = undefined, location: [number, number] = undefined): NSIItem[] {
const path = `${type}s/${key}/${value}`
const entry = NameSuggestionIndex.nsiFile.nsi[path]
return entry?.items?.filter(i => {
if (i.locationSet.include.indexOf("001") >= 0) {
return true
}
if (country === undefined ||
// We prefer the countries provided by lonlat2country, they are more precise
// Country might contain multiple countries, separated by ';'
i.locationSet.include.some(c => country.indexOf(c) >= 0)) {
return true
}
if (location === undefined) {
return true
}
const resolvedSet = NameSuggestionIndex.loco.resolveLocationSet(i.locationSet)
if (resolvedSet) {
// We actually have a location set, so we can check if the feature is in it, by determining if our point is inside the MultiPolygon using @turf/boolean-point-in-polygon
// This might occur for some extra boundaries, such as counties, ...
const setFeature: Feature<MultiPolygon> = resolvedSet.feature
return turf.booleanPointInPolygon(location, setFeature.geometry)
}
return false
})
}
}

9
src/Models/ThemeConfig/Conversion/Validation.ts
View file

@ -26,6 +26,7 @@ import { Translatable } from "../Json/Translatable"
import { ConversionContext } from "./ConversionContext"
import { AvailableRasterLayers } from "../../RasterLayers"
import PointRenderingConfigJson from "../Json/PointRenderingConfigJson"
import NameSuggestionIndex from "../../../Logic/Web/NameSuggestionIndex"
class ValidateLanguageCompleteness extends DesugaringStep<LayoutConfig> {
private readonly _languages: string[]
@ -1032,6 +1033,14 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
)
}
}
if(json.freeform.type === "nsi"){
const [key, value] = json.freeform.helperArgs[0].split("=")
const path = `${json.freeform.key}s/${key}/${value}`
const suggestions = NameSuggestionIndex.getSuggestionsFor(path)
if(suggestions === undefined){
context.enters("freeform","type").err("No entry found in the 'Name Suggestion Index' for "+path)
}
}
}
if (json.render && json["question"] && json.freeform === undefined) {
context.err(

145
src/UI/InputElement/Helpers/NameSuggestionIndexInput.svelte Normal file
View file

@ -0,0 +1,145 @@
<script lang="ts">
import type { Feature, MultiPolygon } from "geojson"
import { UIEventSource } from "../../../Logic/UIEventSource"
import { SearchIcon } from "@rgossiaux/svelte-heroicons/solid"
import { twMerge } from "tailwind-merge"
import { GeoOperations } from "../../../Logic/GeoOperations"
import NameSuggestionIndex from "../../../Logic/Web/NameSuggestionIndex"
import type { NSIItem } from "../../../Logic/Web/NameSuggestionIndex"
export let value: UIEventSource<string> = new UIEventSource<string>(undefined)
// Since we're getting extra tags aside from the standard we need to export them
export let extraTags: UIEventSource<Record<string, string>>
export let feature: Feature
export let helperArgs: (string | number | boolean)[]
/**
* An inputhelper is always used with a freeform.
* The 'key' is what the value will be written into.
* This will probably be `brand` or `operator`
*/
export let key: string
let maintag = helperArgs[0].toString()
let tag = key
let addExtraTags: string[] = []
if (helperArgs[1]) {
addExtraTags = helperArgs[1].split(";")
}
const path = `${key}s/${maintag.split("=")[0]}/${maintag.split("=")[1]}`
let items: NSIItem[] = NameSuggestionIndex.getSuggestionsFor(path, feature.properties["_country"], GeoOperations.centerpointCoordinates(feature))
let selectedItem: NSIItem = items.find((item) => item.tags[key] === value.data)
// Get the coordinates if the feature is a point, otherwise use the center
let [lon, lat] = GeoOperations.centerpointCoordinates(feature)
/**
* Filter the items, first by the display name, then by the location set
*/
let filter = ""
$: filteredItems = items
.filter((item) => item.displayName.toLowerCase().includes(filter.toLowerCase()))
.filter((item) => {
// Check if the feature is in the location set using the location-conflation library
return true
})
/**
* Function called when an item is selected
*/
function select(item: NSIItem) {
value.setData(item.tags[key])
selectedItem = item
// Transform the object into record<string, string> and remove the tag we're using, as well as the maintag
const tags = Object.entries(item.tags).reduce((acc, [k, value]) => {
if (k !== key && key !== maintag.split("=")[0]) {
acc[k] = value
}
return acc
}, {} as Record<string, string>)
// Also check if the object currently matches a different item, based on the key
const otherItem = items.find((item) => item.tags[key] === feature.properties[key])
// If the other item is not the same as the selected item, we need to make sure we clear any old keys we don't need anymore by setting them to an empty string
if (otherItem && otherItem !== item) {
Object.keys(otherItem.tags).forEach((k) => {
// If we have a different value for the key, we don't need to clear it
if (!tags[k] && key !== k && key !== maintag.split("=")[0]) {
console.log(`Clearing key ${key}`)
tags[k] = ""
}
})
}
// If we have layer-defined extra tags, also add them
addExtraTags.forEach((extraTag) => {
tags[extraTag.split("=")[0]] = extraTag.split("=")[1]
})
// Finally, set the extra tags
extraTags.setData(tags)
}
value.addCallback((value) => {
// If the value changes by the user typing we might need to update the selected item or make sure we clear any old keys
// First, check if the value is already selected, in that case we don't need to do anything
if (selectedItem && selectedItem.tags[tag] === value) {
return
}
// If the value is not selected, we check if there is an item with the same value and select it
const item = items.find((item) => item.tags[key] === value)
if (item) {
select(item)
} else {
// If there is no item with the value, we need to clear the extra tags based on the last selected item by looping over the tags from the last selected item
if (selectedItem) {
const tags = Object.entries(selectedItem.tags).reduce((acc, [k, value]) => {
if (key !== k && key !== maintag.split("=")[0]) {
acc[key] = ""
}
return acc
}, {} as Record<string, string>)
extraTags.setData(tags)
}
}
})
</script>
{#if items?.length >= 0}
<div>
<div class="normal-background my-2 flex w-5/6 justify-between rounded-full pl-2">
<input type="text" placeholder="Filter entries" bind:value={filter} class="outline-none" />
<SearchIcon aria-hidden="true" class="h-6 w-6 self-end" />
</div>
<div class="flex h-36 w-full flex-wrap overflow-scroll">
{#each filteredItems as item}
<div
class={twMerge(
"m-1 h-fit rounded-full border-2 border-black p-4 text-center text-black",
selectedItem === item ? "interactive" : "bg-white"
)}
on:click={() => {
select(item)
}}
on:keydown={(e) => {
if (e.key === "Enter") {
select(item)
}
}}
>
{item.displayName}
</div>
{/each}
</div>
</div>
{/if}