Move NSI-code in separate file
This commit is contained in:
parent
037887fea0
commit
07edee550c
4 changed files with 428 additions and 71 deletions
|
|
@ -2,88 +2,138 @@ import known_layers from "../src/assets/generated/known_layers.json"
|
|||
import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
|
||||
import { TagUtils } from "../src/Logic/Tags/TagUtils"
|
||||
import { Utils } from "../src/Utils"
|
||||
import { writeFileSync } from "fs"
|
||||
import { existsSync, readFileSync, writeFileSync } from "fs"
|
||||
import ScriptUtils from "./ScriptUtils"
|
||||
import TagRenderingConfig from "../src/Models/ThemeConfig/TagRenderingConfig"
|
||||
import { And } from "../src/Logic/Tags/And"
|
||||
import Script from "./Script"
|
||||
import NameSuggestionIndex, { NSIItem } from "../src/Logic/Web/NameSuggestionIndex"
|
||||
import TagInfo, { TagInfoStats } from "../src/Logic/Web/TagInfo"
|
||||
|
||||
/* Downloads stats on osmSource-tags and keys from tagInfo */
|
||||
|
||||
async function main(includeTags = true) {
|
||||
ScriptUtils.fixUtils()
|
||||
const layers = <LayerConfigJson[]>known_layers.layers
|
||||
|
||||
const keysAndTags = new Map<string, Set<string>>()
|
||||
|
||||
for (const layer of layers) {
|
||||
if (layer.source["geoJson"] !== undefined && !layer.source["isOsmCache"]) {
|
||||
continue
|
||||
}
|
||||
if (layer.source == null || typeof layer.source === "string") {
|
||||
continue
|
||||
class Utilities {
|
||||
static mapValues<X extends string | number, T, TOut>(record: Record<X, T>, f: ((t: T) => TOut)): Record<X, TOut> {
|
||||
const newR = <Record<X, TOut>>{}
|
||||
for (const x in record) {
|
||||
newR[x] = f(record[x])
|
||||
}
|
||||
return newR
|
||||
}
|
||||
}
|
||||
class GenerateStats extends Script {
|
||||
|
||||
const sourcesList = [TagUtils.Tag(layer.source["osmTags"])]
|
||||
if (layer?.title) {
|
||||
sourcesList.push(...new TagRenderingConfig(layer.title).usedTags())
|
||||
}
|
||||
async createOptimizationFile(includeTags = true) {
|
||||
ScriptUtils.fixUtils()
|
||||
const layers = <LayerConfigJson[]>known_layers.layers
|
||||
|
||||
const sources = new And(sourcesList)
|
||||
const allKeys = sources.usedKeys()
|
||||
for (const key of allKeys) {
|
||||
if (!keysAndTags.has(key)) {
|
||||
keysAndTags.set(key, new Set<string>())
|
||||
const keysAndTags = new Map<string, Set<string>>()
|
||||
|
||||
for (const layer of layers) {
|
||||
if (layer.source["geoJson"] !== undefined && !layer.source["isOsmCache"]) {
|
||||
continue
|
||||
}
|
||||
if (layer.source == null || typeof layer.source === "string") {
|
||||
continue
|
||||
}
|
||||
|
||||
const sourcesList = [TagUtils.Tag(layer.source["osmTags"])]
|
||||
if (layer?.title) {
|
||||
sourcesList.push(...new TagRenderingConfig(layer.title).usedTags())
|
||||
}
|
||||
|
||||
const sources = new And(sourcesList)
|
||||
const allKeys = sources.usedKeys()
|
||||
for (const key of allKeys) {
|
||||
if (!keysAndTags.has(key)) {
|
||||
keysAndTags.set(key, new Set<string>())
|
||||
}
|
||||
}
|
||||
const allTags = includeTags ? sources.usedTags() : []
|
||||
for (const tag of allTags) {
|
||||
if (!keysAndTags.has(tag.key)) {
|
||||
keysAndTags.set(tag.key, new Set<string>())
|
||||
}
|
||||
keysAndTags.get(tag.key).add(tag.value)
|
||||
}
|
||||
}
|
||||
const allTags = includeTags ? sources.usedTags() : []
|
||||
for (const tag of allTags) {
|
||||
if (!keysAndTags.has(tag.key)) {
|
||||
keysAndTags.set(tag.key, new Set<string>())
|
||||
}
|
||||
keysAndTags.get(tag.key).add(tag.value)
|
||||
}
|
||||
|
||||
const keyTotal = new Map<string, number>()
|
||||
const tagTotal = new Map<string, Map<string, number>>()
|
||||
await Promise.all(
|
||||
Array.from(keysAndTags.keys()).map(async (key) => {
|
||||
const values = keysAndTags.get(key)
|
||||
const data = await Utils.downloadJson(
|
||||
`https://taginfo.openstreetmap.org/api/4/key/stats?key=${key}`
|
||||
)
|
||||
const count = data.data.find((item) => item.type === "all").count
|
||||
keyTotal.set(key, count)
|
||||
console.log(key, "-->", count)
|
||||
|
||||
if (values.size > 0) {
|
||||
tagTotal.set(key, new Map<string, number>())
|
||||
await Promise.all(
|
||||
Array.from(values).map(async (value) => {
|
||||
const tagData = await Utils.downloadJson(
|
||||
`https://taginfo.openstreetmap.org/api/4/tag/stats?key=${key}&value=${value}`
|
||||
)
|
||||
const count = tagData.data.find((item) => item.type === "all").count
|
||||
tagTotal.get(key).set(value, count)
|
||||
console.log(key + "=" + value, "-->", count)
|
||||
})
|
||||
)
|
||||
}
|
||||
})
|
||||
)
|
||||
writeFileSync(
|
||||
"./src/assets/key_totals.json",
|
||||
JSON.stringify(
|
||||
{
|
||||
"#": "Generated with generateStats.ts",
|
||||
date: new Date().toISOString(),
|
||||
keys: Utils.MapToObj(keyTotal, (t) => t),
|
||||
tags: Utils.MapToObj(tagTotal, (v) => Utils.MapToObj(v, (t) => t))
|
||||
},
|
||||
null,
|
||||
" "
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
const keyTotal = new Map<string, number>()
|
||||
const tagTotal = new Map<string, Map<string, number>>()
|
||||
await Promise.all(
|
||||
Array.from(keysAndTags.keys()).map(async (key) => {
|
||||
const values = keysAndTags.get(key)
|
||||
const data = await Utils.downloadJson(
|
||||
`https://taginfo.openstreetmap.org/api/4/key/stats?key=${key}`
|
||||
)
|
||||
const count = data.data.find((item) => item.type === "all").count
|
||||
keyTotal.set(key, count)
|
||||
console.log(key, "-->", count)
|
||||
|
||||
if (values.size > 0) {
|
||||
tagTotal.set(key, new Map<string, number>())
|
||||
await Promise.all(
|
||||
Array.from(values).map(async (value) => {
|
||||
const tagData = await Utils.downloadJson(
|
||||
`https://taginfo.openstreetmap.org/api/4/tag/stats?key=${key}&value=${value}`
|
||||
)
|
||||
const count = tagData.data.find((item) => item.type === "all").count
|
||||
tagTotal.get(key).set(value, count)
|
||||
console.log(key + "=" + value, "-->", count)
|
||||
})
|
||||
)
|
||||
async createNameSuggestionIndexFile() {
|
||||
const type = "brand"
|
||||
let allBrands = <Record<string, Record<string, number>>>{}
|
||||
const path = "./src/assets/generated/nsi_stats/" + type + ".json"
|
||||
if (existsSync(path)) {
|
||||
allBrands = JSON.parse(readFileSync(path, "utf8"))
|
||||
console.log("Loaded",Object.keys(allBrands).length," previously loaded brands")
|
||||
}
|
||||
let lastWrite = new Date()
|
||||
const allBrandNames: string[] = NameSuggestionIndex.allPossible(type)
|
||||
for (const brand of allBrandNames) {
|
||||
if(allBrands[brand] !== undefined){
|
||||
console.log("Skipping", brand,", already loaded")
|
||||
continue
|
||||
}
|
||||
})
|
||||
)
|
||||
writeFileSync(
|
||||
"./src/assets/key_totals.json",
|
||||
JSON.stringify(
|
||||
{
|
||||
"#": "Generated with generateStats.ts",
|
||||
date: new Date().toISOString(),
|
||||
keys: Utils.MapToObj(keyTotal, (t) => t),
|
||||
tags: Utils.MapToObj(tagTotal, (v) => Utils.MapToObj(v, (t) => t)),
|
||||
},
|
||||
null,
|
||||
" "
|
||||
)
|
||||
)
|
||||
const distribution: Record<string, number> = Utilities.mapValues(await TagInfo.getGlobalDistributionsFor(type, brand), s => s.data.find(t => t.type === "all").count)
|
||||
allBrands[brand] = distribution
|
||||
if ((new Date().getTime() - lastWrite.getTime()) / 1000 >= 5) {
|
||||
writeFileSync(path, JSON.stringify(allBrands), "utf8")
|
||||
console.log("Checkpointed", path)
|
||||
}
|
||||
}
|
||||
writeFileSync(path, JSON.stringify(allBrands), "utf8")
|
||||
}
|
||||
|
||||
constructor() {
|
||||
super("Downloads stats on osmSource-tags and keys from tagInfo. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index ")
|
||||
}
|
||||
|
||||
async main(_: string[]) {
|
||||
// this.createOptimizationFile()
|
||||
await this.createNameSuggestionIndexFile()
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
main().then(() => console.log("All done"))
|
||||
|
||||
new GenerateStats().run()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue