Bastian-Greshake-Tzovaras/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Merge develop

Browse source
This commit is contained in:
Pieter Vander Vennet 2025-01-12 13:33:38 +01:00
commit baa7379fbf
7880 changed files with 2079327 additions and 39792 deletions
Download patch file Download diff file Expand all files Collapse all files

104
scripts/downloadNsiLogos.ts
View file

@ -1,9 +1,13 @@
import Script from "./Script"
import NameSuggestionIndex, { NSIItem } from "../src/Logic/Web/NameSuggestionIndex"
import * as nsiWD from "../node_modules/name-suggestion-index/dist/wikidata.min.json"
import { existsSync, readFileSync, unlinkSync, writeFileSync } from "fs"
import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
import ScriptUtils from "./ScriptUtils"
import { Utils } from "../src/Utils"
import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
import { FilterConfigOptionJson } from "../src/Models/ThemeConfig/Json/FilterConfigJson"
import { TagUtils } from "../src/Logic/Tags/TagUtils"
import { TagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/TagRenderingConfigJson"
class DownloadNsiLogos extends Script {
constructor() {
@ -26,8 +30,8 @@ class DownloadNsiLogos extends Script {
let path = basePath + nsiItem.id
const logos = nsiWD["wikidata"][nsiItem?.tags?.[type + ":wikidata"]]?.logos
if (NameSuggestionIndex.isSvg(nsiItem, type)) {
const nsi = await NameSuggestionIndex.getNsiIndex()
if (nsi.isSvg(nsiItem, type)) {
path = path + ".svg"
}
@ -43,7 +47,7 @@ class DownloadNsiLogos extends Script {
await ScriptUtils.DownloadFileTo(logos.facebook, path)
// Validate
const content = readFileSync(path, "utf8")
if (content.startsWith('{"error"')) {
if (content.startsWith("{\"error\"")) {
unlinkSync(path)
console.error("Attempted to fetch", logos.facebook, " but this gave an error")
} else {
@ -86,13 +90,10 @@ class DownloadNsiLogos extends Script {
return false
}
async main(): Promise<void> {
await this.downloadFor("operator")
await this.downloadFor("brand")
}
async downloadFor(type: "brand" | "operator"): Promise<void> {
const items = NameSuggestionIndex.allPossible(type)
async downloadFor(type: string): Promise<void> {
const nsi = await NameSuggestionIndex.getNsiIndex()
const items = nsi.allPossible(type)
const basePath = "./public/assets/data/nsi/logos/"
let downloadCount = 0
const stepcount = 5
@ -108,7 +109,7 @@ class DownloadNsiLogos extends Script {
downloadCount++
}
return downloaded
})
}),
)
for (let j = 0; j < results.length; j++) {
let didDownload = results[j]
@ -123,6 +124,87 @@ class DownloadNsiLogos extends Script {
}
}
}
private async generateRendering(type: string) {
const nsi = await NameSuggestionIndex.getNsiIndex()
const items = nsi.allPossible(type)
const filterOptions: FilterConfigOptionJson[] = items.map(item => {
return ({
question: item.displayName,
icon: nsi.getIconUrl(item, type),
osmTags: NameSuggestionIndex.asFilterTags(item),
})
})
const mappings = items.map(item => ({
if: NameSuggestionIndex.asFilterTags(item),
then: nsi.getIconUrl(item, type),
}))
console.log("Checking for shadow-mappings...")
for (let i = mappings.length - 1; i >= 0; i--) {
const condition = TagUtils.Tag(mappings[i].if)
if (i % 100 === 0) {
console.log("Checking for shadow-mappings...", i, "/", mappings.length)
}
const shadowsSomething = mappings.some((m, j) => {
if (i === j) {
return false
}
return condition.shadows(TagUtils.Tag(m.if))
})
// If this one matches, the other one will match as well
// We can thus remove this one in favour of the other one
if (shadowsSomething) {
mappings.splice(i, 1)
}
}
const iconsTr: TagRenderingConfigJson = <any>{
strict: true,
id: "icon",
mappings,
}
const config: LayerConfigJson = {
id: "nsi_" + type,
description: {
en: "Exposes part of the NSI to reuse in other themes, e.g. for rendering",
},
source: "special:library",
pointRendering: null,
tagRenderings: [
iconsTr,
],
filter: [
<any>{
"#": "ignore-possible-duplicate",
id: type,
strict: true,
options: [{ question: type }, ...filterOptions],
},
],
allowMove: false,
"#dont-translate": "*",
}
const path = "./assets/layers/nsi_" + type
mkdirSync(path, { recursive: true })
writeFileSync(path + "/nsi_" + type + ".json", JSON.stringify(config, null, " "))
console.log("Written", path)
}
async main(): Promise<void> {
const nsi = await NameSuggestionIndex.getNsiIndex()
const types = ["brand", "operator"]
for (const type of types) {
await this.downloadFor(type)
}
for (const type of types) {
await this.generateRendering(type)
}
}
}
new DownloadNsiLogos().run()

2
scripts/generateDocs.ts
View file

@ -9,7 +9,7 @@ import ScriptUtils from "./ScriptUtils"
import Translations from "../src/UI/i18n/Translations"
import themeOverview from "../src/assets/generated/theme_overview.json"
import ThemeConfig from "../src/Models/ThemeConfig/ThemeConfig"
import bookcases from "../src/assets/generated/themes/bookcases.json"
import bookcases from "../public/assets/generated/themes/bookcases.json"
import fakedom from "fake-dom"
import unit from "../src/assets/generated/layers/unit.json"
import Hotkeys from "../src/UI/Base/Hotkeys"

23
scripts/generateLayerOverview.ts
View file

@ -140,7 +140,7 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye
class LayerOverviewUtils extends Script {
public static readonly layerPath = "./src/assets/generated/layers/"
public static readonly themePath = "./src/assets/generated/themes/"
public static readonly themePath = "./public/assets/generated/themes/"
constructor() {
super("Reviews and generates the compiled themes")
@ -319,12 +319,12 @@ class LayerOverviewUtils extends Script {
keywords,
layers: theme.layers.filter((l) => sharedLayers.has(l["id"])).map((l) => l["id"]),
}
perId.set(theme.id, data)
perId.set(data.id, data)
}
const sorted = Constants.themeOrder.map((id) => {
if (!perId.has(id)) {
throw "Ordered theme id " + id + " not found"
throw "Ordered theme '" + id + "' not found"
}
return perId.get(id)
})
@ -497,6 +497,8 @@ class LayerOverviewUtils extends Script {
priviliged.delete(key)
})
// These two get a free pass
priviliged.delete("summary")
priviliged.delete("last_click")
@ -527,7 +529,7 @@ class LayerOverviewUtils extends Script {
writeFileSync(
"./src/assets/generated/known_layers.json",
JSON.stringify({
layers: Array.from(sharedLayers.values()).filter((l) => l.id !== "favourite"),
layers: Array.from(sharedLayers.values()).filter((l) => !(l["#no-index"] === "yes")),
})
)
}
@ -575,15 +577,6 @@ class LayerOverviewUtils extends Script {
)
}
if (recompiledThemes.length > 0) {
writeFileSync(
"./src/assets/generated/known_themes.json",
JSON.stringify({
themes: Array.from(sharedThemes.values()),
})
)
}
if (AllSharedLayers.getSharedLayersConfigs().size == 0) {
console.error("This was a bootstrapping-run. Run generate layeroverview again!")
}
@ -647,7 +640,6 @@ class LayerOverviewUtils extends Script {
const sharedLayer = JSON.parse(readFileSync(targetPath, "utf8"))
sharedLayers.set(sharedLayer.id, sharedLayer)
skippedLayers.push(sharedLayer.id)
ScriptUtils.erasableLog("Loaded " + sharedLayer.id)
continue
} catch (e) {
throw "Could not parse " + targetPath + " : " + e
@ -847,6 +839,9 @@ class LayerOverviewUtils extends Script {
const themeInfo = themeFiles[i]
const themePath = themeInfo.path
let themeFile = themeInfo.parsed
if(!themeFile){
throw "Got an empty file for"+themeInfo.path
}
if (whitelist.size > 0 && !whitelist.has(themeFile.id)) {
continue
}

21
scripts/generateLayouts.ts
View file

@ -2,7 +2,6 @@ import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFile, writeFi
import Locale from "../src/UI/i18n/Locale"
import Translations from "../src/UI/i18n/Translations"
import { Translation } from "../src/UI/i18n/Translation"
import all_known_layouts from "../src/assets/generated/known_themes.json"
import { ThemeConfigJson } from "../src/Models/ThemeConfig/Json/ThemeConfigJson"
import ThemeConfig from "../src/Models/ThemeConfig/ThemeConfig"
import xml2js from "xml2js"
@ -12,7 +11,6 @@ import SpecialVisualizations from "../src/UI/SpecialVisualizations"
import Constants from "../src/Models/Constants"
import {
AvailableRasterLayers,
EditorLayerIndexProperties,
RasterLayerPolygon,
} from "../src/Models/RasterLayers"
import { ImmutableStore } from "../src/Logic/UIEventSource"
@ -122,7 +120,7 @@ class GenerateLayouts extends Script {
return path
}
const svg = await ScriptUtils.ReadSvg(layout.icon)
let width: string = svg.$.width
let width: string = svg["$"].width
if (width === undefined) {
throw "The logo at " + layout.icon + " does not have a defined width"
}
@ -185,8 +183,8 @@ class GenerateLayouts extends Script {
"./public/assets/generated/images/theme_" + layout.id + "_white_background.svg"
{
const svg = await ScriptUtils.ReadSvg(icon)
const width: string = svg.$.width
const height: string = svg.$.height
const width: string = svg["$"].width
const height: string = svg["$"].height
const builder = new xml2js.Builder()
const withRect = { rect: { $: { width, height, style: "fill:#ffffff;" } }, ...svg }
@ -300,8 +298,8 @@ class GenerateLayouts extends Script {
Origin: "https://mapcomplete.org",
})
urls.push(...(f.properties["connect-src"] ?? []))
for (const key of Object.keys(styleSpec?.sources ?? {})) {
const url = styleSpec.sources[key].url
for (const key of Object.keys(styleSpec?.["sources"] ?? {})) {
const url = styleSpec["sources"][key].url
if (!url) {
continue
}
@ -587,7 +585,7 @@ class GenerateLayouts extends Script {
const filename = "index_" + theme.id + ".ts"
const imports = [
`import layout from "./src/assets/generated/themes/${theme.id}.json"`,
`import layout from "./public/assets/generated/themes/${theme.id}.json"`,
`import { ThemeMetaTagging } from "./src/assets/generated/metatagging/${theme.id}"`,
]
for (const layerName of Constants.added_by_default) {
@ -637,15 +635,14 @@ class GenerateLayouts extends Script {
"custom",
"theme",
]
// @ts-ignore
const all: ThemeConfigJson[] = all_known_layouts.themes
const args = process.argv
const theme = args[2]
if (theme !== undefined) {
console.warn("Only generating layout " + theme)
}
for (const i in all) {
const layoutConfigJson: ThemeConfigJson = all[i]
const paths = ScriptUtils.readDirRecSync("./public/assets/generated/themes/",1)
for (const i in paths) {
const layoutConfigJson = <ThemeConfigJson> JSON.parse(readFileSync(paths[i], "utf8"))
if (theme !== undefined && layoutConfigJson.id !== theme) {
continue
}

1
scripts/generateLicenseInfo.ts
View file

@ -170,7 +170,6 @@ export class GenerateLicenseInfo extends Script {
const whiteColours = Array.from(colours).map((c) => {
const rgb = Utils.color(c)
if (!rgb) {
console.log("Could not parse ", c)
return false
}
const { r, g, b } = rgb

99
scripts/generateStats.ts
View file

@ -2,7 +2,7 @@ import known_layers from "../src/assets/generated/known_layers.json"
import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
import { TagUtils } from "../src/Logic/Tags/TagUtils"
import { Utils } from "../src/Utils"
import { existsSync, readFileSync, writeFileSync } from "fs"
import { copyFileSync, existsSync, readFileSync, writeFileSync } from "fs"
import ScriptUtils from "./ScriptUtils"
import TagRenderingConfig from "../src/Models/ThemeConfig/TagRenderingConfig"
import { And } from "../src/Logic/Tags/And"
@ -13,7 +13,7 @@ import TagInfo from "../src/Logic/Web/TagInfo"
class Utilities {
static mapValues<X extends string | number, T, TOut>(
record: Record<X, T>,
f: (t: T) => TOut
f: (t: T) => TOut,
): Record<X, TOut> {
const newR = <Record<X, TOut>>{}
for (const x in record) {
@ -26,7 +26,7 @@ class Utilities {
class GenerateStats extends Script {
async createOptimizationFile(includeTags = true) {
ScriptUtils.fixUtils()
const layers = <LayerConfigJson[]>known_layers.layers
const layers = <LayerConfigJson[]>known_layers["layers"]
const keysAndTags = new Map<string, Set<string>>()
@ -77,10 +77,10 @@ class GenerateStats extends Script {
const count = tagData.data.find((item) => item.type === "all").count
tagTotal.get(key).set(value, count)
console.log(key + "=" + value, "-->", count)
})
}),
)
}
})
}),
)
writeFileSync(
"./src/assets/key_totals.json",
@ -92,8 +92,8 @@ class GenerateStats extends Script {
tags: Utils.MapToObj(tagTotal, (v) => Utils.MapToObj(v, (t) => t)),
},
null,
" "
)
" ",
),
)
}
@ -139,7 +139,7 @@ class GenerateStats extends Script {
async createNameSuggestionIndexFile(basepath: string, type: "brand" | "operator" | string) {
const path = basepath + type + ".json"
let allBrands = <Record<string, Record<string, number>>>{}
let allBrands: Record<string, Record<string, number>> = {}
if (existsSync(path)) {
allBrands = JSON.parse(readFileSync(path, "utf8"))
console.log(
@ -147,82 +147,65 @@ class GenerateStats extends Script {
Object.keys(allBrands).length,
" previously loaded " + type,
"from",
path
path,
)
}
let skipped = 0
const nsi = await NameSuggestionIndex.getNsiIndex()
const allBrandNames: string[] = Utils.Dedup(
NameSuggestionIndex.allPossible(type).map((item) => item.tags[type])
nsi.allPossible(<any>type).map((item) => item.tags[type]),
)
const missingBrandNames: string[] = []
for (let i = 0; i < allBrandNames.length; i++) {
const brand = allBrandNames[i]
if (!!allBrands[brand] && Object.keys(allBrands[brand]).length == 0) {
delete allBrands[brand]
console.log("Deleted", brand, "as no entries at all")
}
if (allBrands[brand] !== undefined) {
const max = Math.max(...Object.values(allBrands[brand]))
skipped++
if (skipped % 100 == 0) {
console.warn("Busy; ", i + "/" + allBrandNames.length, "; skipped", skipped)
}
if (max < 0) {
console.log("HMMMM:", allBrands[brand])
delete allBrands[brand]
} else {
continue
}
}
missingBrandNames.push(brand)
}
const batchSize = 101
for (let i = 0; i < missingBrandNames.length; i += batchSize) {
const batchSize = 50
for (let i = 0; i < allBrandNames.length; i += batchSize) {
console.warn(
"Downloading",
batchSize,
"items: ",
i + "/" + missingBrandNames.length,
"; skipped",
skipped,
"total:",
allBrandNames.length
i + "/" + allBrandNames.length,
)
const distributions = await Promise.all(
let downloaded = 0
await Promise.all(
Utils.TimesT(batchSize, async (j) => {
await ScriptUtils.sleep(j * 250)
return TagInfo.getGlobalDistributionsFor(type, missingBrandNames[i + j])
})
)
for (let j = 0; j < distributions.length; j++) {
const brand = missingBrandNames[i + j]
const distribution: Record<string, number> = Utilities.mapValues(
distributions[j],
(s) => s.data.find((t) => t.type === "all").count
)
allBrands[brand] = distribution
}
const brand = allBrandNames[i + j]
if (!allBrands[brand]) {
allBrands[brand] = {}
}
const writeInto = allBrands[brand]
const dloaded = await TagInfo.getGlobalDistributionsFor(
writeInto, (stats) => stats.data.find((t) => t.type === "all").count,
type, brand)
downloaded += dloaded
}))
console.log("Downloaded ", downloaded, " values this batch")
writeFileSync(path, JSON.stringify(allBrands), "utf8")
console.log("Checkpointed", path)
}
console.log("Written:", path)
writeFileSync(path, JSON.stringify(allBrands), "utf8")
}
constructor() {
super(
"Downloads stats on osmSource-tags and keys from tagInfo. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index "
"Downloads stats on osmSource-tags and keys from tagInfo. There are two usecases with separate outputs:\n 1. To optimize the query before sending it to overpass (generates ./src/assets/key_totals.json) \n 2. To amend the Name Suggestion Index ",
)
}
async main(_: string[]) {
const basepath = "./src/assets/generated/stats/"
await this.createOptimizationFile()
const target = "./public/assets/data/nsi/"
const basepath = target + "stats/"
{
const src = "./node_modules/name-suggestion-index/dist/"
const files = ["featureCollection.min.json", "nsi.min.json", "wikidata.min.json"]
console.log(process.cwd())
for (const file of files) {
console.log("Copying ", src + file, target + "/" + file)
copyFileSync(src + file, target + file)
}
}
for (const type of ["operator", "brand"]) {
await this.createNameSuggestionIndexFile(basepath, type)
this.summarizeNSI(basepath + type + ".json", "./public/assets/data/nsi/stats/" + type)
}
await this.createOptimizationFile()
}
}