contrapunctus/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

chore: automated housekeeping...

Browse source
This commit is contained in:
Pieter Vander Vennet 2025-05-03 23:48:35 +02:00
parent 612e71a4b6
commit 511d20eb34
245 changed files with 43884 additions and 8661 deletions
Download patch file Download diff file Expand all files Collapse all files

1
scripts/Script.ts
View file

@ -26,7 +26,6 @@ export default abstract class Script {
const green = (s) => "\x1b[92m" + s + "\x1b[0m"
console.log(green("All done! (" + millisNeeded + " ms)"))
})
}
public printHelp() {

2
scripts/ScriptUtils.ts
View file

@ -49,7 +49,7 @@ export default class ScriptUtils {
return
}
fs.mkdirSync(parent, {
recursive: true
recursive: true,
})
}

5
scripts/downloadEli.ts
View file

@ -128,7 +128,10 @@ class DownloadEli extends Script {
fs.writeFileSync(targetGlobal, JSON.stringify(contentsGlobal, null, " "), {
encoding: "utf8",
})
console.log("Written", keptGlobalLayers.length + ", entries to the global ELI ("+targetGlobal+")")
console.log(
"Written",
keptGlobalLayers.length + ", entries to the global ELI (" + targetGlobal + ")"
)
}
}

2
scripts/generateFavouritesLayer.ts
View file

@ -6,7 +6,7 @@ import { AllKnownLayoutsLazy } from "../src/Customizations/AllKnownLayouts"
import { Utils } from "../src/Utils"
import {
MappingConfigJson,
QuestionableTagRenderingConfigJson
QuestionableTagRenderingConfigJson,
} from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import { TagConfigJson } from "../src/Models/ThemeConfig/Json/TagConfigJson"
import { TagUtils } from "../src/Logic/Tags/TagUtils"

126
scripts/generateLayerOverview.ts
View file

@ -9,12 +9,16 @@ import {
DoesImageExist,
PrevalidateTheme,
ValidateLayer,
ValidateThemeEnsemble
ValidateThemeEnsemble,
} from "../src/Models/ThemeConfig/Conversion/Validation"
import { Translation } from "../src/UI/i18n/Translation"
import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
import { PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme"
import { Conversion, DesugaringContext, DesugaringStep } from "../src/Models/ThemeConfig/Conversion/Conversion"
import {
Conversion,
DesugaringContext,
DesugaringStep,
} from "../src/Models/ThemeConfig/Conversion/Conversion"
import { Utils } from "../src/Utils"
import Script from "./Script"
import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
@ -31,7 +35,10 @@ import { Translatable } from "../src/Models/ThemeConfig/Json/Translatable"
import { ValidateThemeAndLayers } from "../src/Models/ThemeConfig/Conversion/ValidateThemeAndLayers"
import { ExtractImages } from "../src/Models/ThemeConfig/Conversion/FixImages"
import { TagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/TagRenderingConfigJson"
import { LayerConfigDependencyGraph, LevelInfo } from "../src/Models/ThemeConfig/LayerConfigDependencyGraph"
import {
LayerConfigDependencyGraph,
LevelInfo,
} from "../src/Models/ThemeConfig/LayerConfigDependencyGraph"
// This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files.
// It spits out an overview of those to be used to load them
@ -141,7 +148,7 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
private readonly prepareLayer: PrepareLayer
private readonly _levels: LevelInfo[]
private readonly _loadedIds: Set<string> = new Set<string>()
private readonly _layerConfigJsons = new Map<string, LayerConfigJson>
private readonly _layerConfigJsons = new Map<string, LayerConfigJson>()
private readonly _desugaringState: DesugaringContext
constructor(
@ -149,7 +156,8 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
dependencies: Map<string, string[]>,
levels: LevelInfo[],
states: Map<string, "clean" | "dirty" | "changed">,
sharedTagRenderings: QuestionableTagRenderingConfigJson[]) {
sharedTagRenderings: QuestionableTagRenderingConfigJson[]
) {
super("Builds all the layers, writes them to file", [], "LayerBuilder")
this._levels = levels
this._dependencies = dependencies
@ -157,7 +165,7 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
this._desugaringState = {
tagRenderings: LayerOverviewUtils.asDict(sharedTagRenderings),
tagRenderingOrder: sharedTagRenderings.map((tr) => tr.id),
sharedLayers: AllSharedLayers.getSharedLayersConfigs()
sharedLayers: AllSharedLayers.getSharedLayersConfigs(),
}
this.prepareLayer = new PrepareLayer(this._desugaringState)
for (const layerConfigJson of layerConfigJsons) {
@ -177,22 +185,23 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
if (!existsSync(LayerOverviewUtils.layerPath)) {
mkdirSync(LayerOverviewUtils.layerPath)
}
writeFileSync(
LayerBuilder.targetPath(layer.id),
JSON.stringify(layer, null, " "),
{ encoding: "utf8" }
)
writeFileSync(LayerBuilder.targetPath(layer.id), JSON.stringify(layer, null, " "), {
encoding: "utf8",
})
}
public buildLayer(id: string, context: ConversionContext, isLooping: boolean = false): LayerConfigJson {
public buildLayer(
id: string,
context: ConversionContext,
isLooping: boolean = false
): LayerConfigJson {
if (id === "questions") {
return undefined
}
const deps = this._dependencies.get(id)
if (!isLooping) {
// Beware of the looping traps. Bring the leaf to the statue to teleport to "The Lab" (<ref>submachine 4</ref>)
const unbuilt = deps.filter(depId => !this._loadedIds.has(depId))
const unbuilt = deps.filter((depId) => !this._loadedIds.has(depId))
for (const unbuiltId of unbuilt) {
this.buildLayer(unbuiltId, context)
}
@ -211,9 +220,10 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
const origIds: ReadonlyArray<string> = [...ids]
const deps = this._dependencies
const allDeps = Utils.Dedup([].concat(...ids.map(id => deps.get(id))))
const depsRecord = Utils.asRecord(Array.from(deps.keys()), k =>
deps.get(k).filter(dep => ids.indexOf(dep) >= 0))
const allDeps = Utils.Dedup([].concat(...ids.map((id) => deps.get(id))))
const depsRecord = Utils.asRecord(Array.from(deps.keys()), (k) =>
deps.get(k).filter((dep) => ids.indexOf(dep) >= 0)
)
const revDeps = Utils.TransposeMap(depsRecord)
for (const someDep of allDeps) {
if (ids.indexOf(someDep) >= 0) {
@ -225,7 +235,7 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
tagRenderings: [],
filter: [],
source: "special:stub",
allowMove: true
allowMove: true,
})
continue
}
@ -239,8 +249,13 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
if (first === "questions") {
continue
}
const oldConfig = this._desugaringState.sharedLayers.get(first) ?? this._layerConfigJsons.get(first)
const newConfig = this.buildLayer(first, context.inOperation("resolving a looped dependency"), true)
const oldConfig =
this._desugaringState.sharedLayers.get(first) ?? this._layerConfigJsons.get(first)
const newConfig = this.buildLayer(
first,
context.inOperation("resolving a looped dependency"),
true
)
const isDifferent = JSON.stringify(oldConfig) !== JSON.stringify(newConfig)
if (isDifferent) {
@ -258,11 +273,8 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
console.log("Done with the looping layers!")
}
public convert(o, context: ConversionContext):
Map<string, LayerConfigJson> {
for (const level of this._levels
) {
public convert(o, context: ConversionContext): Map<string, LayerConfigJson> {
for (const level of this._levels) {
if (level.loop) {
this.buildLooping(level.ids, context)
continue
@ -270,7 +282,9 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
for (let i = 0; i < level.ids.length; i++) {
const id = level.ids[i]
ScriptUtils.erasableLog(`Building level ${i}: validating layer ${i + 1}/${level.ids.length}: ${id}`)
ScriptUtils.erasableLog(
`Building level ${i}: validating layer ${i + 1}/${level.ids.length}: ${id}`
)
if (id === "questions") {
continue
}
@ -282,7 +296,11 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
this._desugaringState.sharedLayers.set(id, loaded)
continue
} catch (e) {
console.error("Could not load generated layer file for ", id, " building it instead")
console.error(
"Could not load generated layer file for ",
id,
" building it instead"
)
}
}
}
@ -294,8 +312,6 @@ class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
context.info("Recompiled " + this._loadedIds.size + " layers")
return this._desugaringState.sharedLayers
}
}
class LayerOverviewUtils extends Script {
@ -722,7 +738,6 @@ class LayerOverviewUtils extends Script {
(l) => Constants.added_by_default.indexOf(l["id"]) < 0
)
}
}
private parseLayer(
@ -749,32 +764,28 @@ class LayerOverviewUtils extends Script {
const results: LayerConfigJson[] = []
for (let i = 0; i < allPaths.length; i++) {
const path = allPaths[i]
ScriptUtils.erasableLog(`Parsing layerConfig ${i + 1}/${allPaths.length}: ${path} `)
ScriptUtils.erasableLog(
`Parsing layerConfig ${i + 1}/${allPaths.length}: ${path} `
)
try {
const data = JSON.parse(readFileSync(path, "utf8"))
results.push(data)
results.push(data)
} catch (e) {
throw "Could not parse layer file " + path
}
}
return results
}
private buildLayerIndex(
doesImageExist: DoesImageExist
): Map<string, LayerConfigJson> {
private buildLayerIndex(doesImageExist: DoesImageExist): Map<string, LayerConfigJson> {
// First, we expand and validate all builtin layers. These are written to src/assets/generated/layers
// At the same time, an index of available layers is built.
const sharedQuestions = this.getSharedTagRenderings(doesImageExist)
const allLayerConfigs = this.getAllLayerConfigs()
const sharedQuestionsDef = allLayerConfigs.find(l => l.id === "questions")
const sharedQuestionsDef = allLayerConfigs.find((l) => l.id === "questions")
sharedQuestionsDef.tagRenderings = sharedQuestions
const dependencyGraph = LayerConfigDependencyGraph.buildDirectDependencies(allLayerConfigs)
const levels = LayerConfigDependencyGraph.buildLevels(dependencyGraph)
const layerState = new Map<string, "clean" | "dirty" | "changed">()
@ -786,13 +797,13 @@ class LayerOverviewUtils extends Script {
let allClean = true
for (const id of levelInfo.ids) {
const deps = dependencyGraph.get(id) ?? []
const dirtyDeps = deps.filter(dep => {
const dirtyDeps = deps.filter((dep) => {
const depState = layerState.get(dep)
if (levelInfo.loop && depState === undefined) {
const depIsClean =
LayerOverviewUtils.shouldBeUpdated(
LayerBuilder.sourcePath(dep),
LayerBuilder.targetPath(dep))
const depIsClean = LayerOverviewUtils.shouldBeUpdated(
LayerBuilder.sourcePath(dep),
LayerBuilder.targetPath(dep)
)
if (depIsClean) {
return false
}
@ -814,7 +825,7 @@ class LayerOverviewUtils extends Script {
const state = layerState.get(id)
if (state !== "clean") {
allClean = false
console.log(`- ${id} (${state}; ${dirtyDeps.map(dd => dd + "*").join(", ")})`)
console.log(`- ${id} (${state}; ${dirtyDeps.map((dd) => dd + "*").join(", ")})`)
}
}
if (!allClean) {
@ -822,8 +833,13 @@ class LayerOverviewUtils extends Script {
}
}
const builder = new LayerBuilder(allLayerConfigs, dependencyGraph, levels, layerState, sharedQuestions)
const builder = new LayerBuilder(
allLayerConfigs,
dependencyGraph,
levels,
layerState,
sharedQuestions
)
builder.writeLayer(sharedQuestionsDef)
const allLayers = builder.convertStrict({}, ConversionContext.construct([], []))
if (layerState.get("usersettings") !== "clean") {
@ -831,12 +847,11 @@ class LayerOverviewUtils extends Script {
LayerOverviewUtils.extractJavascriptCodeForLayer(
allLayers.get("usersettings"),
"./src/Logic/State/UserSettingsMetaTagging.ts"
)
"./src/Logic/State/UserSettingsMetaTagging.ts"
)
}
return allLayers
}
/**
@ -867,7 +882,9 @@ class LayerOverviewUtils extends Script {
"(feat: Feature, helperFunctions: Record<ExtraFuncType, (feature: Feature) => Function>) {"
)
if (code?.length > 0) {
allCode.push(" const {" + ExtraFunctions.types.join(", ") + "} = helperFunctions")
allCode.push(
" const {" + ExtraFunctions.types.join(", ") + "} = helperFunctions"
)
}
for (const line of code) {
const firstEq = line.indexOf("=")
@ -1016,7 +1033,10 @@ class LayerOverviewUtils extends Script {
LayerOverviewUtils.extractLayerIdsFrom(themeFile, false)
).map((id) => LayerOverviewUtils.layerPath + id + ".json")
if (!forceReload && !LayerOverviewUtils.shouldBeUpdated([themePath, ...usedLayers], targetPath)) {
if (
!forceReload &&
!LayerOverviewUtils.shouldBeUpdated([themePath, ...usedLayers], targetPath)
) {
fixed.set(
themeFile.id,
JSON.parse(

1
scripts/generateNsiStats.ts
View file

@ -11,7 +11,6 @@ import NameSuggestionIndex from "../src/Logic/Web/NameSuggestionIndex"
import TagInfo from "../src/Logic/Web/TagInfo"
import { TagsFilter } from "../src/Logic/Tags/TagsFilter"
class GenerateNsiStats extends Script {
async createOptimizationFile(includeTags = true) {
ScriptUtils.fixUtils()

5
scripts/nsiLogos.ts
View file

@ -1,5 +1,8 @@
import Script from "./Script"
import NameSuggestionIndex, { NamgeSuggestionWikidata, NSIItem } from "../src/Logic/Web/NameSuggestionIndex"
import NameSuggestionIndex, {
NamgeSuggestionWikidata,
NSIItem,
} from "../src/Logic/Web/NameSuggestionIndex"
import * as nsiWD from "../node_modules/name-suggestion-index/dist/wikidata.min.json"
import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "fs"
import ScriptUtils from "./ScriptUtils"

7
scripts/removeTranslationString.ts
View file

@ -42,7 +42,12 @@ async function main(args: string[]) {
if (removedLanguages.length === 0) {
console.warn("No items removed. Doublecheck the paths")
} else {
console.log("Removed items in " + removedLanguages.join(", "), ";", removedLanguages.length, "in total")
console.log(
"Removed items in " + removedLanguages.join(", "),
";",
removedLanguages.length,
"in total"
)
}
}

1
scripts/serverLdScrape.ts
View file

@ -22,7 +22,6 @@ class ServerLdScrape extends Script {
"User-Agent": "MapComplete/openstreetmap scraper; pietervdvn@posteo.net; https://source.mapcomplete.org/MapComplete",
"accept": "application/html"
},*/
]
for (let i = 0; i < headers.length; i++) {
try {