forked from MapComplete/MapComplete
1134 lines
43 KiB
TypeScript
1134 lines
43 KiB
TypeScript
import ScriptUtils from "./ScriptUtils"
|
|
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "fs"
|
|
import licenses from "../src/assets/generated/license_info.json"
|
|
import { ThemeConfigJson } from "../src/Models/ThemeConfig/Json/ThemeConfigJson"
|
|
import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
|
|
import Constants from "../src/Models/Constants"
|
|
import {
|
|
DetectDuplicateFilters,
|
|
DoesImageExist,
|
|
PrevalidateTheme,
|
|
ValidateLayer,
|
|
ValidateThemeEnsemble
|
|
} from "../src/Models/ThemeConfig/Conversion/Validation"
|
|
import { Translation } from "../src/UI/i18n/Translation"
|
|
import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
|
|
import { PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme"
|
|
import { Conversion, DesugaringContext, DesugaringStep } from "../src/Models/ThemeConfig/Conversion/Conversion"
|
|
import { Utils } from "../src/Utils"
|
|
import Script from "./Script"
|
|
import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
|
|
import { parse as parse_html } from "node-html-parser"
|
|
import { ExtraFunctions } from "../src/Logic/ExtraFunctions"
|
|
import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
|
|
import LayerConfig from "../src/Models/ThemeConfig/LayerConfig"
|
|
import PointRenderingConfig from "../src/Models/ThemeConfig/PointRenderingConfig"
|
|
import { ConversionContext } from "../src/Models/ThemeConfig/Conversion/ConversionContext"
|
|
import { GenerateFavouritesLayer } from "./generateFavouritesLayer"
|
|
import ThemeConfig, { MinimalThemeInformation } from "../src/Models/ThemeConfig/ThemeConfig"
|
|
import Translations from "../src/UI/i18n/Translations"
|
|
import { Translatable } from "../src/Models/ThemeConfig/Json/Translatable"
|
|
import { ValidateThemeAndLayers } from "../src/Models/ThemeConfig/Conversion/ValidateThemeAndLayers"
|
|
import { ExtractImages } from "../src/Models/ThemeConfig/Conversion/FixImages"
|
|
import { TagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/TagRenderingConfigJson"
|
|
import { LayerConfigDependencyGraph, LevelInfo } from "../src/Models/ThemeConfig/LayerConfigDependencyGraph"
|
|
|
|
// This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files.
|
|
// It spits out an overview of those to be used to load them
|
|
|
|
class ParseLayer extends Conversion<
|
|
string,
|
|
{
|
|
parsed: LayerConfig
|
|
raw: LayerConfigJson
|
|
}
|
|
> {
|
|
private readonly _prepareLayer: PrepareLayer
|
|
private readonly _doesImageExist: DoesImageExist
|
|
|
|
constructor(prepareLayer: PrepareLayer, doesImageExist: DoesImageExist) {
|
|
super("Parsed a layer from file, validates it", [], "ParseLayer")
|
|
this._prepareLayer = prepareLayer
|
|
this._doesImageExist = doesImageExist
|
|
}
|
|
|
|
convert(
|
|
path: string,
|
|
context: ConversionContext
|
|
): {
|
|
parsed: LayerConfig
|
|
raw: LayerConfigJson
|
|
} {
|
|
let parsed
|
|
let fileContents
|
|
try {
|
|
fileContents = readFileSync(path, "utf8")
|
|
} catch (e) {
|
|
context.err("Could not read file " + path + " due to " + e)
|
|
return undefined
|
|
}
|
|
try {
|
|
parsed = JSON.parse(fileContents)
|
|
} catch (e) {
|
|
context.err("Could not parse file as JSON: " + e)
|
|
return undefined
|
|
}
|
|
if (parsed === undefined) {
|
|
context.err("yielded undefined")
|
|
return undefined
|
|
}
|
|
const fixed = this._prepareLayer.convert(parsed, context.inOperation("PrepareLayer"))
|
|
|
|
if (!fixed.source && fixed.presets?.length < 1) {
|
|
context
|
|
.enter("source")
|
|
.err(
|
|
"No source is configured. (Tags might be automatically derived if presets are given)"
|
|
)
|
|
return undefined
|
|
}
|
|
|
|
if (
|
|
fixed.source &&
|
|
typeof fixed.source !== "string" &&
|
|
fixed.source?.["osmTags"] &&
|
|
fixed.source?.["osmTags"]["and"] === undefined
|
|
) {
|
|
fixed.source["osmTags"] = { and: [fixed.source["osmTags"]] }
|
|
}
|
|
|
|
const validator = new ValidateLayer(path, true, this._doesImageExist)
|
|
return validator.convert(fixed, context.inOperation("ValidateLayer"))
|
|
}
|
|
}
|
|
|
|
class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: LayerConfig }> {
|
|
static singleton = new AddIconSummary()
|
|
|
|
constructor() {
|
|
super("Adds an icon summary for quick reference", ["_layerIcon"], "AddIconSummary")
|
|
}
|
|
|
|
convert(json: { raw: LayerConfigJson; parsed: LayerConfig }) {
|
|
// Add a summary of the icon
|
|
const fixed = json.raw
|
|
const layerConfig = json.parsed
|
|
const pointRendering: PointRenderingConfig = layerConfig.mapRendering.find((pr) =>
|
|
pr.location.has("point")
|
|
)
|
|
const defaultTags = layerConfig.baseTags
|
|
fixed["_layerIcon"] = Utils.NoNull(
|
|
(pointRendering?.marker ?? []).map((i) => {
|
|
const icon = i.icon?.GetRenderValue(defaultTags)?.txt
|
|
if (!icon) {
|
|
return undefined
|
|
}
|
|
const result = { icon }
|
|
const c = i.color?.GetRenderValue(defaultTags)?.txt
|
|
if (c) {
|
|
result["color"] = c
|
|
}
|
|
return result
|
|
})
|
|
)
|
|
return { raw: fixed, parsed: layerConfig }
|
|
}
|
|
}
|
|
|
|
class LayerBuilder extends Conversion<object, Map<string, LayerConfigJson>> {
|
|
private readonly _dependencies: ReadonlyMap<string, string[]>
|
|
private readonly _states: Map<string, "clean" | "dirty" | "changed">
|
|
private readonly prepareLayer: PrepareLayer
|
|
private readonly _levels: LevelInfo[]
|
|
private readonly _loadedIds: Set<string> = new Set<string>()
|
|
private readonly _layerConfigJsons = new Map<string, LayerConfigJson>
|
|
private readonly _desugaringState: DesugaringContext
|
|
|
|
constructor(
|
|
layerConfigJsons: LayerConfigJson[],
|
|
dependencies: Map<string, string[]>,
|
|
levels: LevelInfo[],
|
|
states: Map<string, "clean" | "dirty" | "changed">,
|
|
sharedTagRenderings: QuestionableTagRenderingConfigJson[]) {
|
|
super("Builds all the layers, writes them to file", [], "LayerBuilder")
|
|
this._levels = levels
|
|
this._dependencies = dependencies
|
|
this._states = states
|
|
this._desugaringState = {
|
|
tagRenderings: LayerOverviewUtils.asDict(sharedTagRenderings),
|
|
tagRenderingOrder: sharedTagRenderings.map((tr) => tr.id),
|
|
sharedLayers: AllSharedLayers.getSharedLayersConfigs()
|
|
}
|
|
this.prepareLayer = new PrepareLayer(this._desugaringState)
|
|
for (const layerConfigJson of layerConfigJsons) {
|
|
this._layerConfigJsons.set(layerConfigJson.id, layerConfigJson)
|
|
}
|
|
}
|
|
|
|
public static targetPath(id: string): string {
|
|
return `${LayerOverviewUtils.layerPath}${id}.json`
|
|
}
|
|
|
|
public static sourcePath(id: string): string {
|
|
return `./assets/layers/${id}/${id}.json`
|
|
}
|
|
|
|
writeLayer(layer: LayerConfigJson) {
|
|
if (!existsSync(LayerOverviewUtils.layerPath)) {
|
|
mkdirSync(LayerOverviewUtils.layerPath)
|
|
}
|
|
writeFileSync(
|
|
LayerBuilder.targetPath(layer.id),
|
|
JSON.stringify(layer, null, " "),
|
|
{ encoding: "utf8" }
|
|
)
|
|
}
|
|
|
|
|
|
public buildLayer(id: string, context: ConversionContext, isLooping: boolean = false): LayerConfigJson {
|
|
if (id === "questions") {
|
|
return undefined
|
|
}
|
|
const deps = this._dependencies.get(id)
|
|
if (!isLooping) {
|
|
// Beware of the looping traps. Bring the leaf to the statue to teleport to "The Lab" (<ref>submachine 4</ref>)
|
|
const unbuilt = deps.filter(depId => !this._loadedIds.has(depId))
|
|
for (const unbuiltId of unbuilt) {
|
|
this.buildLayer(unbuiltId, context)
|
|
}
|
|
}
|
|
|
|
context = context.inOperation("building Layer " + id).enters("layer", id)
|
|
|
|
const config = this._layerConfigJsons.get(id)
|
|
const prepped = this.prepareLayer.convert(config, context)
|
|
this._loadedIds.add(id)
|
|
this._desugaringState.sharedLayers.set(id, prepped)
|
|
return prepped
|
|
}
|
|
|
|
private buildLooping(ids: string[], context: ConversionContext) {
|
|
const origIds: ReadonlyArray<string> = [...ids]
|
|
|
|
const deps = this._dependencies
|
|
const allDeps = Utils.Dedup([].concat(...ids.map(id => deps.get(id))))
|
|
const depsRecord = Utils.asRecord(Array.from(deps.keys()), k =>
|
|
deps.get(k).filter(dep => ids.indexOf(dep) >= 0))
|
|
const revDeps = Utils.TransposeMap(depsRecord)
|
|
for (const someDep of allDeps) {
|
|
if (ids.indexOf(someDep) >= 0) {
|
|
// BY definition, we _will_ need this dependency
|
|
// We add a small stub
|
|
this._desugaringState.sharedLayers.set(someDep, {
|
|
id: someDep,
|
|
pointRendering: [],
|
|
tagRenderings: [],
|
|
filter: [],
|
|
source: "special:stub",
|
|
allowMove: true
|
|
})
|
|
continue
|
|
}
|
|
// Make sure all are direct dependencies are loaded
|
|
if (!this._loadedIds.has(someDep)) {
|
|
this.buildLayer(someDep, context)
|
|
}
|
|
}
|
|
while (ids.length > 0) {
|
|
const first = ids.pop()
|
|
if (first === "questions") {
|
|
continue
|
|
}
|
|
const oldConfig = this._desugaringState.sharedLayers.get(first) ?? this._layerConfigJsons.get(first)
|
|
const newConfig = this.buildLayer(first, context.inOperation("resolving a looped dependency"), true)
|
|
const isDifferent = JSON.stringify(oldConfig) !== JSON.stringify(newConfig)
|
|
|
|
if (isDifferent) {
|
|
const toRunAgain = revDeps[first] ?? []
|
|
for (const id of toRunAgain) {
|
|
if (ids.indexOf(id) < 0) {
|
|
ids.push(id)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for (const id of origIds) {
|
|
this.writeLayer(this._desugaringState.sharedLayers.get(id))
|
|
}
|
|
console.log("Done with the looping layers!")
|
|
}
|
|
|
|
public convert(o, context: ConversionContext):
|
|
Map<string, LayerConfigJson> {
|
|
|
|
for (const level of this._levels
|
|
) {
|
|
if (level.loop) {
|
|
this.buildLooping(level.ids, context)
|
|
continue
|
|
}
|
|
|
|
for (let i = 0; i < level.ids.length; i++) {
|
|
const id = level.ids[i]
|
|
ScriptUtils.erasableLog(`Building level ${level.level}: validating layer ${i + 1}/${level.ids.length}: ${id}`)
|
|
if (id === "questions") {
|
|
continue
|
|
}
|
|
if (this._states.get(id) === "clean") {
|
|
const file = readFileSync(LayerBuilder.targetPath(id), "utf-8")
|
|
if (file.length > 3) {
|
|
try {
|
|
const loaded = JSON.parse(file)
|
|
this._desugaringState.sharedLayers.set(id, loaded)
|
|
continue
|
|
} catch (e) {
|
|
console.error("Could not load generated layer file for ", id, " building it instead")
|
|
}
|
|
}
|
|
}
|
|
const prepped = this.buildLayer(id, context)
|
|
this.writeLayer(prepped)
|
|
LayerOverviewUtils.extractJavascriptCodeForLayer(prepped)
|
|
}
|
|
}
|
|
context.info("Recompiled " + this._loadedIds.size + " layers")
|
|
return this._desugaringState.sharedLayers
|
|
}
|
|
|
|
|
|
}
|
|
|
|
class LayerOverviewUtils extends Script {
|
|
public static readonly layerPath = "./public/assets/generated/layers/"
|
|
public static readonly themePath = "./public/assets/generated/themes/"
|
|
|
|
constructor() {
|
|
super("Reviews and generates the compiled themes")
|
|
}
|
|
|
|
private static publicLayerIdsFrom(themefiles: ThemeConfigJson[]): Set<string> {
|
|
const publicThemes = [].concat(...themefiles.filter((th) => !th.hideFromOverview))
|
|
|
|
return new Set([].concat(...publicThemes.map((th) => this.extractLayerIdsFrom(th))))
|
|
}
|
|
|
|
private static extractLayerIdsFrom(
|
|
themeFile: ThemeConfigJson,
|
|
includeInlineLayers = true
|
|
): string[] {
|
|
const publicLayerIds: string[] = []
|
|
if (!Array.isArray(themeFile.layers)) {
|
|
throw (
|
|
"Cannot iterate over 'layers' of " +
|
|
themeFile.id +
|
|
"; it is a " +
|
|
typeof themeFile.layers
|
|
)
|
|
}
|
|
for (const publicLayer of themeFile.layers) {
|
|
if (typeof publicLayer === "string") {
|
|
publicLayerIds.push(publicLayer)
|
|
continue
|
|
}
|
|
if (publicLayer["builtin"] !== undefined) {
|
|
const bi: string | string[] = publicLayer["builtin"]
|
|
if (typeof bi === "string") {
|
|
publicLayerIds.push(bi)
|
|
} else {
|
|
bi.forEach((id) => publicLayerIds.push(id))
|
|
}
|
|
continue
|
|
}
|
|
if (includeInlineLayers) {
|
|
publicLayerIds.push(publicLayer["id"])
|
|
}
|
|
}
|
|
return publicLayerIds
|
|
}
|
|
|
|
public static cleanTranslation(t: string | Record<string, string> | Translation): Translatable {
|
|
return Translations.T(t).OnEveryLanguage((s) => parse_html(s).textContent).translations
|
|
}
|
|
|
|
public static shouldBeUpdated(sourcefile: string | string[], targetfile: string): boolean {
|
|
if (!existsSync(targetfile)) {
|
|
return true
|
|
}
|
|
const targetModified = statSync(targetfile).mtime
|
|
if (typeof sourcefile === "string") {
|
|
sourcefile = [sourcefile]
|
|
}
|
|
|
|
for (const path of sourcefile) {
|
|
const hasChange = statSync(path).mtime > targetModified
|
|
if (hasChange) {
|
|
console.log("File ", targetfile, " should be updated as ", path, "has been changed")
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
static mergeKeywords(
|
|
into: Record<string, string[]>,
|
|
source: Readonly<Record<string, string[]>>
|
|
) {
|
|
for (const key in source) {
|
|
if (into[key]) {
|
|
into[key].push(...source[key])
|
|
} else {
|
|
into[key] = source[key]
|
|
}
|
|
}
|
|
}
|
|
|
|
private layerKeywords(l: LayerConfigJson): Record<string, string[]> {
|
|
const keywords: Record<string, string[]> = {}
|
|
|
|
function addWord(language: string, word: string | string[]) {
|
|
if (Array.isArray(word)) {
|
|
word.forEach((w) => addWord(language, w))
|
|
return
|
|
}
|
|
|
|
word = Utils.SubstituteKeys(word, {})?.trim()
|
|
if (!word) {
|
|
return
|
|
}
|
|
if (!keywords[language]) {
|
|
keywords[language] = []
|
|
}
|
|
keywords[language].push(word)
|
|
}
|
|
|
|
function addWords(
|
|
tr: string | Record<string, string> | Record<string, string[]> | TagRenderingConfigJson
|
|
) {
|
|
if (!tr) {
|
|
return
|
|
}
|
|
if (typeof tr === "string") {
|
|
addWord("*", tr)
|
|
return
|
|
}
|
|
if (tr["render"] !== undefined || tr["mappings"] !== undefined) {
|
|
tr = <TagRenderingConfigJson>tr
|
|
addWords(<Translatable>tr.render)
|
|
for (const mapping of tr.mappings ?? []) {
|
|
if (typeof mapping === "string") {
|
|
addWords(mapping)
|
|
continue
|
|
}
|
|
addWords(mapping.then)
|
|
}
|
|
return
|
|
}
|
|
for (const lang in tr) {
|
|
addWord(lang, tr[lang])
|
|
}
|
|
}
|
|
|
|
addWord("*", l.id)
|
|
addWords(l.title)
|
|
addWords(l.description)
|
|
addWords(l.searchTerms)
|
|
return keywords
|
|
}
|
|
|
|
writeSmallOverview(
|
|
themes: {
|
|
id: string
|
|
title: Translatable
|
|
shortDescription: Translatable
|
|
icon: string
|
|
hideFromOverview: boolean
|
|
mustHaveLanguage: boolean
|
|
layers: (
|
|
| LayerConfigJson
|
|
| string
|
|
| {
|
|
builtin
|
|
}
|
|
)[]
|
|
}[],
|
|
sharedLayers: Map<string, LayerConfigJson>
|
|
) {
|
|
const layerKeywords: Record<string, Record<string, string[]>> = {}
|
|
|
|
sharedLayers.forEach((layer, id) => {
|
|
layerKeywords[id] = this.layerKeywords(layer)
|
|
})
|
|
|
|
const perId = new Map<string, MinimalThemeInformation>()
|
|
for (const theme of themes) {
|
|
const keywords: Record<string, string[]> = {}
|
|
for (const layer of theme.layers ?? []) {
|
|
const l = <LayerConfigJson>layer
|
|
if (sharedLayers.has(l.id)) {
|
|
continue
|
|
}
|
|
LayerOverviewUtils.mergeKeywords(keywords, this.layerKeywords(l))
|
|
}
|
|
|
|
const data = <MinimalThemeInformation>{
|
|
id: theme.id,
|
|
title: theme.title,
|
|
shortDescription: LayerOverviewUtils.cleanTranslation(theme.shortDescription),
|
|
icon: theme.icon,
|
|
hideFromOverview: theme.hideFromOverview,
|
|
mustHaveLanguage: theme.mustHaveLanguage,
|
|
keywords,
|
|
layers: (<LayerConfigJson[]>theme.layers)
|
|
.filter((l) => sharedLayers.has(l.id))
|
|
.filter((l) => l.minzoom < 17)
|
|
.map((l) => l.id),
|
|
}
|
|
perId.set(data.id, data)
|
|
}
|
|
|
|
const sorted = Constants.themeOrder.map((id) => {
|
|
if (!perId.has(id)) {
|
|
throw "Ordered theme '" + id + "' not found"
|
|
}
|
|
return perId.get(id)
|
|
})
|
|
|
|
perId.forEach((value) => {
|
|
if (Constants.themeOrder.indexOf(value.id) >= 0) {
|
|
return // actually a continue
|
|
}
|
|
sorted.push(value)
|
|
})
|
|
|
|
writeFileSync(
|
|
"./src/assets/generated/theme_overview.json",
|
|
JSON.stringify({ layers: layerKeywords, themes: sorted }, null, " "),
|
|
{ encoding: "utf8" }
|
|
)
|
|
}
|
|
|
|
writeTheme(theme: ThemeConfigJson) {
|
|
if (!existsSync(LayerOverviewUtils.themePath)) {
|
|
mkdirSync(LayerOverviewUtils.themePath)
|
|
}
|
|
|
|
writeFileSync(
|
|
`${LayerOverviewUtils.themePath}${theme.id}.json`,
|
|
JSON.stringify(theme, null, " "),
|
|
{ encoding: "utf8" }
|
|
)
|
|
}
|
|
|
|
static asDict(
|
|
trs: QuestionableTagRenderingConfigJson[]
|
|
): Map<string, QuestionableTagRenderingConfigJson> {
|
|
const d = new Map<string, QuestionableTagRenderingConfigJson>()
|
|
for (const tr of trs) {
|
|
d.set(tr.id, tr)
|
|
}
|
|
return d
|
|
}
|
|
|
|
getSharedTagRenderings(doesImageExist: DoesImageExist): QuestionableTagRenderingConfigJson[]
|
|
getSharedTagRenderings(
|
|
doesImageExist: DoesImageExist,
|
|
bootstrapTagRenderings: Map<string, QuestionableTagRenderingConfigJson>,
|
|
bootstrapTagRenderingsOrder: string[]
|
|
): QuestionableTagRenderingConfigJson[]
|
|
getSharedTagRenderings(
|
|
doesImageExist: DoesImageExist,
|
|
bootstrapTagRenderings: Map<string, QuestionableTagRenderingConfigJson> = null,
|
|
bootstrapTagRenderingsOrder: string[] = []
|
|
): QuestionableTagRenderingConfigJson[] {
|
|
const prepareLayer = new PrepareLayer(
|
|
{
|
|
tagRenderings: bootstrapTagRenderings,
|
|
tagRenderingOrder: bootstrapTagRenderingsOrder,
|
|
sharedLayers: null,
|
|
publicLayers: null,
|
|
},
|
|
{
|
|
addTagRenderingsToContext: true,
|
|
}
|
|
)
|
|
|
|
const path = "assets/layers/questions/questions.json"
|
|
const sharedQuestions = this.parseLayer(doesImageExist, prepareLayer, path).raw
|
|
|
|
const dict = new Map<string, QuestionableTagRenderingConfigJson>()
|
|
|
|
for (const tr of sharedQuestions.tagRenderings) {
|
|
const tagRendering = <QuestionableTagRenderingConfigJson>tr
|
|
tagRendering._definedIn = ["questions", tr["id"]]
|
|
dict.set(tagRendering["id"], tagRendering)
|
|
}
|
|
|
|
if (dict.size === bootstrapTagRenderings?.size) {
|
|
return <QuestionableTagRenderingConfigJson[]>sharedQuestions.tagRenderings
|
|
}
|
|
|
|
return this.getSharedTagRenderings(
|
|
doesImageExist,
|
|
dict,
|
|
sharedQuestions.tagRenderings.map((tr) => tr["id"])
|
|
)
|
|
}
|
|
|
|
checkAllSvgs() {
|
|
const allSvgs = ScriptUtils.readDirRecSync("./src/assets")
|
|
.filter((path) => path.endsWith(".svg"))
|
|
.filter((path) => !path.startsWith("./src/assets/generated"))
|
|
let errCount = 0
|
|
const exempt = [
|
|
"src/assets/SocialImageTemplate.svg",
|
|
"src/assets/SocialImageTemplateWide.svg",
|
|
"src/assets/SocialImageBanner.svg",
|
|
"src/assets/SocialImageRepo.svg",
|
|
"src/assets/svg/osm-logo.svg",
|
|
"src/assets/templates/*",
|
|
]
|
|
for (const path of allSvgs) {
|
|
if (
|
|
exempt.some((p) => {
|
|
if (p.endsWith("*") && path.startsWith("./" + p.substring(0, p.length - 1))) {
|
|
return true
|
|
}
|
|
return "./" + p === path
|
|
})
|
|
) {
|
|
continue
|
|
}
|
|
|
|
const contents = readFileSync(path, { encoding: "utf8" })
|
|
if (contents.indexOf("data:image/png;") >= 0) {
|
|
console.warn("The SVG at " + path + " is a fake SVG: it contains PNG data!")
|
|
errCount++
|
|
if (path.startsWith("./src/assets/svg")) {
|
|
throw "A core SVG is actually a PNG. Don't do this!"
|
|
}
|
|
}
|
|
if (contents.indexOf("<text") > 0) {
|
|
console.warn(
|
|
"The SVG at " +
|
|
path +
|
|
" contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path"
|
|
)
|
|
errCount++
|
|
}
|
|
}
|
|
if (errCount > 0) {
|
|
throw `There are ${errCount} invalid svgs`
|
|
}
|
|
}
|
|
|
|
async main(args: string[]) {
|
|
console.log("Generating layer overview...")
|
|
const themeWhitelist = new Set(
|
|
args
|
|
.find((a) => a.startsWith("--themes="))
|
|
?.substring("--themes=".length)
|
|
?.split(",") ?? []
|
|
)
|
|
|
|
const forceReload = args.some((a) => a == "--force")
|
|
|
|
const licensePaths = new Set<string>()
|
|
for (const i in licenses) {
|
|
licensePaths.add(licenses[i].path)
|
|
}
|
|
const doesImageExist = new DoesImageExist(licensePaths, existsSync)
|
|
const sharedLayers = this.buildLayerIndex(doesImageExist)
|
|
|
|
const priviliged = new Set<string>(Constants.priviliged_layers)
|
|
sharedLayers.forEach((_, key) => {
|
|
priviliged.delete(key)
|
|
})
|
|
|
|
// These get a free pass
|
|
priviliged.delete("summary")
|
|
priviliged.delete("last_click")
|
|
priviliged.delete("search")
|
|
|
|
const isBoostrapping = AllSharedLayers.getSharedLayersConfigs().size == 0
|
|
if (!isBoostrapping && priviliged.size > 0) {
|
|
throw (
|
|
"Priviliged layer " +
|
|
Array.from(priviliged).join(", ") +
|
|
" has no definition file, create it at `src/assets/layers/<layername>/<layername.json>"
|
|
)
|
|
}
|
|
const recompiledThemes: string[] = []
|
|
const sharedThemes = this.buildThemeIndex(
|
|
licensePaths,
|
|
sharedLayers,
|
|
recompiledThemes,
|
|
forceReload,
|
|
themeWhitelist
|
|
)
|
|
|
|
new ValidateThemeEnsemble().convertStrict(
|
|
Array.from(sharedThemes.values()).map((th) => new ThemeConfig(th, true))
|
|
)
|
|
|
|
if (recompiledThemes.length > 0) {
|
|
writeFileSync(
|
|
"./src/assets/generated/known_layers.json",
|
|
JSON.stringify({
|
|
layers: Array.from(sharedLayers.values()).filter(
|
|
(l) => !(l["#no-index"] === "yes")
|
|
),
|
|
})
|
|
)
|
|
}
|
|
|
|
const mcChangesPath = "./assets/themes/mapcomplete-changes/mapcomplete-changes.json"
|
|
if (
|
|
(recompiledThemes.length > 0 &&
|
|
!(
|
|
recompiledThemes.length === 1 && recompiledThemes[0] === "mapcomplete-changes"
|
|
)) ||
|
|
args.indexOf("--generate-change-map") >= 0 ||
|
|
!existsSync(mcChangesPath)
|
|
) {
|
|
// mapcomplete-changes shows an icon for each corresponding mapcomplete-theme
|
|
const iconsPerTheme = Array.from(sharedThemes.values()).map((th) => ({
|
|
if: "theme=" + th.id,
|
|
then: th.icon,
|
|
}))
|
|
const proto: ThemeConfigJson = JSON.parse(
|
|
readFileSync("./assets/themes/mapcomplete-changes/mapcomplete-changes.proto.json", {
|
|
encoding: "utf8",
|
|
})
|
|
)
|
|
const protolayer = <LayerConfigJson>(
|
|
proto.layers.filter((l) => l["id"] === "mapcomplete-changes")[0]
|
|
)
|
|
const rendering = protolayer.pointRendering[0]
|
|
rendering.marker[0].icon["mappings"] = iconsPerTheme
|
|
writeFileSync(mcChangesPath, JSON.stringify(proto, null, " "))
|
|
}
|
|
|
|
this.checkAllSvgs()
|
|
|
|
new DetectDuplicateFilters().convertStrict(
|
|
{
|
|
layers: ScriptUtils.getLayerFiles().map((f) => f.parsed),
|
|
themes: ScriptUtils.getThemeFiles().map((f) => f.parsed),
|
|
},
|
|
ConversionContext.construct([], [])
|
|
)
|
|
|
|
for (const [_, theme] of sharedThemes) {
|
|
theme.layers = theme.layers.filter(
|
|
(l) => Constants.added_by_default.indexOf(l["id"]) < 0
|
|
)
|
|
}
|
|
|
|
}
|
|
|
|
private parseLayer(
|
|
doesImageExist: DoesImageExist,
|
|
prepLayer: PrepareLayer,
|
|
sharedLayerPath: string
|
|
): {
|
|
raw: LayerConfigJson
|
|
parsed: LayerConfig
|
|
context: ConversionContext
|
|
} {
|
|
const parser = new ParseLayer(prepLayer, doesImageExist)
|
|
const context = ConversionContext.construct([sharedLayerPath], ["ParseLayer"])
|
|
const parsed = parser.convertStrict(sharedLayerPath, context)
|
|
const result = AddIconSummary.singleton.convertStrict(
|
|
parsed,
|
|
context.inOperation("AddIconSummary")
|
|
)
|
|
return { ...result, context }
|
|
}
|
|
|
|
private getAllLayerConfigs(): LayerConfigJson[] {
|
|
const allPaths = ScriptUtils.getLayerPaths()
|
|
const results: LayerConfigJson[] = []
|
|
for (let i = 0; i < allPaths.length; i++) {
|
|
const path = allPaths[i]
|
|
ScriptUtils.erasableLog(`Parsing layerConfig ${i + 1}/${allPaths.length}: ${path} `)
|
|
const data = JSON.parse(readFileSync(path, "utf8"))
|
|
results.push(data)
|
|
}
|
|
|
|
|
|
return results
|
|
|
|
}
|
|
|
|
private buildLayerIndex(
|
|
doesImageExist: DoesImageExist
|
|
): Map<string, LayerConfigJson> {
|
|
// First, we expand and validate all builtin layers. These are written to src/assets/generated/layers
|
|
// At the same time, an index of available layers is built.
|
|
const sharedQuestions = this.getSharedTagRenderings(doesImageExist)
|
|
const allLayerConfigs = this.getAllLayerConfigs()
|
|
const sharedQuestionsDef = allLayerConfigs.find(l => l.id === "questions")
|
|
sharedQuestionsDef.tagRenderings = sharedQuestions
|
|
|
|
|
|
const dependencyGraph = LayerConfigDependencyGraph.buildDirectDependencies(allLayerConfigs)
|
|
const levels = LayerConfigDependencyGraph.buildLevels(dependencyGraph)
|
|
const layerState = new Map<string, "clean" | "dirty" | "changed">()
|
|
console.log("# BUILD PLAN\n\n")
|
|
for (const levelInfo of levels) {
|
|
console.log(`## LEVEL ${levelInfo.level}${levelInfo.loop ? " (LOOP)" : ""}`)
|
|
for (const id of levelInfo.ids) {
|
|
const deps = dependencyGraph.get(id) ?? []
|
|
const dirtyDeps = deps.filter(dep => {
|
|
const depState = layerState.get(dep)
|
|
if (levelInfo.loop && depState === undefined) {
|
|
const depIsClean =
|
|
LayerOverviewUtils.shouldBeUpdated(
|
|
LayerBuilder.sourcePath(dep),
|
|
LayerBuilder.targetPath(dep))
|
|
if (depIsClean) {
|
|
return false
|
|
}
|
|
}
|
|
return depState !== "clean"
|
|
})
|
|
if (dirtyDeps.length > 0) {
|
|
layerState.set(id, "dirty")
|
|
} else {
|
|
|
|
|
|
const sourcePath = `./assets/layers/${id}/${id}.json`
|
|
const targetPath = `./public/assets/generated/layers/${id}.json`
|
|
|
|
if (id === "questions") {
|
|
layerState.set(id, "clean")
|
|
} else if (LayerOverviewUtils.shouldBeUpdated(sourcePath, targetPath)) {
|
|
layerState.set(id, "changed")
|
|
} else {
|
|
layerState.set(id, "clean")
|
|
}
|
|
}
|
|
const state = layerState.get(id)
|
|
console.log(`- ${id} (${state}; ${dirtyDeps.map(dd => dd + "*").join(", ")})`)
|
|
}
|
|
}
|
|
|
|
|
|
const builder = new LayerBuilder(allLayerConfigs, dependencyGraph, levels, layerState, sharedQuestions)
|
|
builder.writeLayer(sharedQuestionsDef)
|
|
const allLayers = builder.convertStrict({}, ConversionContext.construct([], []))
|
|
if (layerState.get("usersettings") !== "clean") {
|
|
// We always need the calculated tags of 'usersettings', so we export them separately if dirty
|
|
|
|
LayerOverviewUtils.extractJavascriptCodeForLayer(
|
|
allLayers.get("usersettings"),
|
|
"./src/Logic/State/UserSettingsMetaTagging.ts"
|
|
)
|
|
}
|
|
|
|
return allLayers
|
|
|
|
}
|
|
|
|
/**
|
|
* Given: a fully expanded themeConfigJson
|
|
*
|
|
* Will extract a dictionary of the special code and write it into a javascript file which can be imported.
|
|
* This removes the need for _eval_, allowing for a correct CSP
|
|
* @param themeFile
|
|
* @private
|
|
*/
|
|
private extractJavascriptCode(themeFile: ThemeConfigJson) {
|
|
const allCode = [
|
|
"import {Feature} from 'geojson'",
|
|
'import { ExtraFuncType } from "../../../Logic/ExtraFunctions";',
|
|
'import { Utils } from "../../../Utils"',
|
|
"export class ThemeMetaTagging {",
|
|
" public static readonly themeName = " + JSON.stringify(themeFile.id),
|
|
"",
|
|
]
|
|
for (const layer of themeFile.layers) {
|
|
const l = <LayerConfigJson>layer
|
|
const id = l.id.replace(/[^a-zA-Z0-9_]/g, "_")
|
|
const code = l.calculatedTags ?? []
|
|
|
|
allCode.push(
|
|
" public metaTaggging_for_" +
|
|
id +
|
|
"(feat: Feature, helperFunctions: Record<ExtraFuncType, (feature: Feature) => Function>) {"
|
|
)
|
|
allCode.push(" const {" + ExtraFunctions.types.join(", ") + "} = helperFunctions")
|
|
for (const line of code) {
|
|
const firstEq = line.indexOf("=")
|
|
let attributeName = line.substring(0, firstEq).trim()
|
|
const expression = line.substring(firstEq + 1)
|
|
const isStrict = attributeName.endsWith(":")
|
|
if (!isStrict) {
|
|
allCode.push(
|
|
" Utils.AddLazyProperty(feat.properties, '" +
|
|
attributeName +
|
|
"', () => " +
|
|
expression +
|
|
" ) "
|
|
)
|
|
} else {
|
|
attributeName = attributeName.substring(0, attributeName.length - 1).trim()
|
|
allCode.push(" feat.properties['" + attributeName + "'] = " + expression)
|
|
}
|
|
}
|
|
allCode.push(" }")
|
|
}
|
|
|
|
const targetDir = "./src/assets/generated/metatagging/"
|
|
if (!existsSync(targetDir)) {
|
|
mkdirSync(targetDir)
|
|
}
|
|
allCode.push("}")
|
|
|
|
writeFileSync(targetDir + themeFile.id + ".ts", allCode.join("\n"))
|
|
}
|
|
|
|
public static extractJavascriptCodeForLayer(l: LayerConfigJson, targetPath?: string) {
|
|
if (!l) {
|
|
return // Probably a bootstrapping run
|
|
}
|
|
let importPath = "../../../"
|
|
if (targetPath) {
|
|
const l = targetPath.split("/")
|
|
if (l.length == 1) {
|
|
importPath = "./"
|
|
} else {
|
|
importPath = ""
|
|
for (let i = 0; i < l.length - 3; i++) {
|
|
importPath += "../"
|
|
}
|
|
}
|
|
}
|
|
const allCode = [
|
|
`import { Utils } from "${importPath}Utils"`,
|
|
`/** This code is autogenerated - do not edit. Edit ./assets/layers/${l?.id}/${l?.id}.json instead */`,
|
|
"export class ThemeMetaTagging {",
|
|
" public static readonly themeName = " + JSON.stringify(l.id),
|
|
"",
|
|
]
|
|
const code = l.calculatedTags ?? []
|
|
|
|
allCode.push(
|
|
" public metaTaggging_for_" + l.id + "(feat: {properties: Record<string, string>}) {"
|
|
)
|
|
for (const line of code) {
|
|
const firstEq = line.indexOf("=")
|
|
let attributeName = line.substring(0, firstEq).trim()
|
|
const expression = line.substring(firstEq + 1)
|
|
const isStrict = attributeName.endsWith(":")
|
|
if (!isStrict) {
|
|
allCode.push(
|
|
" Utils.AddLazyProperty(feat.properties, '" +
|
|
attributeName +
|
|
"', () => " +
|
|
expression +
|
|
" ) "
|
|
)
|
|
} else {
|
|
attributeName = attributeName.substring(0, attributeName.length - 2).trim()
|
|
allCode.push(" feat.properties['" + attributeName + "'] = " + expression)
|
|
}
|
|
}
|
|
allCode.push(" }")
|
|
allCode.push("}")
|
|
|
|
const targetDir = "./src/assets/generated/metatagging/"
|
|
if (!targetPath) {
|
|
if (!existsSync(targetDir)) {
|
|
mkdirSync(targetDir)
|
|
}
|
|
}
|
|
|
|
writeFileSync(targetPath ?? targetDir + "layer_" + l.id + ".ts", allCode.join("\n"))
|
|
}
|
|
|
|
private buildThemeIndex(
|
|
licensePaths: Set<string>,
|
|
sharedLayers: Map<string, LayerConfigJson>,
|
|
recompiledThemes: string[],
|
|
forceReload: boolean,
|
|
whitelist: Set<string>
|
|
): Map<string, ThemeConfigJson> {
|
|
console.log(" ---------- VALIDATING BUILTIN THEMES ---------")
|
|
const themeFiles = ScriptUtils.getThemeFiles()
|
|
const fixed = new Map<string, ThemeConfigJson>()
|
|
|
|
const publicLayers = LayerOverviewUtils.publicLayerIdsFrom(
|
|
themeFiles.map((th) => th.parsed)
|
|
)
|
|
|
|
const trs = this.getSharedTagRenderings(new DoesImageExist(licensePaths, existsSync))
|
|
|
|
const convertState: DesugaringContext = {
|
|
sharedLayers,
|
|
tagRenderings: LayerOverviewUtils.asDict(trs),
|
|
tagRenderingOrder: trs.map((tr) => tr.id),
|
|
publicLayers,
|
|
}
|
|
const knownTagRenderings = new Set<string>()
|
|
convertState.tagRenderings.forEach((_, key) => knownTagRenderings.add(key))
|
|
sharedLayers.forEach((layer) => {
|
|
for (const tagRendering of layer.tagRenderings ?? []) {
|
|
if (tagRendering["id"]) {
|
|
knownTagRenderings.add(layer.id + "." + tagRendering["id"])
|
|
}
|
|
if (tagRendering["labels"]) {
|
|
for (const label of tagRendering["labels"]) {
|
|
knownTagRenderings.add(layer.id + "." + label)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
const skippedThemes: string[] = []
|
|
|
|
for (let i = 0; i < themeFiles.length; i++) {
|
|
const themeInfo = themeFiles[i]
|
|
const themePath = themeInfo.path
|
|
let themeFile = themeInfo.parsed
|
|
if (!themeFile) {
|
|
throw "Got an empty file for" + themeInfo.path
|
|
}
|
|
if (whitelist.size > 0 && !whitelist.has(themeFile.id)) {
|
|
continue
|
|
}
|
|
|
|
const targetPath =
|
|
LayerOverviewUtils.themePath + "/" + themePath.substring(themePath.lastIndexOf("/"))
|
|
|
|
const usedLayers = Array.from(
|
|
LayerOverviewUtils.extractLayerIdsFrom(themeFile, false)
|
|
).map((id) => LayerOverviewUtils.layerPath + id + ".json")
|
|
|
|
if (!forceReload && !LayerOverviewUtils.shouldBeUpdated([themePath, ...usedLayers], targetPath)) {
|
|
fixed.set(
|
|
themeFile.id,
|
|
JSON.parse(
|
|
readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", "utf8")
|
|
)
|
|
)
|
|
ScriptUtils.erasableLog("Skipping", themeFile.id)
|
|
skippedThemes.push(themeFile.id)
|
|
continue
|
|
}
|
|
|
|
recompiledThemes.push(themeFile.id)
|
|
|
|
new PrevalidateTheme().convertStrict(
|
|
themeFile,
|
|
ConversionContext.construct([themePath], ["PrepareLayer"])
|
|
)
|
|
try {
|
|
themeFile = new PrepareTheme(convertState, {
|
|
skipDefaultLayers: true,
|
|
}).convertStrict(
|
|
themeFile,
|
|
ConversionContext.construct([themePath], ["PrepareLayer"])
|
|
)
|
|
new ValidateThemeAndLayers(
|
|
new DoesImageExist(licensePaths, existsSync, knownTagRenderings),
|
|
themePath,
|
|
true,
|
|
knownTagRenderings
|
|
).convertStrict(
|
|
themeFile,
|
|
ConversionContext.construct([themePath], ["PrepareLayer"])
|
|
)
|
|
|
|
if (themeFile.icon.endsWith(".svg")) {
|
|
try {
|
|
ScriptUtils.ReadSvgSync(themeFile.icon, (svg) => {
|
|
const width: string = svg["$"].width
|
|
if (width === undefined) {
|
|
throw (
|
|
"The logo at " +
|
|
themeFile.icon +
|
|
" does not have a defined width"
|
|
)
|
|
}
|
|
const height: string = svg["$"].height
|
|
const err = themeFile.hideFromOverview ? console.warn : console.error
|
|
if (width !== height) {
|
|
const e =
|
|
`the icon for theme ${themeFile.id} is not square. Please square the icon at ${themeFile.icon}` +
|
|
` Width = ${width} height = ${height}`
|
|
err(e)
|
|
}
|
|
|
|
if (width?.endsWith("%")) {
|
|
throw (
|
|
"The logo at " +
|
|
themeFile.icon +
|
|
" has a relative width; this is not supported"
|
|
)
|
|
}
|
|
|
|
const w = parseInt(width)
|
|
const h = parseInt(height)
|
|
if (w < 370 || h < 370) {
|
|
const e: string = [
|
|
`the icon for theme ${themeFile.id} is too small. Please rescale the icon at ${themeFile.icon}`,
|
|
`Even though an SVG is 'infinitely scaleable', the icon should be dimensioned bigger. One of the build steps of the theme does convert the image to a PNG (to serve as PWA-icon) and having a small dimension will cause blurry images.`,
|
|
` Width = ${width} height = ${height}; we recommend a size of at least 500px * 500px and to use a square aspect ratio.`,
|
|
].join("\n")
|
|
err(e)
|
|
}
|
|
})
|
|
} catch (e) {
|
|
console.error("Could not read " + themeFile.icon + " due to " + e)
|
|
}
|
|
}
|
|
|
|
const usedImages = Utils.Dedup(
|
|
new ExtractImages(true, knownTagRenderings)
|
|
.convertStrict(themeFile)
|
|
.map((x) => x.path)
|
|
)
|
|
usedImages.sort()
|
|
|
|
themeFile["_usedImages"] = usedImages
|
|
|
|
this.writeTheme(themeFile)
|
|
fixed.set(themeFile.id, themeFile)
|
|
|
|
this.extractJavascriptCode(themeFile)
|
|
} catch (e) {
|
|
console.error("ERROR: could not prepare theme " + themePath + " due to " + e)
|
|
throw e
|
|
}
|
|
}
|
|
|
|
if (whitelist.size == 0) {
|
|
this.writeSmallOverview(
|
|
Array.from(fixed.values()).map((t) => {
|
|
return <any>{
|
|
...t,
|
|
hideFromOverview: t.hideFromOverview ?? false,
|
|
shortDescription:
|
|
t.shortDescription ?? new Translation(t.description).FirstSentence(),
|
|
mustHaveLanguage: t.mustHaveLanguage?.length > 0,
|
|
}
|
|
}),
|
|
sharedLayers
|
|
)
|
|
}
|
|
|
|
console.log(
|
|
"Recompiled themes " +
|
|
recompiledThemes.join(", ") +
|
|
" and skipped " +
|
|
skippedThemes.length +
|
|
" themes"
|
|
)
|
|
|
|
return fixed
|
|
}
|
|
}
|
|
|
|
new GenerateFavouritesLayer().run()
|
|
new LayerOverviewUtils().run()
|