ilja_space/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Refactoring: finish move to 'src' directory

Browse source
This commit is contained in:
Pieter Vander Vennet 2023-07-15 18:04:30 +02:00
parent e75d2789d2
commit 1bba106a86
158 changed files with 7715 additions and 7241 deletions
Download patch file Download diff file Expand all files Collapse all files

6
scripts/ScriptUtils.ts
View file

@ -1,9 +1,9 @@
import * as fs from "fs"
import { existsSync, lstatSync, readdirSync, readFileSync } from "fs"
import { Utils } from "../Utils"
import { Utils } from "../src/Utils"
import * as https from "https"
import { LayoutConfigJson } from "../Models/ThemeConfig/Json/LayoutConfigJson"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import { LayoutConfigJson } from "../src/Models/ThemeConfig/Json/LayoutConfigJson"
import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
import xml2js from "xml2js"
export default class ScriptUtils {

5
scripts/build.sh
View file

@ -13,7 +13,7 @@ export NODE_OPTIONS="--max-old-space-size=8192"
# This script ends every line with '&&' to chain everything. A failure will thus stop the build
npm run generate:editor-layer-index &&
npm run generate &&
# npm run generate &&
npm run generate:layouts
if [ $? -ne 0 ]; then
@ -51,8 +51,5 @@ vite build $SRC_MAPS
cp -r assets/layers/ dist/assets/layers/
cp -r assets/themes/ dist/assets/themes/
cp -r assets/svg/ dist/assets/svg/
cp -r assets/tagRenderings/ dist/assets/tagRenderings/
cp assets/*.png dist/assets/
cp assets/*.svg dist/assets/
export NODE_OPTIONS=""

4
scripts/downloadEli.ts
View file

@ -1,5 +1,5 @@
import Script from "./Script"
import { Utils } from "../Utils"
import { Utils } from "../src/Utils"
import { FeatureCollection } from "geojson"
import fs from "fs"
@ -10,7 +10,7 @@ class DownloadEli extends Script {
async main(args: string[]): Promise<void> {
const url = "https://osmlab.github.io/editor-layer-index/imagery.geojson"
// Target should use '.json' instead of '.geojson', as the latter cannot be imported by the build systems
const target = args[0] ?? "assets/editor-layer-index.json"
const target = args[0] ?? "src/assets/editor-layer-index.json"
const eli = <FeatureCollection>await Utils.downloadJson(url)
const keptLayers = []

21
scripts/fetchLanguages.ts
View file

@ -4,12 +4,12 @@
*/
import * as wds from "wikidata-sdk"
import { Utils } from "../Utils"
import { Utils } from "../src/Utils"
import ScriptUtils from "./ScriptUtils"
import { existsSync, readFileSync, writeFileSync } from "fs"
import WikidataUtils from "../Utils/WikidataUtils"
import LanguageUtils from "../Utils/LanguageUtils"
import Wikidata from "../Logic/Web/Wikidata"
import WikidataUtils from "../src/Utils/WikidataUtils"
import LanguageUtils from "../src/Utils/LanguageUtils"
import Wikidata from "../src/Logic/Web/Wikidata"
interface value<T> {
value: T
@ -150,8 +150,8 @@ async function getOfficialLanguagesPerCountryCached(
wipeCache: boolean
): Promise<Record<string /*Country code*/, string[] /*Language codes*/>> {
let officialLanguages: Record<string, string[]>
const officialLanguagesPath = "./assets/language_in_country.json"
if (existsSync("./assets/languages_in_country.json") && !wipeCache) {
const officialLanguagesPath = "./src/assets/language_in_country.json"
if (existsSync("./src/assets/languages_in_country.json") && !wipeCache) {
officialLanguages = JSON.parse(readFileSync(officialLanguagesPath, "utf8"))
} else {
officialLanguages = Utils.MapToObj(await getOfficialLanguagesPerCountry(), (t) => t)
@ -161,7 +161,7 @@ async function getOfficialLanguagesPerCountryCached(
}
async function main(wipeCache = false) {
const cacheFile = "./assets/generated/languages-wd.json"
const cacheFile = "./src/assets/generated/languages-wd.json"
if (wipeCache || !existsSync(cacheFile)) {
console.log("Refreshing cache")
await fetch(cacheFile)
@ -172,7 +172,7 @@ async function main(wipeCache = false) {
const data = JSON.parse(readFileSync(cacheFile, { encoding: "utf8" }))
const perId = WikidataUtils.extractLanguageData(data, WikidataUtils.languageRemapping)
const nativeList = getNativeList(perId)
writeFileSync("./assets/language_native.json", JSON.stringify(nativeList, null, " "))
writeFileSync("./src/assets/language_native.json", JSON.stringify(nativeList, null, " "))
const languagesPerCountry = Utils.TransposeMap(
await getOfficialLanguagesPerCountryCached(wipeCache)
)
@ -195,7 +195,10 @@ async function main(wipeCache = false) {
return translatedForId
})
writeFileSync("./assets/language_translations.json", JSON.stringify(translations, null, " "))
writeFileSync(
"./src/assets/language_translations.json",
JSON.stringify(translations, null, " ")
)
}
const forceRefresh = process.argv[2] === "--force-refresh"

4
scripts/generateContributors.ts
View file

@ -49,9 +49,9 @@ function main() {
hist.set(author, 1 + (hist.get(author) ?? 0))
}
const codeContributorsTarget = "assets/contributors.json"
const codeContributorsTarget = "src/assets/contributors.json"
writeFileSync(codeContributorsTarget, JSON.stringify(asList(codeContributors), null, " "))
const translatorsTarget = "assets/translators.json"
const translatorsTarget = "src/assets/translators.json"
writeFileSync(
translatorsTarget,
JSON.stringify(asList(translationContributors), null, " ")

177
scripts/generateLayerOverview.ts
View file

@ -1,38 +1,37 @@
import ScriptUtils from "./ScriptUtils"
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "fs"
import licenses from "../assets/generated/license_info.json"
import { LayoutConfigJson } from "../Models/ThemeConfig/Json/LayoutConfigJson"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import Constants from "../Models/Constants"
import licenses from "../src/assets/generated/license_info.json"
import { LayoutConfigJson } from "../src/Models/ThemeConfig/Json/LayoutConfigJson"
import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
import Constants from "../src/Models/Constants"
import {
DetectDuplicateFilters,
DoesImageExist,
PrevalidateTheme,
ValidateLayer,
ValidateTagRenderings,
ValidateThemeAndLayers,
} from "../Models/ThemeConfig/Conversion/Validation"
import { Translation } from "../UI/i18n/Translation"
import { TagRenderingConfigJson } from "../Models/ThemeConfig/Json/TagRenderingConfigJson"
import questions from "../assets/tagRenderings/questions.json"
import PointRenderingConfigJson from "../Models/ThemeConfig/Json/PointRenderingConfigJson"
import { PrepareLayer, RewriteSpecial } from "../Models/ThemeConfig/Conversion/PrepareLayer"
import { PrepareTheme } from "../Models/ThemeConfig/Conversion/PrepareTheme"
import { DesugaringContext } from "../Models/ThemeConfig/Conversion/Conversion"
import { Utils } from "../Utils"
} from "../src/Models/ThemeConfig/Conversion/Validation"
import { Translation } from "../src/UI/i18n/Translation"
import { TagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/TagRenderingConfigJson"
import PointRenderingConfigJson from "../src/Models/ThemeConfig/Json/PointRenderingConfigJson"
import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer"
import { PrepareTheme } from "../src/Models/ThemeConfig/Conversion/PrepareTheme"
import { DesugaringContext } from "../src/Models/ThemeConfig/Conversion/Conversion"
import { Utils } from "../src/Utils"
import Script from "./Script"
import { AllSharedLayers } from "../Customizations/AllSharedLayers"
import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
// This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files.
// This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files.
// It spits out an overview of those to be used to load them
class LayerOverviewUtils extends Script {
public static readonly layerPath = "./assets/generated/layers/"
public static readonly themePath = "./assets/generated/themes/"
public static readonly layerPath = "./src/assets/generated/layers/"
public static readonly themePath = "./src/assets/generated/themes/"
constructor() {
super("Reviews and generates the compiled themes")
}
private static publicLayerIdsFrom(themefiles: LayoutConfigJson[]): Set<string> {
const publicThemes = [].concat(...themefiles.filter((th) => !th.hideFromOverview))
@ -125,7 +124,7 @@ class LayerOverviewUtils extends Script {
})
writeFileSync(
"./assets/generated/theme_overview.json",
"./src/assets/generated/theme_overview.json",
JSON.stringify(sorted, null, " "),
{ encoding: "utf8" }
)
@ -153,51 +152,45 @@ class LayerOverviewUtils extends Script {
)
}
getSharedTagRenderings(doesImageExist: DoesImageExist): Map<string, TagRenderingConfigJson> {
const dict = new Map<string, TagRenderingConfigJson>()
const prep = new RewriteSpecial()
const validator = new ValidateTagRenderings(undefined, doesImageExist)
for (const key in questions) {
if (key === "id") {
continue
}
questions[key].id = key
questions[key]["source"] = "shared-questions"
const config = prep.convertStrict(
<TagRenderingConfigJson>questions[key],
"questions.json:" + key
)
delete config["#"]
validator.convertStrict(
config,
"generate-layer-overview:tagRenderings/questions.json:" + key
)
dict.set(key, config)
}
dict.forEach((value, key) => {
if (key === "id") {
return
}
value.id = value.id ?? key
getSharedTagRenderings(
doesImageExist: DoesImageExist,
bootstrapTagRenderings: Map<string, TagRenderingConfigJson> = null
): Map<string, TagRenderingConfigJson> {
const prepareLayer = new PrepareLayer({
tagRenderings: bootstrapTagRenderings,
sharedLayers: null,
publicLayers: null,
})
return dict
let path = "assets/layers/questions/questions.json"
const sharedQuestions = this.parseLayer(doesImageExist, prepareLayer, path)
const dict = new Map<string, TagRenderingConfigJson>()
for (const tr of sharedQuestions.tagRenderings) {
const tagRendering = <TagRenderingConfigJson>tr
dict.set(tagRendering.id, tagRendering)
}
if (dict.size === bootstrapTagRenderings?.size) {
return dict
}
return this.getSharedTagRenderings(doesImageExist, dict)
}
checkAllSvgs() {
const allSvgs = ScriptUtils.readDirRecSync("./assets")
const allSvgs = ScriptUtils.readDirRecSync("./src/assets")
.filter((path) => path.endsWith(".svg"))
.filter((path) => !path.startsWith("./assets/generated"))
.filter((path) => !path.startsWith("./src/assets/generated"))
let errCount = 0
const exempt = [
"assets/SocialImageTemplate.svg",
"assets/SocialImageTemplateWide.svg",
"assets/SocialImageBanner.svg",
"assets/SocialImageRepo.svg",
"assets/svg/osm-logo.svg",
"assets/templates/*",
"src/assets/SocialImageTemplate.svg",
"src/assets/SocialImageTemplateWide.svg",
"src/assets/SocialImageBanner.svg",
"src/assets/SocialImageRepo.svg",
"src/assets/svg/osm-logo.svg",
"src/assets/templates/*",
]
for (const path of allSvgs) {
if (
@ -215,7 +208,7 @@ class LayerOverviewUtils extends Script {
if (contents.indexOf("data:image/png;") >= 0) {
console.warn("The SVG at " + path + " is a fake SVG: it contains PNG data!")
errCount++
if (path.startsWith("./assets/svg")) {
if (path.startsWith("./src/assets/svg")) {
throw "A core SVG is actually a PNG. Don't do this!"
}
}
@ -251,7 +244,7 @@ class LayerOverviewUtils extends Script {
throw (
"Priviliged layer " +
Array.from(priviliged).join(", ") +
" has no definition file, create it at `assets/layers/<layername>/<layername.json>"
" has no definition file, create it at `src/assets/layers/<layername>/<layername.json>"
)
}
const recompiledThemes: string[] = []
@ -263,14 +256,14 @@ class LayerOverviewUtils extends Script {
)
writeFileSync(
"./assets/generated/known_themes.json",
"./src/assets/generated/known_themes.json",
JSON.stringify({
themes: Array.from(sharedThemes.values()),
})
)
writeFileSync(
"./assets/generated/known_layers.json",
"./src/assets/generated/known_layers.json",
JSON.stringify({ layers: Array.from(sharedLayers.values()) })
)
@ -317,15 +310,49 @@ class LayerOverviewUtils extends Script {
}
}
private parseLayer(
doesImageExist: DoesImageExist,
prepLayer: PrepareLayer,
sharedLayerPath: string
): LayerConfigJson {
let parsed
try {
parsed = JSON.parse(readFileSync(sharedLayerPath, "utf8"))
} catch (e) {
throw "Could not parse or read file " + sharedLayerPath
}
const context = "While building builtin layer " + sharedLayerPath
const fixed = prepLayer.convertStrict(parsed, context)
if (!fixed.source) {
console.error(sharedLayerPath, "has no source configured:", fixed)
throw sharedLayerPath + " layer has no source configured"
}
if (
typeof fixed.source !== "string" &&
fixed.source["osmTags"] &&
fixed.source["osmTags"]["and"] === undefined
) {
fixed.source["osmTags"] = { and: [fixed.source["osmTags"]] }
}
const validator = new ValidateLayer(sharedLayerPath, true, doesImageExist)
validator.convertStrict(fixed, context)
return fixed
}
private buildLayerIndex(
doesImageExist: DoesImageExist,
forceReload: boolean
): Map<string, LayerConfigJson> {
// First, we expand and validate all builtin layers. These are written to assets/generated/layers
// First, we expand and validate all builtin layers. These are written to src/assets/generated/layers
// At the same time, an index of available layers is built.
console.log(" ---------- VALIDATING BUILTIN LAYERS ---------")
console.log("------------- VALIDATING THE BUILTIN QUESTIONS ---------------")
const sharedTagRenderings = this.getSharedTagRenderings(doesImageExist)
console.log("Shared questions are:", Array.from(sharedTagRenderings.keys()).join(", "))
console.log(" ---------- VALIDATING BUILTIN LAYERS ---------")
const state: DesugaringContext = {
tagRenderings: sharedTagRenderings,
sharedLayers: AllSharedLayers.getSharedLayersConfigs(),
@ -347,30 +374,8 @@ class LayerOverviewUtils extends Script {
continue
}
}
let parsed
try {
parsed = JSON.parse(readFileSync(sharedLayerPath, "utf8"))
} catch (e) {
throw "Could not parse or read file " + sharedLayerPath
}
const context = "While building builtin layer " + sharedLayerPath
const fixed = prepLayer.convertStrict(parsed, context)
if (!fixed.source) {
console.error(sharedLayerPath, "has no source configured:", fixed)
throw sharedLayerPath + " layer has no source configured"
}
if (
typeof fixed.source !== "string" &&
fixed.source["osmTags"] &&
fixed.source["osmTags"]["and"] === undefined
) {
fixed.source["osmTags"] = { and: [fixed.source["osmTags"]] }
}
const validator = new ValidateLayer(sharedLayerPath, true, doesImageExist)
validator.convertStrict(fixed, context)
const fixed = this.parseLayer(doesImageExist, prepLayer, sharedLayerPath)
if (sharedLayers.has(fixed.id)) {
throw "There are multiple layers with the id " + fixed.id

16
scripts/generateLayouts.ts
View file

@ -1,17 +1,17 @@
import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFile, writeFileSync } from "fs"
import Locale from "../UI/i18n/Locale"
import Translations from "../UI/i18n/Translations"
import { Translation } from "../UI/i18n/Translation"
import all_known_layouts from "../assets/generated/known_themes.json"
import { LayoutConfigJson } from "../Models/ThemeConfig/Json/LayoutConfigJson"
import LayoutConfig from "../Models/ThemeConfig/LayoutConfig"
import Locale from "../src/UI/i18n/Locale"
import Translations from "../src/UI/i18n/Translations"
import { Translation } from "../src/UI/i18n/Translation"
import all_known_layouts from "../src/assets/generated/known_themes.json"
import { LayoutConfigJson } from "../src/Models/ThemeConfig/Json/LayoutConfigJson"
import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig"
import xml2js from "xml2js"
import ScriptUtils from "./ScriptUtils"
import { Utils } from "../Utils"
import { Utils } from "../src/Utils"
const sharp = require("sharp")
const template = readFileSync("theme.html", "utf8")
const codeTemplate = readFileSync("index_theme.ts.template", "utf8")
const codeTemplate = readFileSync("src/index_theme.ts.template", "utf8")
function enc(str: string): string {
return encodeURIComponent(str.toLowerCase())

14
scripts/generateLicenseInfo.ts
View file

@ -1,5 +1,5 @@
import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
import SmallLicense from "../Models/smallLicense"
import SmallLicense from "../src/Models/smallLicense"
import ScriptUtils from "./ScriptUtils"
import Script from "./Script"
@ -280,7 +280,7 @@ export class GenerateLicenseInfo extends Script {
}
writeFileSync(
"./assets/generated/license_info.json",
"./src/assets/generated/license_info.json",
JSON.stringify(allLicenses, null, " ")
)
}
@ -288,13 +288,13 @@ export class GenerateLicenseInfo extends Script {
async main(args: string[]) {
console.log("Checking and compiling license info")
if (!existsSync("./assets/generated")) {
mkdirSync("./assets/generated")
if (!existsSync("./src/assets/generated")) {
mkdirSync("./src/assets/generated")
}
let contents = ScriptUtils.readDirRecSync("./assets")
.filter((p) => !p.startsWith("./assets/templates/"))
.filter((entry) => entry.indexOf("./assets/generated") != 0)
let contents = ScriptUtils.readDirRecSync("./assets").filter(
(entry) => entry.indexOf("./assets/generated") != 0
)
let licensePaths = contents.filter((entry) => entry.indexOf("license_info.json") >= 0)
let licenseInfos = this.generateLicenseInfos(licensePaths)

10
scripts/generateStats.ts
View file

@ -1,7 +1,7 @@
import known_layers from "../assets/generated/known_layers.json"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import { TagUtils } from "../Logic/Tags/TagUtils"
import { Utils } from "../Utils"
import known_layers from "../src/assets/generated/known_layers.json"
import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
import { TagUtils } from "../src/Logic/Tags/TagUtils"
import { Utils } from "../src/Utils"
import { writeFileSync } from "fs"
import ScriptUtils from "./ScriptUtils"
@ -65,7 +65,7 @@ async function main(includeTags = true) {
})
)
writeFileSync(
"./assets/key_totals.json",
"./src/assets/key_totals.json",
JSON.stringify(
{
keys: Utils.MapToObj(keyTotal, (t) => t),

33
scripts/generateTranslations.ts
View file

@ -1,6 +1,6 @@
import * as fs from "fs"
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
import { Utils } from "../Utils"
import { Utils } from "../src/Utils"
import ScriptUtils from "./ScriptUtils"
const knownLanguages = ["en", "nl", "de", "fr", "es", "gl", "ca"]
@ -460,8 +460,11 @@ function formatFile(path) {
* Generates the big compiledTranslations file
*/
function genTranslations() {
if (!fs.existsSync("./src/assets/generated/")) {
fs.mkdirSync("./src/assets/generated/")
}
const translations = JSON.parse(
fs.readFileSync("./assets/generated/translations.json", "utf-8")
fs.readFileSync("./src/assets/generated/translations.json", "utf-8")
)
const transformed = transformTranslation(translations)
@ -469,7 +472,7 @@ function genTranslations() {
module += " public static t = " + transformed
module += "\n }"
fs.writeFileSync("./assets/generated/CompiledTranslations.ts", module)
fs.writeFileSync("./src/assets/generated/CompiledTranslations.ts", module)
}
/**
@ -497,7 +500,7 @@ function compileTranslationsFromWeblate() {
}
writeFileSync(
"./assets/generated/translations.json",
"./src/assets/generated/translations.json",
JSON.stringify(JSON.parse(allTranslations.toJson()), null, " ")
)
}
@ -692,22 +695,9 @@ if (!existsSync("./langs/themes")) {
mkdirSync("./langs/themes")
}
const themeOverwritesWeblate = process.argv[2] === "--ignore-weblate"
const questionsPath = "assets/tagRenderings/questions.json"
const questionsParsed = JSON.parse(readFileSync(questionsPath, "utf8"))
if (!themeOverwritesWeblate) {
mergeLayerTranslations()
mergeThemeTranslations()
mergeLayerTranslation(
questionsParsed,
questionsPath,
loadTranslationFilesFrom("shared-questions")
)
const endsWithNewline = readFileSync(questionsPath, { encoding: "utf8" }).endsWith("\n")
writeFileSync(
questionsPath,
JSON.stringify(questionsParsed, null, " ") + (endsWithNewline ? "\n" : "")
)
} else {
console.log("Ignore weblate")
}
@ -717,14 +707,10 @@ const l2 = generateTranslationsObjectFrom(
ScriptUtils.getThemeFiles().filter((th) => th.parsed.mustHaveLanguage === undefined),
"themes"
)
const l3 = generateTranslationsObjectFrom(
[{ path: questionsPath, parsed: questionsParsed }],
"shared-questions"
)
const usedLanguages: string[] = Utils.Dedup(l1.concat(l2).concat(l3)).filter((v) => v !== "*")
const usedLanguages: string[] = Utils.Dedup(l1.concat(l2)).filter((v) => v !== "*")
usedLanguages.sort()
fs.writeFileSync("./assets/used_languages.json", JSON.stringify({ languages: usedLanguages }))
fs.writeFileSync("./src/assets/used_languages.json", JSON.stringify({ languages: usedLanguages }))
if (!themeOverwritesWeblate) {
// Generates the core translations
@ -742,5 +728,4 @@ for (const path of allTranslationFiles) {
TranslationPart.fromDirectory("./langs").validateStrict("./langs")
TranslationPart.fromDirectory("./langs/layers").validateStrict("layers")
TranslationPart.fromDirectory("./langs/themes").validateStrict("themes")
TranslationPart.fromDirectory("./langs/shared-questions").validateStrict("shared-questions")
console.log("All done!")