ilja_space/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Generate layer overview now only recompiles files that need to be recompiled

Browse source
This commit is contained in:
Pieter Vander Vennet 2022-07-06 13:58:56 +02:00
parent 063d7e4637
commit 8c036e159f
4 changed files with 204 additions and 137 deletions
Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -28,7 +28,7 @@
"generate:cache:speelplekken": "npm run generate:layeroverview && ts-node scripts/generateCache.ts speelplekken 14 ../MapComplete-data/speelplekken_cache/ 51.20 4.35 51.09 4.56",
"generate:cache:natuurpunt": "npm run generate:layeroverview && ts-node scripts/generateCache.ts natuurpunt 12 ../MapComplete-data/natuurpunt_cache/ 50.40 2.1 51.54 6.4 --generate-point-overview nature_reserve,visitor_information_centre",
"generate:cache:natuurpunt:mini": "ts-node scripts/generateCache.ts natuurpunt 12 ../../git/MapComplete-data/natuurpunt_cache_mini/ 51.00792239979105 4.497699737548828 51.0353492224462554 4.539070129394531 --generate-point-overview nature_reserve,visitor_information_centre",
"generate:layeroverview": "ts-node scripts/generateLayerOverview.ts --no-fail",
"generate:layeroverview": "ts-node scripts/generateLayerOverview.ts",
"generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail",
"query:licenses": "ts-node scripts/generateLicenseInfo.ts --query",
"generate:report": "cd Docs/Tools && ./compileStats.sh && git commit . -m 'New statistics ands graphs' && git push",

176
scripts/ScriptUtils.ts
View file

@ -45,13 +45,103 @@ export default class ScriptUtils {
})
}
private static async DownloadJSON(url: string, headers?: any): Promise<any>{
public static erasableLog(...text) {
process.stdout.write("\r " + text.join(" ") + " \r")
}
public static sleep(ms) {
if (ms <= 0) {
process.stdout.write("\r \r")
return;
}
return new Promise((resolve) => {
process.stdout.write("\r Sleeping for " + (ms / 1000) + "s \r")
setTimeout(resolve, 1000);
}).then(() => ScriptUtils.sleep(ms - 1000));
}
public static getLayerPaths(): string[] {
return ScriptUtils.readDirRecSync("./assets/layers")
.filter(path => path.indexOf(".json") > 0)
.filter(path => path.indexOf(".proto.json") < 0)
.filter(path => path.indexOf("license_info.json") < 0)
}
public static getLayerFiles(): { parsed: LayerConfigJson, path: string }[] {
return ScriptUtils.readDirRecSync("./assets/layers")
.filter(path => path.indexOf(".json") > 0)
.filter(path => path.indexOf(".proto.json") < 0)
.filter(path => path.indexOf("license_info.json") < 0)
.map(path => {
try {
const contents = readFileSync(path, "UTF8")
if (contents === "") {
throw "The file " + path + " is empty, did you properly save?"
}
const parsed = JSON.parse(contents);
return {parsed, path}
} catch (e) {
console.error("Could not parse file ", "./assets/layers/" + path, "due to ", e)
throw e
}
})
}
public static getThemePaths(): string[] {
return ScriptUtils.readDirRecSync("./assets/themes")
.filter(path => path.endsWith(".json") && !path.endsWith(".proto.json"))
.filter(path => path.indexOf("license_info.json") < 0)
}
public static getThemeFiles(): { parsed: LayoutConfigJson, path: string }[] {
return this.getThemePaths()
.map(path => {
try {
const contents = readFileSync(path, "UTF8");
if (contents === "") {
throw "The file " + path + " is empty, did you properly save?"
}
const parsed = JSON.parse(contents);
return {parsed: parsed, path: path}
} catch (e) {
console.error("Could not read file ", path, "due to ", e)
throw e
}
});
}
public static TagInfoHistogram(key: string): Promise<{
data: { count: number, value: string, fraction: number }[]
}> {
const url = `https://taginfo.openstreetmap.org/api/4/key/values?key=${key}&filter=all&lang=en&sortname=count&sortorder=desc&page=1&rp=17&qtype=value`
return ScriptUtils.DownloadJSON(url)
}
public static async ReadSvg(path: string): Promise<any> {
if (!existsSync(path)) {
throw "File not found: " + path
}
const root = await xml2js.parseStringPromise(readFileSync(path, "UTF8"))
return root.svg
}
public static async ReadSvgSync(path: string, callback: ((svg: any) => void)): Promise<any> {
xml2js.parseString(readFileSync(path, "UTF8"), {async: false}, (err, root) => {
if (err) {
throw err
}
callback(root["svg"]);
})
}
private static async DownloadJSON(url: string, headers?: any): Promise<any> {
const data = await ScriptUtils.Download(url, headers);
return JSON.parse(data.content)
}
private static Download(url, headers?: any): Promise<{content: string}> {
private static Download(url, headers?: any): Promise<{ content: string }> {
return new Promise((resolve, reject) => {
try {
headers = headers ?? {}
@ -83,84 +173,4 @@ export default class ScriptUtils {
}
public static erasableLog(...text) {
process.stdout.write("\r " + text.join(" ") + " \r")
}
public static sleep(ms) {
if (ms <= 0) {
process.stdout.write("\r \r")
return;
}
return new Promise((resolve) => {
process.stdout.write("\r Sleeping for " + (ms / 1000) + "s \r")
setTimeout(resolve, 1000);
}).then(() => ScriptUtils.sleep(ms - 1000));
}
public static getLayerFiles(): { parsed: LayerConfigJson, path: string }[] {
return ScriptUtils.readDirRecSync("./assets/layers")
.filter(path => path.indexOf(".json") > 0)
.filter(path => path.indexOf(".proto.json") < 0)
.filter(path => path.indexOf("license_info.json") < 0)
.map(path => {
try {
const contents = readFileSync(path, "UTF8")
if (contents === "") {
throw "The file " + path + " is empty, did you properly save?"
}
const parsed = JSON.parse(contents);
return {parsed, path}
} catch (e) {
console.error("Could not parse file ", "./assets/layers/" + path, "due to ", e)
throw e
}
})
}
public static getThemeFiles(): { parsed: LayoutConfigJson, path: string }[] {
return ScriptUtils.readDirRecSync("./assets/themes")
.filter(path => path.endsWith(".json") && !path.endsWith(".proto.json"))
.filter(path => path.indexOf("license_info.json") < 0)
.map(path => {
try {
const contents = readFileSync(path, "UTF8");
if (contents === "") {
throw "The file " + path + " is empty, did you properly save?"
}
const parsed = JSON.parse(contents);
return {parsed: parsed, path: path}
} catch (e) {
console.error("Could not read file ", path, "due to ", e)
throw e
}
});
}
public static TagInfoHistogram(key: string): Promise<{
data: { count: number, value: string, fraction: number }[]
}> {
const url = `https://taginfo.openstreetmap.org/api/4/key/values?key=${key}&filter=all&lang=en&sortname=count&sortorder=desc&page=1&rp=17&qtype=value`
return ScriptUtils.DownloadJSON(url)
}
public static async ReadSvg(path: string): Promise<any>{
if(!existsSync(path)){
throw "File not found: "+path
}
const root = await xml2js.parseStringPromise(readFileSync(path, "UTF8"))
return root.svg
}
public static async ReadSvgSync(path: string, callback: ((svg: any) => void)): Promise<any>{
xml2js.parseString(readFileSync(path, "UTF8"),{async: false} , (err, root) => {
if(err){
throw err
}
callback(root["svg"]);
})
}
}

2
scripts/build.sh
View file

@ -12,7 +12,7 @@ mkdir dist/assets 2> /dev/null
# This script ends every line with '&&' to chain everything. A failure will thus stop the build
npm run generate:editor-layer-index
npm run generate &&
npm run generate:layeroverview && # generate:layeroverview has to be run twice: the personal theme won't pick up all the layers otherwise
ts-node ./scripts/generateLayeroverview --force && # generate:layeroverview has to be run twice: the personal theme won't pick up all the layers otherwise
npm run test &&
npm run generate:layouts

161
scripts/generateLayerOverview.ts
View file

@ -1,5 +1,5 @@
import ScriptUtils from "./ScriptUtils";
import {existsSync, mkdirSync, readFileSync, writeFileSync} from "fs";
import {existsSync, mkdirSync, readFileSync, statSync, writeFileSync} from "fs";
import * as licenses from "../assets/generated/license_info.json"
import {LayoutConfigJson} from "../Models/ThemeConfig/Json/LayoutConfigJson";
import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
@ -26,6 +26,51 @@ import {Utils} from "../Utils";
class LayerOverviewUtils {
public static readonly layerPath = "./assets/generated/layers/"
public static readonly themePath = "./assets/generated/themes/"
private static publicLayerIdsFrom(themefiles: LayoutConfigJson[]): Set<string> {
const publicThemes = [].concat(...themefiles
.filter(th => !th.hideFromOverview))
return new Set([].concat(...publicThemes.map(th => this.extractLayerIdsFrom(th))))
}
private static extractLayerIdsFrom(themeFile: LayoutConfigJson, includeInlineLayers = true): string[] {
const publicLayerIds = []
for (const publicLayer of themeFile.layers) {
if (typeof publicLayer === "string") {
publicLayerIds.push(publicLayer)
continue
}
if (publicLayer["builtin"] !== undefined) {
const bi = publicLayer["builtin"]
if (typeof bi === "string") {
publicLayerIds.push(bi)
continue
}
bi.forEach(id => publicLayerIds.push(id))
continue
}
if (includeInlineLayers) {
publicLayerIds.push(publicLayer["id"])
}
}
return publicLayerIds
}
shouldBeUpdated(sourcefile: string | string[], targetfile: string): boolean {
if (!existsSync(targetfile)) {
return true;
}
const targetModified = statSync(targetfile).mtime
if (typeof sourcefile === "string") {
sourcefile = [sourcefile]
}
return sourcefile.some(sourcefile => statSync(sourcefile).mtime > targetModified)
}
writeSmallOverview(themes: { id: string, title: any, shortDescription: any, icon: string, hideFromOverview: boolean, mustHaveLanguage: boolean, layers: (LayerConfigJson | string | { builtin })[] }[]) {
const perId = new Map<string, any>();
for (const theme of themes) {
@ -70,22 +115,22 @@ class LayerOverviewUtils {
}
writeTheme(theme: LayoutConfigJson) {
if (!existsSync("./assets/generated/themes")) {
mkdirSync("./assets/generated/themes");
if (!existsSync(LayerOverviewUtils.themePath)) {
mkdirSync(LayerOverviewUtils.themePath);
}
writeFileSync(`./assets/generated/themes/${theme.id}.json`, JSON.stringify(theme, null, " "), "UTF8");
writeFileSync(`${LayerOverviewUtils.themePath}${theme.id}.json`, JSON.stringify(theme, null, " "), "UTF8");
}
writeLayer(layer: LayerConfigJson) {
if (!existsSync("./assets/generated/layers")) {
mkdirSync("./assets/generated/layers");
if (!existsSync(LayerOverviewUtils.layerPath)) {
mkdirSync(LayerOverviewUtils.layerPath);
}
writeFileSync(`./assets/generated/layers/${layer.id}.json`, JSON.stringify(layer, null, " "), "UTF8");
writeFileSync(`${LayerOverviewUtils.layerPath}${layer.id}.json`, JSON.stringify(layer, null, " "), "UTF8");
}
getSharedTagRenderings(doesImageExist: DoesImageExist): Map<string, TagRenderingConfigJson> {
const dict = new Map<string, TagRenderingConfigJson>();
const validator = new ValidateTagRenderings(undefined, doesImageExist);
for (const key in questions["default"]) {
if (key === "id") {
@ -94,7 +139,7 @@ class LayerOverviewUtils {
questions[key].id = key;
questions[key]["source"] = "shared-questions"
const config = <TagRenderingConfigJson>questions[key]
validator.convertStrict(config, "generate-layer-overview:tagRenderings/questions.json:"+key)
validator.convertStrict(config, "generate-layer-overview:tagRenderings/questions.json:" + key)
dict.set(key, config)
}
for (const key in icons["default"]) {
@ -105,9 +150,9 @@ class LayerOverviewUtils {
continue
}
icons[key].id = key;
const config = <TagRenderingConfigJson>icons[key]
validator.convertStrict(config, "generate-layer-overview:tagRenderings/icons.json:"+key)
dict.set(key,config)
const config = <TagRenderingConfigJson>icons[key]
validator.convertStrict(config, "generate-layer-overview:tagRenderings/icons.json:" + key)
dict.set(key, config)
}
dict.forEach((value, key) => {
@ -150,16 +195,18 @@ class LayerOverviewUtils {
}
}
main(_: string[]) {
main(args: string[]) {
const forceReload = args.some(a => a == "--force")
const licensePaths = new Set<string>()
for (const i in licenses) {
licensePaths.add(licenses[i].path)
}
const doesImageExist = new DoesImageExist(licensePaths, existsSync)
const sharedLayers = this.buildLayerIndex(doesImageExist);
const sharedThemes = this.buildThemeIndex(doesImageExist, sharedLayers)
const sharedLayers = this.buildLayerIndex(doesImageExist, forceReload);
const recompiledThemes : string[] = []
const sharedThemes = this.buildThemeIndex(doesImageExist, sharedLayers, recompiledThemes, forceReload)
writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
"layers": Array.from(sharedLayers.values()),
@ -169,7 +216,7 @@ class LayerOverviewUtils {
writeFileSync("./assets/generated/known_layers.json", JSON.stringify({layers: Array.from(sharedLayers.values())}))
{
if(recompiledThemes.length > 0) {
// mapcomplete-changes shows an icon for each corresponding mapcomplete-theme
const iconsPerTheme =
Array.from(sharedThemes.values()).map(th => ({
@ -189,28 +236,42 @@ class LayerOverviewUtils {
console.log(green("All done!"))
}
private buildLayerIndex(doesImageExist: DoesImageExist): Map<string, LayerConfigJson> {
private buildLayerIndex(doesImageExist: DoesImageExist, forceReload: boolean): Map<string, LayerConfigJson> {
// First, we expand and validate all builtin layers. These are written to assets/generated/layers
// At the same time, an index of available layers is built.
console.log(" ---------- VALIDATING BUILTIN LAYERS ---------")
const sharedTagRenderings = this.getSharedTagRenderings(doesImageExist);
const layerFiles = ScriptUtils.getLayerFiles();
const sharedLayers = new Map<string, LayerConfigJson>()
const state: DesugaringContext = {
tagRenderings: sharedTagRenderings,
sharedLayers
}
const prepLayer = new PrepareLayer(state);
for (const sharedLayerJson of layerFiles) {
const context = "While building builtin layer " + sharedLayerJson.path
const fixed = prepLayer.convertStrict(sharedLayerJson.parsed, context)
const skippedLayers: string[] = []
const recompiledLayers: string[] = []
for (const sharedLayerPath of ScriptUtils.getLayerPaths()) {
{
const targetPath = LayerOverviewUtils.layerPath + sharedLayerPath.substring(sharedLayerPath.lastIndexOf("/"))
if (!forceReload && !this.shouldBeUpdated(sharedLayerPath, targetPath)) {
const sharedLayer = JSON.parse(readFileSync(targetPath, "utf8"))
sharedLayers.set(sharedLayer.id, sharedLayer)
skippedLayers.push(sharedLayer.id)
continue;
}
if(fixed.source.osmTags["and"] === undefined){
}
const parsed = JSON.parse(readFileSync(sharedLayerPath, "utf8"))
const context = "While building builtin layer " + sharedLayerPath
const fixed = prepLayer.convertStrict(parsed, context)
if (fixed.source.osmTags["and"] === undefined) {
fixed.source.osmTags = {"and": [fixed.source.osmTags]}
}
const validator = new ValidateLayer(sharedLayerJson.path, true, doesImageExist);
const validator = new ValidateLayer(sharedLayerPath, true, doesImageExist);
validator.convertStrict(fixed, context)
if (sharedLayers.has(fixed.id)) {
@ -218,39 +279,18 @@ class LayerOverviewUtils {
}
sharedLayers.set(fixed.id, fixed)
recompiledLayers.push(fixed.id)
this.writeLayer(fixed)
}
console.log("Recompiled layers " + recompiledLayers.join(", ") + " and skipped " + skippedLayers.length + " layers")
return sharedLayers;
}
private static publicLayerIdsFrom(themefiles: LayoutConfigJson[]): Set<string> {
const publicLayers = [].concat(...themefiles
.filter(th => !th.hideFromOverview)
.map(th => th.layers))
const publicLayerIds = new Set<string>()
for (const publicLayer of publicLayers) {
if (typeof publicLayer === "string") {
publicLayerIds.add(publicLayer)
continue
}
if (publicLayer["builtin"] !== undefined) {
const bi = publicLayer["builtin"]
if (typeof bi === "string") {
publicLayerIds.add(bi)
continue
}
bi.forEach(id => publicLayerIds.add(id))
continue
}
publicLayerIds.add(publicLayer.id)
}
return publicLayerIds
}
private buildThemeIndex(doesImageExist: DoesImageExist, sharedLayers: Map<string, LayerConfigJson>): Map<string, LayoutConfigJson> {
private buildThemeIndex(doesImageExist: DoesImageExist, sharedLayers: Map<string, LayerConfigJson>, recompiledThemes: string[], forceReload: boolean): Map<string, LayoutConfigJson> {
console.log(" ---------- VALIDATING BUILTIN THEMES ---------")
const themeFiles = ScriptUtils.getThemeFiles();
const fixed = new Map<string, LayoutConfigJson>();
@ -262,9 +302,23 @@ class LayerOverviewUtils {
tagRenderings: this.getSharedTagRenderings(doesImageExist),
publicLayers
}
const skippedThemes: string[] = []
for (const themeInfo of themeFiles) {
const themePath = themeInfo.path;
let themeFile = themeInfo.parsed
const themePath = themeInfo.path
{
const targetPath = LayerOverviewUtils.themePath + "/" + themePath.substring(themePath.lastIndexOf("/"))
const usedLayers = Array.from(LayerOverviewUtils.extractLayerIdsFrom(themeFile, false))
.map(id => LayerOverviewUtils.layerPath + id + ".json")
if (!forceReload && !this.shouldBeUpdated([themePath, ...usedLayers], targetPath)) {
fixed.set(themeFile.id, themeFile)
skippedThemes.push(themeFile.id)
continue;
}
recompiledThemes.push(themeFile.id)
}
new PrevalidateTheme().convertStrict(themeFile, themePath)
try {
@ -290,6 +344,9 @@ class LayerOverviewUtils {
mustHaveLanguage: t.mustHaveLanguage?.length > 0,
}
}));
console.log("Recompiled themes " + recompiledThemes.join(", ") + " and skipped " + skippedThemes.length + " themes")
return fixed;
}