Bastian-Greshake-Tzovaras/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Merge master

Browse source
This commit is contained in:
Pieter Vander Vennet 2022-06-20 03:22:27 +02:00
commit f0c90b4d1c
50 changed files with 815 additions and 282 deletions
Download patch file Download diff file Expand all files Collapse all files

74
scripts/generateLayerOverview.ts
View file

@ -4,7 +4,12 @@ import * as licenses from "../assets/generated/license_info.json"
import {LayoutConfigJson} from "../Models/ThemeConfig/Json/LayoutConfigJson";
import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
import Constants from "../Models/Constants";
import {PrevalidateTheme, ValidateLayer, ValidateThemeAndLayers} from "../Models/ThemeConfig/Conversion/Validation";
import {
PrevalidateTheme,
ValidateLayer,
ValidateTagRenderings,
ValidateThemeAndLayers
} from "../Models/ThemeConfig/Conversion/Validation";
import {Translation} from "../UI/i18n/Translation";
import {TagRenderingConfigJson} from "../Models/ThemeConfig/Json/TagRenderingConfigJson";
import * as questions from "../assets/tagRenderings/questions.json";
@ -14,24 +19,25 @@ import {PrepareLayer} from "../Models/ThemeConfig/Conversion/PrepareLayer";
import {PrepareTheme} from "../Models/ThemeConfig/Conversion/PrepareTheme";
import {DesugaringContext} from "../Models/ThemeConfig/Conversion/Conversion";
import {Utils} from "../Utils";
import {And} from "../Logic/Tags/And";
// This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files.
// It spits out an overview of those to be used to load them
class LayerOverviewUtils {
writeSmallOverview(themes: { id: string, title: any, shortDescription: any, icon: string, hideFromOverview: boolean, mustHaveLanguage: boolean, layers: (LayerConfigJson | string | {builtin})[] }[]) {
writeSmallOverview(themes: { id: string, title: any, shortDescription: any, icon: string, hideFromOverview: boolean, mustHaveLanguage: boolean, layers: (LayerConfigJson | string | { builtin })[] }[]) {
const perId = new Map<string, any>();
for (const theme of themes) {
const keywords : {}[] = []
const keywords: {}[] = []
for (const layer of (theme.layers ?? [])) {
const l = <LayerConfigJson> layer;
const l = <LayerConfigJson>layer;
keywords.push({"*": l.id})
keywords.push(l.title)
keywords.push(l.description)
}
const data = {
id: theme.id,
title: theme.title,
@ -77,16 +83,19 @@ class LayerOverviewUtils {
writeFileSync(`./assets/generated/layers/${layer.id}.json`, JSON.stringify(layer, null, " "), "UTF8");
}
getSharedTagRenderings(): Map<string, TagRenderingConfigJson> {
getSharedTagRenderings(knownImagePaths: Set<string>): Map<string, TagRenderingConfigJson> {
const dict = new Map<string, TagRenderingConfigJson>();
const validator = new ValidateTagRenderings(undefined, knownImagePaths);
for (const key in questions["default"]) {
if (key === "id") {
continue
}
questions[key].id = key;
questions[key]["source"] = "shared-questions"
dict.set(key, <TagRenderingConfigJson>questions[key])
const config = <TagRenderingConfigJson>questions[key]
validator.convertStrict(config, "generate-layer-overview:tagRenderings/questions.json:"+key)
dict.set(key, config)
}
for (const key in icons["default"]) {
if (key === "id") {
@ -96,7 +105,9 @@ class LayerOverviewUtils {
continue
}
icons[key].id = key;
dict.set(key, <TagRenderingConfigJson>icons[key])
const config = <TagRenderingConfigJson>icons[key]
validator.convertStrict(config, "generate-layer-overview:tagRenderings/icons.json:"+key)
dict.set(key,config)
}
dict.forEach((value, key) => {
@ -114,9 +125,9 @@ class LayerOverviewUtils {
.filter(path => path.endsWith(".svg"))
.filter(path => !path.startsWith("./assets/generated"))
let errCount = 0;
const exempt = ["assets/SocialImageTemplate.svg","assets/SocialImageTemplateWide.svg","assets/SocialImageBanner.svg", "assets/svg/osm-logo.svg"];
const exempt = ["assets/SocialImageTemplate.svg", "assets/SocialImageTemplateWide.svg", "assets/SocialImageBanner.svg", "assets/svg/osm-logo.svg"];
for (const path of allSvgs) {
if(exempt.some(p => "./"+p === path)) {
if (exempt.some(p => "./" + p === path)) {
continue
}
@ -128,7 +139,7 @@ class LayerOverviewUtils {
throw "A core SVG is actually a PNG. Don't do this!"
}
}
if(contents.indexOf("<text")>0){
if (contents.indexOf("<text") > 0) {
console.warn("The SVG at " + path + " contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path")
errCount++;
@ -183,7 +194,7 @@ class LayerOverviewUtils {
// At the same time, an index of available layers is built.
console.log(" ---------- VALIDATING BUILTIN LAYERS ---------")
const sharedTagRenderings = this.getSharedTagRenderings();
const sharedTagRenderings = this.getSharedTagRenderings(knownImagePaths);
const layerFiles = ScriptUtils.getLayerFiles();
const sharedLayers = new Map<string, LayerConfigJson>()
const state: DesugaringContext = {
@ -194,7 +205,12 @@ class LayerOverviewUtils {
for (const sharedLayerJson of layerFiles) {
const context = "While building builtin layer " + sharedLayerJson.path
const fixed = prepLayer.convertStrict(sharedLayerJson.parsed, context)
const validator = new ValidateLayer(sharedLayerJson.path, true);
if(fixed.source.osmTags["and"] === undefined){
fixed.source.osmTags = {"and": [fixed.source.osmTags]}
}
const validator = new ValidateLayer(sharedLayerJson.path, true, knownImagePaths);
validator.convertStrict(fixed, context)
if (sharedLayers.has(fixed.id)) {
@ -208,25 +224,25 @@ class LayerOverviewUtils {
}
return sharedLayers;
}
private static publicLayerIdsFrom(themefiles: LayoutConfigJson[]): Set<string>{
private static publicLayerIdsFrom(themefiles: LayoutConfigJson[]): Set<string> {
const publicLayers = [].concat(...themefiles
.filter(th => !th.hideFromOverview)
.map(th => th.layers))
const publicLayerIds = new Set<string>()
for (const publicLayer of publicLayers) {
if(typeof publicLayer === "string"){
if (typeof publicLayer === "string") {
publicLayerIds.add(publicLayer)
continue
}
if(publicLayer["builtin"] !== undefined){
if (publicLayer["builtin"] !== undefined) {
const bi = publicLayer["builtin"]
if(typeof bi === "string"){
if (typeof bi === "string") {
publicLayerIds.add(bi)
continue
}
bi.forEach(id=>publicLayerIds.add(id))
bi.forEach(id => publicLayerIds.add(id))
continue
}
publicLayerIds.add(publicLayer.id)
@ -243,7 +259,7 @@ class LayerOverviewUtils {
const convertState: DesugaringContext = {
sharedLayers,
tagRenderings: this.getSharedTagRenderings(),
tagRenderings: this.getSharedTagRenderings(knownImagePaths),
publicLayers
}
for (const themeInfo of themeFiles) {
@ -251,20 +267,20 @@ class LayerOverviewUtils {
const themePath = themeInfo.path
new PrevalidateTheme().convertStrict(themeFile, themePath)
try{
try {
themeFile = new PrepareTheme(convertState).convertStrict(themeFile, themePath)
if(knownImagePaths === undefined){
if (knownImagePaths === undefined) {
throw "Could not load known images/licenses"
}
new ValidateThemeAndLayers(knownImagePaths, themePath, true, convertState.tagRenderings)
.convertStrict(themeFile, themePath)
this.writeTheme(themeFile)
fixed.set(themeFile.id, themeFile)
}catch(e){
console.error("ERROR: could not prepare theme "+themePath+" due to "+e)
} catch (e) {
console.error("ERROR: could not prepare theme " + themePath + " due to " + e)
throw e;
}
}

106
scripts/generateLicenseInfo.ts
View file

@ -2,6 +2,7 @@ import {existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync} from "fs
import SmallLicense from "../Models/smallLicense";
import ScriptUtils from "./ScriptUtils";
const prompt = require('prompt-sync')();
function validateLicenseInfo(l : SmallLicense){
l.sources.map(s => new URL(s))
@ -55,7 +56,6 @@ function missingLicenseInfos(licenseInfos: SmallLicense[], allIcons: string[]) {
return missing;
}
const prompt = require('prompt-sync')();
const knownLicenses = new Map<string, SmallLicense>()
knownLicenses.set("me", {
@ -64,45 +64,36 @@ knownLicenses.set("me", {
license: "CC0",
sources: []
})
knownLicenses.set("streetcomplete", {
authors: ["Tobias Zwick (westnordost)"],
path: undefined,
license: "CC0",
sources: ["https://github.com/streetcomplete/StreetComplete/tree/master/res/graphics", "https://f-droid.org/packages/de.westnordost.streetcomplete/"]
})
knownLicenses.set("t", {
authors: [],
path: undefined,
license: "CC0; trivial",
sources: []
})
knownLicenses.set("na", {
authors: [],
path: undefined,
license: "CC0",
sources: []
})
knownLicenses.set("tv", {
authors: ["Toerisme Vlaanderen"],
path: undefined,
license: "CC0",
sources: ["https://toerismevlaanderen.be/pinjepunt","https://mapcomplete.osm.be/toerisme_vlaanderenn"]
})
knownLicenses.set("tvf", {
authors: ["Jo De Baerdemaeker "],
path: undefined,
license: "All rights reserved",
sources: ["https://www.studiotype.be/fonts/flandersart"]
})
knownLicenses.set("twemoji", {
authors: ["Twemoji"],
path: undefined,
@ -154,7 +145,7 @@ function createLicenseInfoFor(path): void {
function cleanLicenseInfo(allPaths: string[], allLicenseInfos: SmallLicense[]) {
// Read the license info file from the generated assets, creates a compiled license info in every directory
// Note: this removes all the old license infos
for (const licensePath of licensePaths) {
for (const licensePath of allPaths) {
unlinkSync(licensePath)
}
@ -219,10 +210,13 @@ function queryMissingLicenses(missingLicenses: string[]) {
* Creates the humongous license_info in the generated assets, containing all licenses with a path relative to the root
* @param licensePaths
*/
function createFullLicenseOverview(licensePaths) {
function createFullLicenseOverview(licensePaths: string[]) {
const allLicenses: SmallLicense[] = []
for (const licensePath of licensePaths) {
if(!existsSync(licensePath)){
continue
}
const licenses = <SmallLicense[]>JSON.parse(readFileSync(licensePath, "UTF-8"))
for (const license of licenses) {
validateLicenseInfo(license)
@ -235,52 +229,54 @@ function createFullLicenseOverview(licensePaths) {
writeFileSync("./assets/generated/license_info.json", JSON.stringify(allLicenses, null, " "))
}
console.log("Checking and compiling license info")
if (!existsSync("./assets/generated")) {
mkdirSync("./assets/generated")
}
let contents = ScriptUtils.readDirRecSync("./assets")
.filter(entry => entry.indexOf("./assets/generated") != 0)
let licensePaths = contents.filter(entry => entry.indexOf("license_info.json") >= 0)
let licenseInfos = generateLicenseInfos(licensePaths);
const artwork = contents.filter(pth => pth.match(/(.svg|.png|.jpg|.ttf|.otf|.woff)$/i) != null)
const missingLicenses = missingLicenseInfos(licenseInfos, artwork)
if (process.argv.indexOf("--prompt") >= 0 || process.argv.indexOf("--query") >= 0) {
queryMissingLicenses(missingLicenses)
contents = ScriptUtils.readDirRecSync("./assets")
function main(args: string[]){
console.log("Checking and compiling license info")
if (!existsSync("./assets/generated")) {
mkdirSync("./assets/generated")
}
let contents = ScriptUtils.readDirRecSync("./assets")
.filter(entry => entry.indexOf("./assets/generated") != 0)
licensePaths = contents.filter(entry => entry.indexOf("license_info.json") >= 0)
licenseInfos = generateLicenseInfos(licensePaths);
}
const invalidLicenses = licenseInfos.filter(l => (l.license ?? "") === "").map(l => `License for artwork ${l.path} is empty string or undefined`)
for (const licenseInfo of licenseInfos) {
for (const source of licenseInfo.sources) {
if (source == "") {
invalidLicenses.push("Invalid license: empty string in " + JSON.stringify(licenseInfo))
}
try {
new URL(source);
} catch {
invalidLicenses.push("Not a valid URL: " + source)
let licensePaths = contents.filter(entry => entry.indexOf("license_info.json") >= 0)
let licenseInfos = generateLicenseInfos(licensePaths);
const artwork = contents.filter(pth => pth.match(/(.svg|.png|.jpg|.ttf|.otf|.woff)$/i) != null)
const missingLicenses = missingLicenseInfos(licenseInfos, artwork)
if (args.indexOf("--prompt") >= 0 || args.indexOf("--query") >= 0) {
queryMissingLicenses(missingLicenses)
return main([])
}
const invalidLicenses = licenseInfos.filter(l => (l.license ?? "") === "").map(l => `License for artwork ${l.path} is empty string or undefined`)
for (const licenseInfo of licenseInfos) {
for (const source of licenseInfo.sources) {
if (source == "") {
invalidLicenses.push("Invalid license: empty string in " + JSON.stringify(licenseInfo))
}
try {
new URL(source);
} catch {
invalidLicenses.push("Not a valid URL: " + source)
}
}
}
}
if (missingLicenses.length > 0) {
const msg = `There are ${missingLicenses.length} licenses missing and ${invalidLicenses.length} invalid licenses.`
console.log(missingLicenses.concat(invalidLicenses).join("\n"))
console.error(msg)
if (process.argv.indexOf("--no-fail") < 0) {
throw msg
if (missingLicenses.length > 0) {
const msg = `There are ${missingLicenses.length} licenses missing and ${invalidLicenses.length} invalid licenses.`
console.log(missingLicenses.concat(invalidLicenses).join("\n"))
console.error(msg)
if (args.indexOf("--no-fail") < 0) {
throw msg
}
}
cleanLicenseInfo(licensePaths, licenseInfos)
createFullLicenseOverview(licensePaths)
}
cleanLicenseInfo(licensePaths, licenseInfos)
createFullLicenseOverview(licensePaths)
main(process.argv.slice(2))