contrapunctus/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Reformat all files with prettier

Browse source
This commit is contained in:
Pieter Vander Vennet 2022-09-08 21:40:48 +02:00
parent e22d189376
commit b541d3eab4
382 changed files with 50893 additions and 35566 deletions
Download patch file Download diff file Expand all files Collapse all files

45
scripts/CycleHighwayFix.ts
View file

@ -1,21 +1,34 @@
import ScriptUtils from "./ScriptUtils";
import {appendFileSync, readFileSync, writeFileSync} from "fs";
import {OsmObject} from "../Logic/Osm/OsmObject";
import ScriptUtils from "./ScriptUtils"
import { appendFileSync, readFileSync, writeFileSync } from "fs"
import { OsmObject } from "../Logic/Osm/OsmObject"
ScriptUtils.fixUtils()
ScriptUtils.erasableLog("Fixing the cycle highways...")
writeFileSync("cycleHighwayFix.osc", "<osmChange version=\"0.6\" generator=\"Handmade\" copyright=\"OpenStreetMap and Contributors\"\n" +
" attribution=\"http://www.openstreetmap.org/copyright\" license=\"http://opendatacommons.org/licenses/odbl/1-0/\">\n" +
" <modify>", "utf8")
const ids = JSON.parse(readFileSync("export.geojson", "utf-8")).features.map(f => f.properties["@id"])
writeFileSync(
"cycleHighwayFix.osc",
'<osmChange version="0.6" generator="Handmade" copyright="OpenStreetMap and Contributors"\n' +
' attribution="http://www.openstreetmap.org/copyright" license="http://opendatacommons.org/licenses/odbl/1-0/">\n' +
" <modify>",
"utf8"
)
const ids = JSON.parse(readFileSync("export.geojson", "utf-8")).features.map(
(f) => f.properties["@id"]
)
console.log(ids)
ids.map(id => OsmObject.DownloadReferencingRelations(id).then(relations => {
console.log(relations)
const changeparts = relations.filter(relation => relation.tags["cycle_highway"] == "yes" && relation.tags["note:state"] == undefined)
.map(relation => {
relation.tags["note:state"] = "has_highway_under_construction";
return relation.ChangesetXML(undefined)
})
appendFileSync("cycleHighwayFix.osc", changeparts.join("\n"), "utf8")
}))
ids.map((id) =>
OsmObject.DownloadReferencingRelations(id).then((relations) => {
console.log(relations)
const changeparts = relations
.filter(
(relation) =>
relation.tags["cycle_highway"] == "yes" &&
relation.tags["note:state"] == undefined
)
.map((relation) => {
relation.tags["note:state"] = "has_highway_under_construction"
return relation.ChangesetXML(undefined)
})
appendFileSync("cycleHighwayFix.osc", changeparts.join("\n"), "utf8")
})
)

136
scripts/ScriptUtils.ts
View file

@ -1,18 +1,16 @@
import * as fs from "fs";
import {existsSync, lstatSync, readdirSync, readFileSync} from "fs";
import {Utils} from "../Utils";
import * as https from "https";
import {LayoutConfigJson} from "../Models/ThemeConfig/Json/LayoutConfigJson";
import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
import xml2js from 'xml2js';
import * as fs from "fs"
import { existsSync, lstatSync, readdirSync, readFileSync } from "fs"
import { Utils } from "../Utils"
import * as https from "https"
import { LayoutConfigJson } from "../Models/ThemeConfig/Json/LayoutConfigJson"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import xml2js from "xml2js"
export default class ScriptUtils {
public static fixUtils() {
Utils.externalDownloadFunction = ScriptUtils.Download
}
public static readDirRecSync(path, maxDepth = 999): string[] {
const result = []
if (maxDepth <= 0) {
@ -29,20 +27,18 @@ export default class ScriptUtils {
result.push(fullEntry)
}
}
return result;
return result
}
public static DownloadFileTo(url, targetFilePath: string): void {
console.log("Downloading ", url, "to", targetFilePath)
https.get(url, (res) => {
const filePath = fs.createWriteStream(targetFilePath);
res.pipe(filePath);
filePath.on('finish', () => {
filePath.close();
console.log('Download Completed');
const filePath = fs.createWriteStream(targetFilePath)
res.pipe(filePath)
filePath.on("finish", () => {
filePath.close()
console.log("Download Completed")
})
})
}
@ -53,35 +49,35 @@ export default class ScriptUtils {
public static sleep(ms) {
if (ms <= 0) {
process.stdout.write("\r \r")
return;
return
}
return new Promise((resolve) => {
process.stdout.write("\r Sleeping for " + (ms / 1000) + "s \r")
setTimeout(resolve, 1000);
}).then(() => ScriptUtils.sleep(ms - 1000));
process.stdout.write("\r Sleeping for " + ms / 1000 + "s \r")
setTimeout(resolve, 1000)
}).then(() => ScriptUtils.sleep(ms - 1000))
}
public static getLayerPaths(): string[] {
return ScriptUtils.readDirRecSync("./assets/layers")
.filter(path => path.indexOf(".json") > 0)
.filter(path => path.indexOf(".proto.json") < 0)
.filter(path => path.indexOf("license_info.json") < 0)
.filter((path) => path.indexOf(".json") > 0)
.filter((path) => path.indexOf(".proto.json") < 0)
.filter((path) => path.indexOf("license_info.json") < 0)
}
public static getLayerFiles(): { parsed: LayerConfigJson, path: string }[] {
public static getLayerFiles(): { parsed: LayerConfigJson; path: string }[] {
return ScriptUtils.readDirRecSync("./assets/layers")
.filter(path => path.indexOf(".json") > 0)
.filter(path => path.indexOf(".proto.json") < 0)
.filter(path => path.indexOf("license_info.json") < 0)
.map(path => {
.filter((path) => path.indexOf(".json") > 0)
.filter((path) => path.indexOf(".proto.json") < 0)
.filter((path) => path.indexOf("license_info.json") < 0)
.map((path) => {
try {
const contents = readFileSync(path, "UTF8")
if (contents === "") {
throw "The file " + path + " is empty, did you properly save?"
}
const parsed = JSON.parse(contents);
return {parsed, path}
const parsed = JSON.parse(contents)
return { parsed, path }
} catch (e) {
console.error("Could not parse file ", "./assets/layers/" + path, "due to ", e)
throw e
@ -91,29 +87,28 @@ export default class ScriptUtils {
public static getThemePaths(): string[] {
return ScriptUtils.readDirRecSync("./assets/themes")
.filter(path => path.endsWith(".json") && !path.endsWith(".proto.json"))
.filter(path => path.indexOf("license_info.json") < 0)
.filter((path) => path.endsWith(".json") && !path.endsWith(".proto.json"))
.filter((path) => path.indexOf("license_info.json") < 0)
}
public static getThemeFiles(): { parsed: LayoutConfigJson, path: string }[] {
return this.getThemePaths()
.map(path => {
try {
const contents = readFileSync(path, "UTF8");
if (contents === "") {
throw "The file " + path + " is empty, did you properly save?"
}
const parsed = JSON.parse(contents);
return {parsed: parsed, path: path}
} catch (e) {
console.error("Could not read file ", path, "due to ", e)
throw e
public static getThemeFiles(): { parsed: LayoutConfigJson; path: string }[] {
return this.getThemePaths().map((path) => {
try {
const contents = readFileSync(path, "UTF8")
if (contents === "") {
throw "The file " + path + " is empty, did you properly save?"
}
});
const parsed = JSON.parse(contents)
return { parsed: parsed, path: path }
} catch (e) {
console.error("Could not read file ", path, "due to ", e)
throw e
}
})
}
public static TagInfoHistogram(key: string): Promise<{
data: { count: number, value: string, fraction: number }[]
data: { count: number; value: string; fraction: number }[]
}> {
const url = `https://taginfo.openstreetmap.org/api/4/key/values?key=${key}&filter=all&lang=en&sortname=count&sortorder=desc&page=1&rp=17&qtype=value`
return ScriptUtils.DownloadJSON(url)
@ -127,17 +122,17 @@ export default class ScriptUtils {
return root.svg
}
public static async ReadSvgSync(path: string, callback: ((svg: any) => void)): Promise<any> {
xml2js.parseString(readFileSync(path, "UTF8"), {async: false}, (err, root) => {
public static async ReadSvgSync(path: string, callback: (svg: any) => void): Promise<any> {
xml2js.parseString(readFileSync(path, "UTF8"), { async: false }, (err, root) => {
if (err) {
throw err
}
callback(root["svg"]);
callback(root["svg"])
})
}
private static async DownloadJSON(url: string, headers?: any): Promise<any> {
const data = await ScriptUtils.Download(url, headers);
const data = await ScriptUtils.Download(url, headers)
return JSON.parse(data.content)
}
@ -148,29 +143,30 @@ export default class ScriptUtils {
headers.accept = "application/json"
console.log(" > ScriptUtils.DownloadJson(", url, ")")
const urlObj = new URL(url)
https.get({
host: urlObj.host,
path: urlObj.pathname + urlObj.search,
https.get(
{
host: urlObj.host,
path: urlObj.pathname + urlObj.search,
port: urlObj.port,
headers: headers
}, (res) => {
const parts: string[] = []
res.setEncoding('utf8');
res.on('data', function (chunk) {
// @ts-ignore
parts.push(chunk)
});
port: urlObj.port,
headers: headers,
},
(res) => {
const parts: string[] = []
res.setEncoding("utf8")
res.on("data", function (chunk) {
// @ts-ignore
parts.push(chunk)
})
res.addListener('end', function () {
resolve({content: parts.join("")})
});
})
res.addListener("end", function () {
resolve({ content: parts.join("") })
})
}
)
} catch (e) {
reject(e)
}
})
}
}

47
scripts/automoveTranslations.ts
View file

@ -1,5 +1,5 @@
import * as languages from "../assets/generated/used_languages.json"
import {readFileSync, writeFileSync} from "fs";
import { readFileSync, writeFileSync } from "fs"
/**
* Moves values around in 'section'. Section will be changed
@ -8,54 +8,57 @@ import {readFileSync, writeFileSync} from "fs";
* @param language
*/
function fixSection(section, referenceSection, language: string) {
if(section === undefined){
if (section === undefined) {
return
}
outer: for (const key of Object.keys(section)) {
const v = section[key]
if(typeof v ==="string" && referenceSection[key] === undefined){
if (typeof v === "string" && referenceSection[key] === undefined) {
// Not found in reference, search for a subsection with this key
for (const subkey of Object.keys(referenceSection)) {
const subreference = referenceSection[subkey]
if(subreference[key] !== undefined){
if(section[subkey] !== undefined && section[subkey][key] !== undefined) {
if (subreference[key] !== undefined) {
if (section[subkey] !== undefined && section[subkey][key] !== undefined) {
console.log(`${subkey}${key} is already defined... Looking furhter`)
continue
}
if(typeof section[subkey] === "string"){
console.log(`NOT overwriting '${section[subkey]}' for ${subkey} (needed for ${key})`)
}else{
if (typeof section[subkey] === "string") {
console.log(
`NOT overwriting '${section[subkey]}' for ${subkey} (needed for ${key})`
)
} else {
// apply fix
if(section[subkey] === undefined){
if (section[subkey] === undefined) {
section[subkey] = {}
}
section[subkey][key] = section[key]
delete section[key]
console.log(`Rewritten key: ${key} --> ${subkey}.${key} in language ${language}`)
delete section[key]
console.log(
`Rewritten key: ${key} --> ${subkey}.${key} in language ${language}`
)
continue outer
}
}
}
console.log("No solution found for "+key)
console.log("No solution found for " + key)
}
}
}
function main(args:string[]):void{
function main(args: string[]): void {
const sectionName = args[0]
const l = args[1]
if(sectionName === undefined){
console.log("Tries to automatically move translations to a new subsegment. Usage: 'sectionToCheck' 'language'")
if (sectionName === undefined) {
console.log(
"Tries to automatically move translations to a new subsegment. Usage: 'sectionToCheck' 'language'"
)
return
}
const reference = JSON.parse( readFileSync("./langs/en.json","UTF8"))
const reference = JSON.parse(readFileSync("./langs/en.json", "UTF8"))
const path = `./langs/${l}.json`
const file = JSON.parse( readFileSync(path,"UTF8"))
const file = JSON.parse(readFileSync(path, "UTF8"))
fixSection(file[sectionName], reference[sectionName], l)
writeFileSync(path, JSON.stringify(file, null, " ")+"\n")
writeFileSync(path, JSON.stringify(file, null, " ") + "\n")
}
main(process.argv.slice(2))
main(process.argv.slice(2))

88
scripts/csvToGeojson.ts
View file

@ -1,9 +1,8 @@
import {parse} from 'csv-parse/sync';
import {readFileSync} from "fs";
import { parse } from "csv-parse/sync"
import { readFileSync } from "fs"
var lambert72toWGS84 = function(x, y){
var newLongitude, newLatitude;
var lambert72toWGS84 = function (x, y) {
var newLongitude, newLatitude
var n = 0.77164219,
F = 1.81329763,
@ -12,78 +11,81 @@ var lambert72toWGS84 = function(x, y){
a = 6378388,
xDiff = 149910,
yDiff = 5400150,
theta0 = 0.07604294;
theta0 = 0.07604294
var xReal = xDiff - x,
yReal = yDiff - y;
yReal = yDiff - y
var rho = Math.sqrt(xReal * xReal + yReal * yReal),
theta = Math.atan(xReal / -yReal);
theta = Math.atan(xReal / -yReal)
newLongitude = (theta0 + (theta + thetaFudge) / n) * 180 / Math.PI;
newLatitude = 0;
newLongitude = ((theta0 + (theta + thetaFudge) / n) * 180) / Math.PI
newLatitude = 0
for (var i = 0; i < 5 ; ++i) {
newLatitude = (2 * Math.atan(Math.pow(F * a / rho, 1 / n) * Math.pow((1 + e * Math.sin(newLatitude)) / (1 - e * Math.sin(newLatitude)), e / 2))) - Math.PI / 2;
for (var i = 0; i < 5; ++i) {
newLatitude =
2 *
Math.atan(
Math.pow((F * a) / rho, 1 / n) *
Math.pow(
(1 + e * Math.sin(newLatitude)) / (1 - e * Math.sin(newLatitude)),
e / 2
)
) -
Math.PI / 2
}
newLatitude *= 180 / Math.PI;
return [newLongitude, newLatitude];
newLatitude *= 180 / Math.PI
return [newLongitude, newLatitude]
}
function main(args: string[]): void {
if (args.length == 0) {
/* args = ["/home/pietervdvn/Downloads/Scholen/aantallen.csv",
/* args = ["/home/pietervdvn/Downloads/Scholen/aantallen.csv",
"/home/pietervdvn/Downloads/Scholen/perschool.csv",
"/home/pietervdvn/Downloads/Scholen/Vestigingsplaatsen-van-scholen-gewoon-secundair-onderwijs-cleaned.csv"]
*/
console.log("Usage: csvToGeojson input.csv name-of-lat-field name-of-lon-field")
console.log("Usage: csvToGeojson input.csv name-of-lat-field name-of-lon-field")
return
}
let file = args[0]
if(file.startsWith("file://")){
let file = args[0]
if (file.startsWith("file://")) {
file = file.substr("file://".length)
}
const latField = args[1]
const lonField = args[2]
const csvOptions = {
columns: true,
skip_empty_lines: true,
trim: true
trim: true,
}
const csv: Record<any, string>[] = parse(readFileSync(file), csvOptions)
const features = csv.map((csvElement, i) => {
const features = csv.map((csvElement, i) => {
const lat = Number(csvElement[latField])
const lon = Number(csvElement[lonField])
if(isNaN(lat) || isNaN(lon)){
throw `Not a valid lat or lon for entry ${i}: ${JSON.stringify(csvElement)}`
}
if (isNaN(lat) || isNaN(lon)) {
throw `Not a valid lat or lon for entry ${i}: ${JSON.stringify(csvElement)}`
}
return {
return {
type: "Feature",
properties: csvElement,
geometry: {
type: "Point",
coordinates: lambert72toWGS84(lon, lat)
}
coordinates: lambert72toWGS84(lon, lat),
},
}
})
console.log(JSON.stringify({
type: "FeatureCollection",
features
}))
console.log(
JSON.stringify({
type: "FeatureCollection",
features,
})
)
}
main(process.argv.slice(2))
main(process.argv.slice(2))

14
scripts/downloadFile.ts
View file

@ -1,12 +1,12 @@
const http = require('https');
const fs = require('fs');
const http = require("https")
const fs = require("fs")
// Could use yargs to have more validation but wanted to keep it simple
const args = process.argv.slice(2);
const FILE_URL = args[0];
const DESTINATION = args[1];
const args = process.argv.slice(2)
const FILE_URL = args[0]
const DESTINATION = args[1]
console.log(`Downloading ${FILE_URL} to ${DESTINATION}`)
const file = fs.createWriteStream(DESTINATION);
http.get(FILE_URL, response => response.pipe(file));
const file = fs.createWriteStream(DESTINATION)
http.get(FILE_URL, (response) => response.pipe(file))

77
scripts/extractBikeRental.ts
View file

@ -1,6 +1,5 @@
import * as fs from "fs";
import {OH} from "../UI/OpeningHours/OpeningHours";
import * as fs from "fs"
import { OH } from "../UI/OpeningHours/OpeningHours"
function extractValue(vs: { __value }[]) {
if (vs === undefined) {
@ -10,12 +9,11 @@ function extractValue(vs: { __value }[]) {
if ((v.__value ?? "") === "") {
continue
}
return v.__value;
return v.__value
}
return undefined
}
function extract_oh_block(days): string {
const oh = []
for (const day of days.day) {
@ -23,7 +21,7 @@ function extract_oh_block(days): string {
const block = day.time_block[0]
const from = block.time_from.substr(0, 5)
const to = block.time_until.substr(0, 5)
const by_appointment = block.by_appointment ? " \"by appointment\"" : ""
const by_appointment = block.by_appointment ? ' "by appointment"' : ""
oh.push(`${abbr} ${from}-${to}${by_appointment}`)
}
return oh.join("; ")
@ -32,7 +30,7 @@ function extract_oh_block(days): string {
function extract_oh(opening_periods) {
const rules = []
if (opening_periods === undefined) {
return undefined;
return undefined
}
for (const openingPeriod of opening_periods.opening_period ?? []) {
let rule = extract_oh_block(openingPeriod.days)
@ -51,17 +49,19 @@ function rewrite(obj, key) {
obj[key] = extractValue(obj[key]["value"])
}
const stuff = fs.readFileSync("/home/pietervdvn/Documents/Freelance/ToerismeVlaanderen 2021-09/TeImporteren/allchannels-bike_rental.json", "UTF8")
const stuff = fs.readFileSync(
"/home/pietervdvn/Documents/Freelance/ToerismeVlaanderen 2021-09/TeImporteren/allchannels-bike_rental.json",
"UTF8"
)
const data: any[] = JSON.parse(stuff)
const results: {
geometry: {
type: "Point",
type: "Point"
coordinates: [number, number]
},
type: "Feature",
}
type: "Feature"
properties: any
}[] = []
const skipped = []
console.log("[")
@ -77,7 +77,11 @@ for (const item of data) {
skipped.push(item)
continue
}
const toDelete = ["id", "uuid", "update_date", "creation_date",
const toDelete = [
"id",
"uuid",
"update_date",
"creation_date",
"deleted",
"aborted",
"partner_id",
@ -85,7 +89,7 @@ for (const item of data) {
"winref",
"winref_uuid",
"root_product_type",
"parent"
"parent",
]
for (const key of toDelete) {
delete metadata[key]
@ -111,7 +115,7 @@ for (const item of data) {
metadata["phone"] = item.contact_info["telephone"] ?? item.contact_info["mobile"]
metadata["email"] = item.contact_info["email_address"]
const links = item.links?.link?.map(l => l.url) ?? []
const links = item.links?.link?.map((l) => l.url) ?? []
metadata["website"] = item.contact_info["website"] ?? links[0]
delete item["links"]
@ -127,7 +131,8 @@ for (const item of data) {
console.error("Unkown product type: ", metadata["touristic_product_type"])
}
const descriptions = item.descriptions?.description?.map(d => extractValue(d?.text?.value)) ?? []
const descriptions =
item.descriptions?.description?.map((d) => extractValue(d?.text?.value)) ?? []
delete item.descriptions
metadata["description"] = metadata["description"] ?? descriptions[0]
if (item.price_info?.prices?.free == true) {
@ -138,28 +143,25 @@ for (const item of data) {
metadata.charge = extractValue(item.price_info?.extra_information?.value)
const methods = item.price_info?.payment_methods?.payment_method
if (methods !== undefined) {
methods.map(v => extractValue(v.value)).forEach(method => {
metadata["payment:" + method.toLowerCase()] = "yes"
})
methods
.map((v) => extractValue(v.value))
.forEach((method) => {
metadata["payment:" + method.toLowerCase()] = "yes"
})
}
delete item.price_info
} else if (item.price_info?.prices?.length === 0) {
delete item.price_info
}
try {
if (item.labels_info?.labels_own?.label[0]?.code === "Billenkar") {
metadata.rental = "quadricycle"
delete item.labels_info
}
} catch (e) {
}
} catch (e) {}
delete item["publishing_channels"]
try {
metadata["image"] = item.media.file[0].url[0]
} catch (e) {
@ -167,7 +169,6 @@ for (const item of data) {
}
delete item.media
const time_info = item.time_info?.time_info_regular
if (time_info?.permantly_open === true) {
metadata.opening_hours = "24/7"
@ -176,7 +177,6 @@ for (const item of data) {
}
delete item.time_info
const properties = {}
for (const key in metadata) {
const v = metadata[key]
@ -189,22 +189,25 @@ for (const item of data) {
results.push({
geometry: {
type: "Point",
coordinates: item.coordinates
coordinates: item.coordinates,
},
type: "Feature",
properties
properties,
})
delete item.coordinates
delete item.properties
console.log(JSON.stringify(item, null, " ") + ",")
}
console.log("]")
fs.writeFileSync("west-vlaanderen.geojson", JSON.stringify(
{
type: "FeatureCollection",
features: results
}
, null, " "
))
fs.writeFileSync(
"west-vlaanderen.geojson",
JSON.stringify(
{
type: "FeatureCollection",
features: results,
},
null,
" "
)
)

53
scripts/extractLayer.ts
View file

@ -1,10 +1,12 @@
import {LayoutConfigJson} from "../Models/ThemeConfig/Json/LayoutConfigJson";
import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
import {existsSync, mkdirSync, readFileSync, writeFileSync} from "fs";
import { LayoutConfigJson } from "../Models/ThemeConfig/Json/LayoutConfigJson"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
function main(args: string[]){
if(args.length === 0){
console.log("Extracts an inline layer from a theme and places it in it's own layer directory.")
function main(args: string[]) {
if (args.length === 0) {
console.log(
"Extracts an inline layer from a theme and places it in it's own layer directory."
)
console.log("USAGE: ts-node scripts/extractLayerFromTheme.ts <themeid> <layerid>")
console.log("(Invoke with only the themename to see which layers can be extracted)")
return
@ -12,43 +14,46 @@ function main(args: string[]){
const themeId = args[0]
const layerId = args[1]
const themePath = "./assets/themes/"+themeId+"/"+themeId+".json"
const contents = <LayoutConfigJson> JSON.parse(readFileSync(themePath, "UTF-8"))
const layers = <LayerConfigJson[]> contents.layers.filter(l => {
if(typeof l === "string"){
const themePath = "./assets/themes/" + themeId + "/" + themeId + ".json"
const contents = <LayoutConfigJson>JSON.parse(readFileSync(themePath, "UTF-8"))
const layers = <LayerConfigJson[]>contents.layers.filter((l) => {
if (typeof l === "string") {
return false
}
if(l["override"] !== undefined){
if (l["override"] !== undefined) {
return false
}
return true
})
if(layers.length === 0){
console.log("No layers can be extracted from this theme. The "+contents.layers.length+" layers are already substituted layers")
if (layers.length === 0) {
console.log(
"No layers can be extracted from this theme. The " +
contents.layers.length +
" layers are already substituted layers"
)
return
}
const layerConfig = layers.find(l => l.id === layerId)
if(layerId === undefined || layerConfig === undefined){
if(layerId !== undefined){
console.error( "Layer "+layerId+" not found as inline layer")
const layerConfig = layers.find((l) => l.id === layerId)
if (layerId === undefined || layerConfig === undefined) {
if (layerId !== undefined) {
console.error("Layer " + layerId + " not found as inline layer")
}
console.log("Layers available for extraction are:")
console.log(layers.map(l => l.id).join("\n"))
console.log(layers.map((l) => l.id).join("\n"))
return
}
const dir = "./assets/layers/"+layerId
if(!existsSync(dir)){
const dir = "./assets/layers/" + layerId
if (!existsSync(dir)) {
mkdirSync(dir)
}
writeFileSync(dir+"/"+layerId+".json", JSON.stringify(layerConfig, null, " "))
writeFileSync(dir + "/" + layerId + ".json", JSON.stringify(layerConfig, null, " "))
const index = contents.layers.findIndex(l => l["id"] === layerId)
const index = contents.layers.findIndex((l) => l["id"] === layerId)
contents.layers[index] = layerId
writeFileSync(themePath, JSON.stringify(contents, null, " "))
}
main(process.argv.slice(2))
main(process.argv.slice(2))

154
scripts/fetchLanguages.ts
View file

@ -3,76 +3,77 @@
*/
import * as wds from "wikidata-sdk"
import {Utils} from "../Utils";
import ScriptUtils from "./ScriptUtils";
import {existsSync, readFileSync, writeFileSync} from "fs";
import {QuestionableTagRenderingConfigJson} from "../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson";
import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
import WikidataUtils from "../Utils/WikidataUtils";
import LanguageUtils from "../Utils/LanguageUtils";
import { Utils } from "../Utils"
import ScriptUtils from "./ScriptUtils"
import { existsSync, readFileSync, writeFileSync } from "fs"
import { QuestionableTagRenderingConfigJson } from "../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import WikidataUtils from "../Utils/WikidataUtils"
import LanguageUtils from "../Utils/LanguageUtils"
async function fetch(target: string){
async function fetch(target: string) {
const regular = await fetchRegularLanguages()
writeFileSync(target, JSON.stringify(regular, null, " "))
console.log("Written to "+target)
console.log("Written to " + target)
}
async function fetchRegularLanguages() {
console.log("Fetching languages")
const sparql = 'SELECT ?lang ?label ?code \n' +
'WHERE \n' +
'{ \n' +
' ?lang wdt:P31 wd:Q1288568. \n' + // language instanceOf (p31) modern language(Q1288568)
' ?lang rdfs:label ?label. \n' +
' ?lang wdt:P424 ?code' + // Wikimedia language code seems to be close to the weblate entries
'} '
const sparql =
"SELECT ?lang ?label ?code \n" +
"WHERE \n" +
"{ \n" +
" ?lang wdt:P31 wd:Q1288568. \n" + // language instanceOf (p31) modern language(Q1288568)
" ?lang rdfs:label ?label. \n" +
" ?lang wdt:P424 ?code" + // Wikimedia language code seems to be close to the weblate entries
"} "
const url = wds.sparqlQuery(sparql)
// request the generated URL with your favorite HTTP request library
const result = await Utils.downloadJson(url, {"User-Agent": "MapComplete script"})
// request the generated URL with your favorite HTTP request library
const result = await Utils.downloadJson(url, { "User-Agent": "MapComplete script" })
const bindings = result.results.bindings
const zh_hant = await fetchSpecial(18130932, "zh_Hant")
const zh_hans = await fetchSpecial(13414913, "zh_Hant")
const pt_br = await fetchSpecial( 750553, "pt_BR")
const fil = await fetchSpecial( 33298, "fil")
const pt_br = await fetchSpecial(750553, "pt_BR")
const fil = await fetchSpecial(33298, "fil")
bindings.push(...zh_hant)
bindings.push(...zh_hans)
bindings.push(...pt_br)
bindings.push(...fil)
return result.results.bindings
return result.results.bindings
}
async function fetchSpecial(id: number, code: string) {
ScriptUtils.fixUtils()
console.log("Fetching languages")
const sparql = 'SELECT ?lang ?label ?code \n' +
'WHERE \n' +
'{ \n' +
' wd:Q'+id+' rdfs:label ?label. \n' +
'} '
const sparql =
"SELECT ?lang ?label ?code \n" +
"WHERE \n" +
"{ \n" +
" wd:Q" +
id +
" rdfs:label ?label. \n" +
"} "
const url = wds.sparqlQuery(sparql)
const result = await Utils.downloadJson(url, {"User-Agent": "MapComplete script"})
const result = await Utils.downloadJson(url, { "User-Agent": "MapComplete script" })
const bindings = result.results.bindings
bindings.forEach(binding => binding["code"] = {value: code})
bindings.forEach((binding) => (binding["code"] = { value: code }))
return bindings
}
function getNativeList(langs: Map<string, Map<string, string>>){
function getNativeList(langs: Map<string, Map<string, string>>) {
const native = {}
const keys: string[] = Array.from(langs.keys())
keys.sort()
for (const key of keys) {
const translations: Map<string, string> = langs.get(key)
if(!LanguageUtils.usedLanguages.has(key)){
if (!LanguageUtils.usedLanguages.has(key)) {
continue
}
native[key] = translations.get(key)
@ -80,8 +81,8 @@ function getNativeList(langs: Map<string, Map<string, string>>){
return native
}
async function getOfficialLanguagesPerCountry() : Promise<Map<string, string[]>>{
const lngs = new Map<string, string[]>();
async function getOfficialLanguagesPerCountry(): Promise<Map<string, string[]>> {
const lngs = new Map<string, string[]>()
const sparql = `SELECT ?country ?countryLabel ?countryCode ?language ?languageCode ?languageLabel
WHERE
{
@ -93,85 +94,88 @@ async function getOfficialLanguagesPerCountry() : Promise<Map<string, string[]>>
}`
const url = wds.sparqlQuery(sparql)
const result = await Utils.downloadJson(url, {"User-Agent": "MapComplete script"})
const bindings : {countryCode: {value: string}, languageCode: {value: string}}[]= result.results.bindings
const result = await Utils.downloadJson(url, { "User-Agent": "MapComplete script" })
const bindings: { countryCode: { value: string }; languageCode: { value: string } }[] =
result.results.bindings
for (const binding of bindings) {
const countryCode = binding.countryCode.value
const language = binding.languageCode.value
if(lngs.get(countryCode) === undefined){
if (lngs.get(countryCode) === undefined) {
lngs.set(countryCode, [])
}
lngs.get(countryCode).push(language)
}
return lngs;
return lngs
}
async function main(wipeCache = false){
async function main(wipeCache = false) {
const cacheFile = "./assets/generated/languages-wd.json"
if(wipeCache || !existsSync(cacheFile)){
if (wipeCache || !existsSync(cacheFile)) {
console.log("Refreshing cache")
await fetch(cacheFile);
}else{
await fetch(cacheFile)
} else {
console.log("Reusing the cached file")
}
const data = JSON.parse(readFileSync( cacheFile, "UTF8"))
const data = JSON.parse(readFileSync(cacheFile, "UTF8"))
const perId = WikidataUtils.extractLanguageData(data, WikidataUtils.languageRemapping)
const nativeList = getNativeList(perId)
writeFileSync("./assets/language_native.json", JSON.stringify(nativeList, null, " "))
const translations = Utils.MapToObj(perId, (value, key) => {
if(!LanguageUtils.usedLanguages.has(key)){
if (!LanguageUtils.usedLanguages.has(key)) {
return undefined // Remove unused languages
}
return Utils.MapToObj(value, (v, k ) => {
if(!LanguageUtils.usedLanguages.has(k)){
return Utils.MapToObj(value, (v, k) => {
if (!LanguageUtils.usedLanguages.has(k)) {
return undefined
}
return v
})
})
writeFileSync("./assets/language_translations.json",
JSON.stringify(translations, null, " "))
let officialLanguages : Record<string, string[]>;
writeFileSync("./assets/language_translations.json", JSON.stringify(translations, null, " "))
let officialLanguages: Record<string, string[]>
const officialLanguagesPath = "./assets/language_in_country.json"
if(existsSync("./assets/languages_in_country.json") && !wipeCache){
if (existsSync("./assets/languages_in_country.json") && !wipeCache) {
officialLanguages = JSON.parse(readFileSync(officialLanguagesPath, "utf8"))
}else {
officialLanguages = Utils.MapToObj(await getOfficialLanguagesPerCountry(), t => t)
} else {
officialLanguages = Utils.MapToObj(await getOfficialLanguagesPerCountry(), (t) => t)
writeFileSync(officialLanguagesPath, JSON.stringify(officialLanguages, null, " "))
}
const perLanguage = Utils.TransposeMap(officialLanguages);
const perLanguage = Utils.TransposeMap(officialLanguages)
console.log(JSON.stringify(perLanguage, null, " "))
const mappings: {if: string, then: Record<string, string>, hideInAnswer: string}[] = []
const mappings: { if: string; then: Record<string, string>; hideInAnswer: string }[] = []
for (const language of Object.keys(perLanguage)) {
const countries = Utils.Dedup(perLanguage[language].map(c => c.toLowerCase()))
const countries = Utils.Dedup(perLanguage[language].map((c) => c.toLowerCase()))
mappings.push({
if: "language="+language,
if: "language=" + language,
then: translations[language],
hideInAnswer : "_country="+countries.join("|")
hideInAnswer: "_country=" + countries.join("|"),
})
}
const tagRenderings = <QuestionableTagRenderingConfigJson> {
const tagRenderings = <QuestionableTagRenderingConfigJson>{
id: "official-language",
mappings,
question: "What languages are spoken here?"
question: "What languages are spoken here?",
}
writeFileSync("./assets/layers/language/language.json", JSON.stringify(<LayerConfigJson>{
id:"language",
description: "Various tagRenderings to help language tooling",
tagRenderings
}, null, " "))
writeFileSync(
"./assets/layers/language/language.json",
JSON.stringify(
<LayerConfigJson>{
id: "language",
description: "Various tagRenderings to help language tooling",
tagRenderings,
},
null,
" "
)
)
}
const forceRefresh = process.argv[2] === "--force-refresh"
ScriptUtils.fixUtils()
main(forceRefresh).then(() => console.log("Done!"))

48
scripts/filter.ts
View file

@ -1,11 +1,13 @@
import * as fs from "fs";
import {TagUtils} from "../Logic/Tags/TagUtils";
import {writeFileSync} from "fs";
import {TagsFilter} from "../Logic/Tags/TagsFilter";
import * as fs from "fs"
import { TagUtils } from "../Logic/Tags/TagUtils"
import { writeFileSync } from "fs"
import { TagsFilter } from "../Logic/Tags/TagsFilter"
function main(args) {
if (args.length < 2) {
console.log("Given a single geojson file and a filter specification, will print all the entries to std-out which pass the property")
console.log(
"Given a single geojson file and a filter specification, will print all the entries to std-out which pass the property"
)
console.log("USAGE: perProperty `file.geojson` `key=value` [outputfile]")
return
}
@ -14,32 +16,38 @@ function main(args) {
const output = args[2]
const data = JSON.parse(fs.readFileSync(path, "UTF8"))
let filter : TagsFilter ;
try{
filter = TagUtils.Tag(JSON.parse(spec))
}catch(e){
let filter: TagsFilter
try {
filter = TagUtils.Tag(JSON.parse(spec))
} catch (e) {
filter = TagUtils.Tag(spec)
}
const features = data.features.filter(f => filter.matchesProperties(f.properties))
const features = data.features.filter((f) => filter.matchesProperties(f.properties))
if(features.length === 0){
if (features.length === 0) {
console.log("Warning: no features matched the filter. Exiting now")
return
}
const collection = {
type:"FeatureCollection",
features
type: "FeatureCollection",
features,
}
const stringified = JSON.stringify(collection, null, " ")
if(output === undefined){
if (output === undefined) {
console.log(stringified)
}else{
console.log("Filtered "+path+": kept "+features.length+" out of "+data.features.length+" objects")
} else {
console.log(
"Filtered " +
path +
": kept " +
features.length +
" out of " +
data.features.length +
" objects"
)
writeFileSync(output, stringified)
}
}
main(process.argv.slice(2))
main(process.argv.slice(2))

78
scripts/fixImagesInTagRenderings.ts
View file

@ -1,49 +1,56 @@
import {readFileSync, writeFileSync} from "fs";
import {DesugaringStep} from "../Models/ThemeConfig/Conversion/Conversion";
import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
import {Utils} from "../Utils";
import Translations from "../UI/i18n/Translations";
import { readFileSync, writeFileSync } from "fs"
import { DesugaringStep } from "../Models/ThemeConfig/Conversion/Conversion"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import { Utils } from "../Utils"
import Translations from "../UI/i18n/Translations"
class ConvertImagesToIcon extends DesugaringStep<LayerConfigJson> {
private _iconClass: string;
private _iconClass: string
constructor(iconClass: string) {
super("Searches for images in the 'then' path, removes the <img> block and extracts the image itself a 'icon'",
[], "ConvertImagesToIcon")
this._iconClass = iconClass;
super(
"Searches for images in the 'then' path, removes the <img> block and extracts the image itself a 'icon'",
[],
"ConvertImagesToIcon"
)
this._iconClass = iconClass
}
convert(json: LayerConfigJson, context: string): { result: LayerConfigJson; errors?: string[]; warnings?: string[]; information?: string[] } {
convert(
json: LayerConfigJson,
context: string
): { result: LayerConfigJson; errors?: string[]; warnings?: string[]; information?: string[] } {
const information = []
const errors = []
json = Utils.Clone(json)
Utils.WalkPath(
["tagRenderings", "mappings"],
json,
mapping => {
const then = Translations.T(mapping.then)
const images = Utils.Dedup(then.ExtractImages())
if (images.length == 0) {
return mapping
}
if (images.length > 1) {
errors.push("The mapping " + mapping.then + " has multiple images: " + images.join(", "))
}
information.push("Replaced image " + images[0])
const replaced = then.OnEveryLanguage((s) => {
return s.replace(/(<div [^>]*>)?<img [^>]*> ?/, "").replace(/<\/div>$/, "").trim()
})
mapping.then = replaced.translations
mapping.icon = {path: images[0], class: this._iconClass}
Utils.WalkPath(["tagRenderings", "mappings"], json, (mapping) => {
const then = Translations.T(mapping.then)
const images = Utils.Dedup(then.ExtractImages())
if (images.length == 0) {
return mapping
}
)
if (images.length > 1) {
errors.push(
"The mapping " + mapping.then + " has multiple images: " + images.join(", ")
)
}
information.push("Replaced image " + images[0])
const replaced = then.OnEveryLanguage((s) => {
return s
.replace(/(<div [^>]*>)?<img [^>]*> ?/, "")
.replace(/<\/div>$/, "")
.trim()
})
mapping.then = replaced.translations
mapping.icon = { path: images[0], class: this._iconClass }
return mapping
})
return {
information,
result: json
};
result: json,
}
}
}
@ -57,9 +64,12 @@ function main() {
const iconClass = args[1] ?? "small"
const targetFile = args[2] ?? path + ".autoconverted.json"
const parsed = JSON.parse(readFileSync(path, "UTF8"))
const converted = new ConvertImagesToIcon(iconClass).convertStrict(parsed, "While running the fixImagesInTagRenderings-script")
const converted = new ConvertImagesToIcon(iconClass).convertStrict(
parsed,
"While running the fixImagesInTagRenderings-script"
)
writeFileSync(targetFile, JSON.stringify(converted, null, " "))
console.log("Written fixed version to " + targetFile)
}
main();
main()

79
scripts/fixSchemas.ts
View file

@ -1,21 +1,25 @@
import ScriptUtils from "./ScriptUtils";
import {readFileSync, writeFileSync} from "fs";
import ScriptUtils from "./ScriptUtils"
import { readFileSync, writeFileSync } from "fs"
/**
* Extracts the data from the scheme file and writes them in a flatter structure
*/
export type JsonSchemaType = string | {$ref: string, description: string} | {type: string} | JsonSchemaType[]
export type JsonSchemaType =
| string
| { $ref: string; description: string }
| { type: string }
| JsonSchemaType[]
export interface JsonSchema {
description?: string,
type?: JsonSchemaType,
properties?: any,
items?: JsonSchema,
allOf?: JsonSchema[],
anyOf: JsonSchema[],
enum: JsonSchema[],
"$ref": string
description?: string
type?: JsonSchemaType
properties?: any
items?: JsonSchema
allOf?: JsonSchema[]
anyOf: JsonSchema[]
enum: JsonSchema[]
$ref: string
}
function WalkScheme<T>(
@ -24,9 +28,8 @@ function WalkScheme<T>(
fullScheme: JsonSchema & { definitions?: any } = undefined,
path: string[] = [],
isHandlingReference = []
): { path: string[], t: T }[] {
const results: { path: string[], t: T } [] = []
): { path: string[]; t: T }[] {
const results: { path: string[]; t: T }[] = []
if (scheme === undefined) {
return []
}
@ -39,10 +42,13 @@ function WalkScheme<T>(
}
const definitionName = ref.substr(prefix.length)
if (isHandlingReference.indexOf(definitionName) >= 0) {
return;
return
}
const loadedScheme = fullScheme.definitions[definitionName]
return WalkScheme(onEach, loadedScheme, fullScheme, path, [...isHandlingReference, definitionName]);
return WalkScheme(onEach, loadedScheme, fullScheme, path, [
...isHandlingReference,
definitionName,
])
}
fullScheme = fullScheme ?? scheme
@ -50,11 +56,10 @@ function WalkScheme<T>(
if (t !== undefined) {
results.push({
path,
t
t,
})
}
function walk(v: JsonSchema) {
if (v === undefined) {
return
@ -67,11 +72,9 @@ function WalkScheme<T>(
return
}
scheme.forEach(v => walk(v))
scheme.forEach((v) => walk(v))
}
{
walkEach(scheme.enum)
walkEach(scheme.anyOf)
@ -85,7 +88,9 @@ function WalkScheme<T>(
for (const key in scheme.properties) {
const prop = scheme.properties[key]
results.push(...WalkScheme(onEach, prop, fullScheme, [...path, key], isHandlingReference))
results.push(
...WalkScheme(onEach, prop, fullScheme, [...path, key], isHandlingReference)
)
}
}
@ -93,30 +98,31 @@ function WalkScheme<T>(
}
function extractMeta(typename: string, path: string) {
const themeSchema = JSON.parse(readFileSync("./Docs/Schemas/" + typename + ".schema.json", "UTF-8"))
const themeSchema = JSON.parse(
readFileSync("./Docs/Schemas/" + typename + ".schema.json", "UTF-8")
)
const withTypes = WalkScheme((schemePart) => {
if (schemePart.description === undefined) {
return;
return
}
const typeHint = schemePart.description.split("\n")
.find(line => line.trim().toLocaleLowerCase().startsWith("type:"))
?.substr("type:".length)?.trim()
const type = schemePart.items?.anyOf ?? schemePart.type ?? schemePart.anyOf;
return {typeHint, type, description: schemePart.description}
const typeHint = schemePart.description
.split("\n")
.find((line) => line.trim().toLocaleLowerCase().startsWith("type:"))
?.substr("type:".length)
?.trim()
const type = schemePart.items?.anyOf ?? schemePart.type ?? schemePart.anyOf
return { typeHint, type, description: schemePart.description }
}, themeSchema)
const paths = withTypes.map(({
path,
t
}) => ({path, ...t}))
const paths = withTypes.map(({ path, t }) => ({ path, ...t }))
writeFileSync("./assets/" + path + ".json", JSON.stringify(paths, null, " "))
console.log("Written meta to ./assets/" + path)
}
function main() {
const allSchemas = ScriptUtils.readDirRecSync("./Docs/Schemas").filter(pth => pth.endsWith("JSC.ts"))
const allSchemas = ScriptUtils.readDirRecSync("./Docs/Schemas").filter((pth) =>
pth.endsWith("JSC.ts")
)
for (const path of allSchemas) {
const dir = path.substring(0, path.lastIndexOf("/"))
const name = path.substring(path.lastIndexOf("/"), path.length - "JSC.ts".length)
@ -137,7 +143,6 @@ function main() {
extractMeta("LayoutConfigJson", "layoutconfigmeta")
extractMeta("TagRenderingConfigJson", "tagrenderingconfigmeta")
extractMeta("QuestionableTagRenderingConfigJson", "questionabletagrenderingconfigmeta")
}
main()

376
scripts/generateCache.ts
View file

@ -1,56 +1,59 @@
/**
* Generates a collection of geojson files based on an overpass query for a given theme
*/
import {Utils} from "../Utils";
import {Overpass} from "../Logic/Osm/Overpass";
import {existsSync, readFileSync, writeFileSync} from "fs";
import {TagsFilter} from "../Logic/Tags/TagsFilter";
import {Or} from "../Logic/Tags/Or";
import {AllKnownLayouts} from "../Customizations/AllKnownLayouts";
import RelationsTracker from "../Logic/Osm/RelationsTracker";
import * as OsmToGeoJson from "osmtogeojson";
import MetaTagging from "../Logic/MetaTagging";
import {ImmutableStore, UIEventSource} from "../Logic/UIEventSource";
import {TileRange, Tiles} from "../Models/TileRange";
import LayoutConfig from "../Models/ThemeConfig/LayoutConfig";
import ScriptUtils from "./ScriptUtils";
import PerLayerFeatureSourceSplitter from "../Logic/FeatureSource/PerLayerFeatureSourceSplitter";
import FilteredLayer from "../Models/FilteredLayer";
import FeatureSource, {FeatureSourceForLayer} from "../Logic/FeatureSource/FeatureSource";
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource";
import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource";
import Constants from "../Models/Constants";
import {GeoOperations} from "../Logic/GeoOperations";
import SimpleMetaTaggers from "../Logic/SimpleMetaTagger";
import FilteringFeatureSource from "../Logic/FeatureSource/Sources/FilteringFeatureSource";
import Loc from "../Models/Loc";
import { Utils } from "../Utils"
import { Overpass } from "../Logic/Osm/Overpass"
import { existsSync, readFileSync, writeFileSync } from "fs"
import { TagsFilter } from "../Logic/Tags/TagsFilter"
import { Or } from "../Logic/Tags/Or"
import { AllKnownLayouts } from "../Customizations/AllKnownLayouts"
import RelationsTracker from "../Logic/Osm/RelationsTracker"
import * as OsmToGeoJson from "osmtogeojson"
import MetaTagging from "../Logic/MetaTagging"
import { ImmutableStore, UIEventSource } from "../Logic/UIEventSource"
import { TileRange, Tiles } from "../Models/TileRange"
import LayoutConfig from "../Models/ThemeConfig/LayoutConfig"
import ScriptUtils from "./ScriptUtils"
import PerLayerFeatureSourceSplitter from "../Logic/FeatureSource/PerLayerFeatureSourceSplitter"
import FilteredLayer from "../Models/FilteredLayer"
import FeatureSource, { FeatureSourceForLayer } from "../Logic/FeatureSource/FeatureSource"
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource"
import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource"
import Constants from "../Models/Constants"
import { GeoOperations } from "../Logic/GeoOperations"
import SimpleMetaTaggers from "../Logic/SimpleMetaTagger"
import FilteringFeatureSource from "../Logic/FeatureSource/Sources/FilteringFeatureSource"
import Loc from "../Models/Loc"
ScriptUtils.fixUtils()
function createOverpassObject(theme: LayoutConfig, relationTracker: RelationsTracker, backend: string) {
let filters: TagsFilter[] = [];
let extraScripts: string[] = [];
function createOverpassObject(
theme: LayoutConfig,
relationTracker: RelationsTracker,
backend: string
) {
let filters: TagsFilter[] = []
let extraScripts: string[] = []
for (const layer of theme.layers) {
if (typeof (layer) === "string") {
if (typeof layer === "string") {
throw "A layer was not expanded!"
}
if (layer.doNotDownload) {
continue;
continue
}
if (layer.source.geojsonSource !== undefined) {
// This layer defines a geoJson-source
// SHould it be cached?
if (layer.source.isOsmCacheLayer !== true) {
continue;
continue
}
}
// Check if data for this layer has already been loaded
if (layer.source.overpassScript !== undefined) {
extraScripts.push(layer.source.overpassScript)
} else {
filters.push(layer.source.osmTags);
filters.push(layer.source.osmTags)
}
}
filters = Utils.NoNull(filters)
@ -58,8 +61,13 @@ function createOverpassObject(theme: LayoutConfig, relationTracker: RelationsTra
if (filters.length + extraScripts.length === 0) {
throw "Nothing to download! The theme doesn't declare anything to download"
}
return new Overpass(new Or(filters), extraScripts, backend,
new UIEventSource<number>(60), relationTracker);
return new Overpass(
new Or(filters),
extraScripts,
backend,
new UIEventSource<number>(60),
relationTracker
)
}
function rawJsonName(targetDir: string, x: number, y: number, z: number): string {
@ -71,102 +79,143 @@ function geoJsonName(targetDir: string, x: number, y: number, z: number): string
}
/// Downloads the given feature and saves them to disk
async function downloadRaw(targetdir: string, r: TileRange, theme: LayoutConfig, relationTracker: RelationsTracker)/* : {failed: number, skipped :number} */ {
async function downloadRaw(
targetdir: string,
r: TileRange,
theme: LayoutConfig,
relationTracker: RelationsTracker
) /* : {failed: number, skipped :number} */ {
let downloaded = 0
let failed = 0
let skipped = 0
const startTime = new Date().getTime()
for (let x = r.xstart; x <= r.xend; x++) {
for (let y = r.ystart; y <= r.yend; y++) {
downloaded++;
downloaded++
const filename = rawJsonName(targetdir, x, y, r.zoomlevel)
if (existsSync(filename)) {
console.log("Already exists (not downloading again): ", filename)
skipped++
continue;
continue
}
const runningSeconds = (new Date().getTime() - startTime) / 1000
const resting = failed + (r.total - downloaded)
const perTile = (runningSeconds / (downloaded - skipped))
const perTile = runningSeconds / (downloaded - skipped)
const estimated = Math.floor(resting * perTile)
console.log("total: ", downloaded, "/", r.total, "failed: ", failed, "skipped: ", skipped, "running time: ", Utils.toHumanTime(runningSeconds) + "s", "estimated left: ", Utils.toHumanTime(estimated), "(" + Math.floor(perTile) + "s/tile)")
console.log(
"total: ",
downloaded,
"/",
r.total,
"failed: ",
failed,
"skipped: ",
skipped,
"running time: ",
Utils.toHumanTime(runningSeconds) + "s",
"estimated left: ",
Utils.toHumanTime(estimated),
"(" + Math.floor(perTile) + "s/tile)"
)
const boundsArr = Tiles.tile_bounds(r.zoomlevel, x, y)
const bounds = {
north: Math.max(boundsArr[0][0], boundsArr[1][0]),
south: Math.min(boundsArr[0][0], boundsArr[1][0]),
east: Math.max(boundsArr[0][1], boundsArr[1][1]),
west: Math.min(boundsArr[0][1], boundsArr[1][1])
west: Math.min(boundsArr[0][1], boundsArr[1][1]),
}
const overpass = createOverpassObject(theme, relationTracker, Constants.defaultOverpassUrls[(failed) % Constants.defaultOverpassUrls.length])
const url = overpass.buildQuery("[bbox:" + bounds.south + "," + bounds.west + "," + bounds.north + "," + bounds.east + "]")
const overpass = createOverpassObject(
theme,
relationTracker,
Constants.defaultOverpassUrls[failed % Constants.defaultOverpassUrls.length]
)
const url = overpass.buildQuery(
"[bbox:" +
bounds.south +
"," +
bounds.west +
"," +
bounds.north +
"," +
bounds.east +
"]"
)
try {
const json = await Utils.downloadJson(url)
if ((<string>json.remark ?? "").startsWith("runtime error")) {
console.error("Got a runtime error: ", json.remark)
failed++;
failed++
} else if (json.elements.length === 0) {
console.log("Got an empty response! Writing anyway")
}
console.log("Got the response - writing ",json.elements.length," elements to ", filename)
writeFileSync(filename, JSON.stringify(json, null, " "));
console.log(
"Got the response - writing ",
json.elements.length,
" elements to ",
filename
)
writeFileSync(filename, JSON.stringify(json, null, " "))
} catch (err) {
console.log(url)
console.log("Could not download - probably hit the rate limit; waiting a bit. (" + err + ")")
failed++;
console.log(
"Could not download - probably hit the rate limit; waiting a bit. (" + err + ")"
)
failed++
await ScriptUtils.sleep(1000)
}
}
}
return {failed: failed, skipped: skipped}
return { failed: failed, skipped: skipped }
}
/*
/*
* Downloads extra geojson sources and returns the features.
* Extra geojson layers should not be tiled
*/
async function downloadExtraData(theme: LayoutConfig)/* : any[] */ {
async function downloadExtraData(theme: LayoutConfig) /* : any[] */ {
const allFeatures: any[] = []
for (const layer of theme.layers) {
const source = layer.source.geojsonSource;
const source = layer.source.geojsonSource
if (source === undefined) {
continue;
continue
}
if (layer.source.isOsmCacheLayer !== undefined && layer.source.isOsmCacheLayer !== false) {
// Cached layers are not considered here
continue;
continue
}
console.log("Downloading extra data: ", source)
await Utils.downloadJson(source).then(json => allFeatures.push(...json.features))
await Utils.downloadJson(source).then((json) => allFeatures.push(...json.features))
}
return allFeatures;
return allFeatures
}
function loadAllTiles(targetdir: string, r: TileRange, theme: LayoutConfig, extraFeatures: any[]): FeatureSource {
function loadAllTiles(
targetdir: string,
r: TileRange,
theme: LayoutConfig,
extraFeatures: any[]
): FeatureSource {
let allFeatures = [...extraFeatures]
let processed = 0;
let processed = 0
for (let x = r.xstart; x <= r.xend; x++) {
for (let y = r.ystart; y <= r.yend; y++) {
processed++;
processed++
const filename = rawJsonName(targetdir, x, y, r.zoomlevel)
console.log(" Loading and processing", processed, "/", r.total, filename)
if (!existsSync(filename)) {
console.error("Not found - and not downloaded. Run this script again!: " + filename)
continue;
continue
}
// We read the raw OSM-file and convert it to a geojson
const rawOsm = JSON.parse(readFileSync(filename, "UTF8"))
// Create and save the geojson file - which is the main chunk of the data
const geojson = OsmToGeoJson.default(rawOsm);
const geojson = OsmToGeoJson.default(rawOsm)
console.log(" which as", geojson.features.length, "features")
allFeatures.push(...geojson.features)
@ -178,18 +227,24 @@ function loadAllTiles(targetdir: string, r: TileRange, theme: LayoutConfig, extr
/**
* Load all the tiles into memory from disk
*/
function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relationsTracker: RelationsTracker, targetdir: string, pointsOnlyLayers: string[]) {
function sliceToTiles(
allFeatures: FeatureSource,
theme: LayoutConfig,
relationsTracker: RelationsTracker,
targetdir: string,
pointsOnlyLayers: string[]
) {
const skippedLayers = new Set<string>()
const indexedFeatures: Map<string, any> = new Map<string, any>()
let indexisBuilt = false;
let indexisBuilt = false
function buildIndex() {
for (const ff of allFeatures.features.data) {
const f = ff.feature
indexedFeatures.set(f.properties.id, f)
}
indexisBuilt = true;
indexisBuilt = true
}
function getFeatureById(id) {
@ -200,38 +255,49 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
}
async function handleLayer(source: FeatureSourceForLayer) {
const layer = source.layer.layerDef;
const layer = source.layer.layerDef
const targetZoomLevel = layer.source.geojsonZoomLevel ?? 0
const layerId = layer.id
if (layer.source.isOsmCacheLayer !== true) {
console.log("Skipping layer ", layerId, ": not a caching layer")
skippedLayers.add(layer.id)
return;
return
}
console.log("Handling layer ", layerId, "which has", source.features.data.length, "features")
console.log(
"Handling layer ",
layerId,
"which has",
source.features.data.length,
"features"
)
if (source.features.data.length === 0) {
return;
return
}
MetaTagging.addMetatags(source.features.data,
MetaTagging.addMetatags(
source.features.data,
{
memberships: relationsTracker,
getFeaturesWithin: _ => {
return [allFeatures.features.data.map(f => f.feature)]
getFeaturesWithin: (_) => {
return [allFeatures.features.data.map((f) => f.feature)]
},
getFeatureById: getFeatureById
getFeatureById: getFeatureById,
},
layer,
{},
{
includeDates: false,
includeNonDates: true,
evaluateStrict: true
});
evaluateStrict: true,
}
)
while (SimpleMetaTaggers.country.runningTasks.size > 0) {
console.log("Still waiting for ", SimpleMetaTaggers.country.runningTasks.size, " features which don't have a country yet")
console.log(
"Still waiting for ",
SimpleMetaTaggers.country.runningTasks.size,
" features which don't have a country yet"
)
await ScriptUtils.sleep(1)
}
@ -242,25 +308,36 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
minZoomLevel: targetZoomLevel,
maxZoomLevel: targetZoomLevel,
maxFeatureCount: undefined,
registerTile: tile => {
const tileIndex = tile.tileIndex;
registerTile: (tile) => {
const tileIndex = tile.tileIndex
console.log("Got tile:", tileIndex, tile.layer.layerDef.id)
if (tile.features.data.length === 0) {
return
}
const filteredTile = new FilteringFeatureSource({
const filteredTile = new FilteringFeatureSource(
{
locationControl: new ImmutableStore<Loc>(undefined),
allElements: undefined,
selectedElement: new ImmutableStore<any>(undefined),
globalFilters: new ImmutableStore([])
globalFilters: new ImmutableStore([]),
},
tileIndex,
tile,
new UIEventSource<any>(undefined)
)
console.log("Tile " + layer.id + "." + tileIndex + " contains " + filteredTile.features.data.length + " features after filtering (" + tile.features.data.length + ") features before")
console.log(
"Tile " +
layer.id +
"." +
tileIndex +
" contains " +
filteredTile.features.data.length +
" features after filtering (" +
tile.features.data.length +
") features before"
)
if (filteredTile.features.data.length === 0) {
return
}
@ -271,26 +348,37 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
delete feature.feature["bbox"]
if (tile.layer.layerDef.calculatedTags !== undefined) {
// Evaluate all the calculated tags strictly
const calculatedTagKeys = tile.layer.layerDef.calculatedTags.map(ct => ct[0])
const calculatedTagKeys = tile.layer.layerDef.calculatedTags.map(
(ct) => ct[0]
)
featureCount++
const props = feature.feature.properties
for (const calculatedTagKey of calculatedTagKeys) {
const strict = props[calculatedTagKey]
if(props.hasOwnProperty(calculatedTagKey)){
if (props.hasOwnProperty(calculatedTagKey)) {
delete props[calculatedTagKey]
}
props[calculatedTagKey] = strict
strictlyCalculated++;
strictlyCalculated++
if (strictlyCalculated % 100 === 0) {
console.log("Strictly calculated ", strictlyCalculated, "values for tile", tileIndex, ": now at ", featureCount, "/", filteredTile.features.data.length, "examle value: ", strict)
console.log(
"Strictly calculated ",
strictlyCalculated,
"values for tile",
tileIndex,
": now at ",
featureCount,
"/",
filteredTile.features.data.length,
"examle value: ",
strict
)
}
}
}
}
// Lets save this tile!
const [z, x, y] = Tiles.tile_from_index(tileIndex)
@ -298,78 +386,100 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
const targetPath = geoJsonName(targetdir + "_" + layerId, x, y, z)
createdTiles.push(tileIndex)
// This is the geojson file containing all features for this tile
writeFileSync(targetPath, JSON.stringify({
type: "FeatureCollection",
features: filteredTile.features.data.map(f => f.feature)
}, null, " "))
writeFileSync(
targetPath,
JSON.stringify(
{
type: "FeatureCollection",
features: filteredTile.features.data.map((f) => f.feature),
},
null,
" "
)
)
console.log("Written tile", targetPath, "with", filteredTile.features.data.length)
}
},
})
// All the tiles are written at this point
// Only thing left to do is to create the index
const path = targetdir + "_" + layerId + "_" + targetZoomLevel + "_overview.json"
const perX = {}
createdTiles.map(i => Tiles.tile_from_index(i)).forEach(([z, x, y]) => {
const key = "" + x
if (perX[key] === undefined) {
perX[key] = []
}
perX[key].push(y)
})
createdTiles
.map((i) => Tiles.tile_from_index(i))
.forEach(([z, x, y]) => {
const key = "" + x
if (perX[key] === undefined) {
perX[key] = []
}
perX[key].push(y)
})
console.log("Written overview: ", path, "with ", createdTiles.length, "tiles")
writeFileSync(path, JSON.stringify(perX))
// And, if needed, to create a points-only layer
if (pointsOnlyLayers.indexOf(layer.id) >= 0) {
const filtered = new FilteringFeatureSource({
const filtered = new FilteringFeatureSource(
{
locationControl: new ImmutableStore<Loc>(undefined),
allElements: undefined,
selectedElement: new ImmutableStore<any>(undefined),
globalFilters: new ImmutableStore([])
globalFilters: new ImmutableStore([]),
},
Tiles.tile_index(0, 0, 0),
source,
new UIEventSource<any>(undefined)
)
const features = filtered.features.data.map(f => f.feature)
const features = filtered.features.data.map((f) => f.feature)
const points = features.map(feature => GeoOperations.centerpoint(feature))
const points = features.map((feature) => GeoOperations.centerpoint(feature))
console.log("Writing points overview for ", layerId)
const targetPath = targetdir + "_" + layerId + "_points.geojson"
// This is the geojson file containing all features for this tile
writeFileSync(targetPath, JSON.stringify({
type: "FeatureCollection",
features: points
}, null, " "))
writeFileSync(
targetPath,
JSON.stringify(
{
type: "FeatureCollection",
features: points,
},
null,
" "
)
)
}
}
new PerLayerFeatureSourceSplitter(
new UIEventSource<FilteredLayer[]>(theme.layers.map(l => ({
layerDef: l,
isDisplayed: new UIEventSource<boolean>(true),
appliedFilters: new UIEventSource(undefined)
}))),
new UIEventSource<FilteredLayer[]>(
theme.layers.map((l) => ({
layerDef: l,
isDisplayed: new UIEventSource<boolean>(true),
appliedFilters: new UIEventSource(undefined),
}))
),
handleLayer,
allFeatures
)
const skipped = Array.from(skippedLayers)
if (skipped.length > 0) {
console.warn("Did not save any cache files for layers " + skipped.join(", ") + " as these didn't set the flag `isOsmCache` to true")
console.warn(
"Did not save any cache files for layers " +
skipped.join(", ") +
" as these didn't set the flag `isOsmCache` to true"
)
}
}
export async function main(args: string[]) {
console.log("Cache builder started with args ", args.join(", "))
if (args.length < 6) {
console.error("Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1 [--generate-point-overview layer-name,layer-name,...] [--force-zoom-level z] \n" +
"Note: a new directory named <theme> will be created in targetdirectory")
return;
console.error(
"Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1 [--generate-point-overview layer-name,layer-name,...] [--force-zoom-level z] \n" +
"Note: a new directory named <theme> will be created in targetdirectory"
)
return
}
const themeName = args[0]
const zoomlevel = Number(args[1])
@ -400,7 +510,6 @@ export async function main(args: string[]) {
throw "The fourth number (a longitude) is not a valid number"
}
const tileRange = Tiles.TileRangeBetween(zoomlevel, lat0, lon0, lat1, lon1)
if (isNaN(tileRange.total)) {
@ -418,7 +527,7 @@ export async function main(args: string[]) {
AllKnownLayouts.allKnownLayouts.forEach((_, key) => {
keys.push(key)
})
console.error("The theme " + theme + " was not found; try one of ", keys);
console.error("The theme " + theme + " was not found; try one of ", keys)
return
}
@ -426,15 +535,17 @@ export async function main(args: string[]) {
if (args[7] == "--generate-point-overview") {
if (args[8] === undefined) {
throw "--generate-point-overview needs a list of layers to generate the overview for (or * for all)"
} else if (args[8] === '*') {
generatePointLayersFor = theme.layers.map(l => l.id)
} else if (args[8] === "*") {
generatePointLayersFor = theme.layers.map((l) => l.id)
} else {
generatePointLayersFor = args[8].split(",")
}
console.log("Also generating a point overview for layers ", generatePointLayersFor.join(","))
console.log(
"Also generating a point overview for layers ",
generatePointLayersFor.join(",")
)
}
{
const index = args.indexOf("--force-zoom-level")
if (index >= 0) {
const forcedZoomLevel = Number(args[index + 1])
@ -446,10 +557,9 @@ export async function main(args: string[]) {
}
}
const relationTracker = new RelationsTracker()
let failed = 0;
let failed = 0
do {
const cachingResult = await downloadRaw(targetdir, tileRange, theme, relationTracker)
failed = cachingResult.failed
@ -458,20 +568,18 @@ export async function main(args: string[]) {
}
} while (failed > 0)
const extraFeatures = await downloadExtraData(theme);
const extraFeatures = await downloadExtraData(theme)
const allFeaturesSource = loadAllTiles(targetdir, tileRange, theme, extraFeatures)
sliceToTiles(allFeaturesSource, theme, relationTracker, targetdir, generatePointLayersFor)
}
let args = [...process.argv]
if (!args[1]?.endsWith("test/TestAll.ts")) {
args.splice(0, 2)
try {
main(args)
.then(() => console.log("All done!"))
.catch(e => console.error("Error building cache:", e));
.catch((e) => console.error("Error building cache:", e))
} catch (e) {
console.error("Error building cache:", e)
}

43
scripts/generateContributors.ts
View file

@ -1,44 +1,49 @@
import {exec} from "child_process";
import {writeFile, writeFileSync} from "fs";
import { exec } from "child_process"
import { writeFile, writeFileSync } from "fs"
function asList(hist: Map<string, number>): {contributors: { contributor: string, commits: number }[]
}{
function asList(hist: Map<string, number>): {
contributors: { contributor: string; commits: number }[]
} {
const ls = []
hist.forEach((commits, contributor) => {
ls.push({commits, contributor})
ls.push({ commits, contributor })
})
ls.sort((a, b) => (b.commits - a.commits))
return {contributors: ls}
ls.sort((a, b) => b.commits - a.commits)
return { contributors: ls }
}
function main() {
exec("git log --pretty='%aN %%!%% %s' ", ((error, stdout, stderr) => {
const entries = stdout.split("\n").filter(str => str !== "")
exec("git log --pretty='%aN %%!%% %s' ", (error, stdout, stderr) => {
const entries = stdout.split("\n").filter((str) => str !== "")
const codeContributors = new Map<string, number>()
const translationContributors = new Map<string, number>()
for (const entry of entries) {
console.log(entry)
let [author, message] = entry.split("%!%").map(s => s.trim())
if(author === "Weblate"){
let [author, message] = entry.split("%!%").map((s) => s.trim())
if (author === "Weblate") {
continue
}
if (author === "pietervdvn") {
author = "Pieter Vander Vennet"
}
let hist = codeContributors;
if (message.startsWith("Translated using Weblate") || message.startsWith("Translated using Hosted Weblate")) {
let hist = codeContributors
if (
message.startsWith("Translated using Weblate") ||
message.startsWith("Translated using Hosted Weblate")
) {
hist = translationContributors
}
hist.set(author, 1 + (hist.get(author) ?? 0))
}
const codeContributorsTarget = "assets/contributors.json"
writeFileSync(codeContributorsTarget, JSON.stringify(asList(codeContributors), null, " "))
const translatorsTarget = "assets/translators.json"
writeFileSync(translatorsTarget, JSON.stringify(asList(translationContributors), null, " "))
}));
writeFileSync(
translatorsTarget,
JSON.stringify(asList(translationContributors), null, " ")
)
})
}
main()
main()

169
scripts/generateDocs.ts
View file

@ -1,131 +1,152 @@
import Combine from "../UI/Base/Combine";
import BaseUIElement from "../UI/BaseUIElement";
import Translations from "../UI/i18n/Translations";
import {existsSync, mkdirSync, writeFileSync} from "fs";
import {AllKnownLayouts} from "../Customizations/AllKnownLayouts";
import TableOfContents from "../UI/Base/TableOfContents";
import SimpleMetaTaggers from "../Logic/SimpleMetaTagger";
import ValidatedTextField from "../UI/Input/ValidatedTextField";
import SpecialVisualizations from "../UI/SpecialVisualizations";
import {ExtraFunctions} from "../Logic/ExtraFunctions";
import Title from "../UI/Base/Title";
import Minimap from "../UI/Base/Minimap";
import QueryParameterDocumentation from "../UI/QueryParameterDocumentation";
import ScriptUtils from "./ScriptUtils";
import List from "../UI/Base/List";
import SharedTagRenderings from "../Customizations/SharedTagRenderings";
function WriteFile(filename, html: BaseUIElement, autogenSource: string[], options?: {
noTableOfContents: boolean
}): void {
import Combine from "../UI/Base/Combine"
import BaseUIElement from "../UI/BaseUIElement"
import Translations from "../UI/i18n/Translations"
import { existsSync, mkdirSync, writeFileSync } from "fs"
import { AllKnownLayouts } from "../Customizations/AllKnownLayouts"
import TableOfContents from "../UI/Base/TableOfContents"
import SimpleMetaTaggers from "../Logic/SimpleMetaTagger"
import ValidatedTextField from "../UI/Input/ValidatedTextField"
import SpecialVisualizations from "../UI/SpecialVisualizations"
import { ExtraFunctions } from "../Logic/ExtraFunctions"
import Title from "../UI/Base/Title"
import Minimap from "../UI/Base/Minimap"
import QueryParameterDocumentation from "../UI/QueryParameterDocumentation"
import ScriptUtils from "./ScriptUtils"
import List from "../UI/Base/List"
import SharedTagRenderings from "../Customizations/SharedTagRenderings"
function WriteFile(
filename,
html: BaseUIElement,
autogenSource: string[],
options?: {
noTableOfContents: boolean
}
): void {
for (const source of autogenSource) {
if(source.indexOf("*") > 0){
if (source.indexOf("*") > 0) {
continue
}
if(!existsSync(source)){
throw "While creating a documentation file and checking that the generation sources are properly linked: source file "+source+" was not found. Typo?"
if (!existsSync(source)) {
throw (
"While creating a documentation file and checking that the generation sources are properly linked: source file " +
source +
" was not found. Typo?"
)
}
}
if (html instanceof Combine && !(options?.noTableOfContents)) {
const toc = new TableOfContents(html);
const els = html.getElements();
html = new Combine(
[els.shift(),
toc,
...els
]
).SetClass("flex flex-col")
if (html instanceof Combine && !options?.noTableOfContents) {
const toc = new TableOfContents(html)
const els = html.getElements()
html = new Combine([els.shift(), toc, ...els]).SetClass("flex flex-col")
}
let md = new Combine([Translations.W(html),
"\n\nThis document is autogenerated from " + autogenSource.map(file => `[${file}](https://github.com/pietervdvn/MapComplete/blob/develop/${file})`).join(", ")
let md = new Combine([
Translations.W(html),
"\n\nThis document is autogenerated from " +
autogenSource
.map(
(file) =>
`[${file}](https://github.com/pietervdvn/MapComplete/blob/develop/${file})`
)
.join(", "),
]).AsMarkdown()
md.replace(/\n\n\n+/g, "\n\n");
md.replace(/\n\n\n+/g, "\n\n")
writeFileSync(filename, md);
writeFileSync(filename, md)
}
console.log("Starting documentation generation...")
AllKnownLayouts.GenOverviewsForSingleLayer((layer, element, inlineSource) => {
console.log("Exporting ", layer.id)
if(!existsSync("./Docs/Layers")){
if (!existsSync("./Docs/Layers")) {
mkdirSync("./Docs/Layers")
}
let source: string = `assets/layers/${layer.id}/${layer.id}.json`
if(inlineSource !== undefined){
if (inlineSource !== undefined) {
source = `assets/themes/${inlineSource}/${inlineSource}.json`
}
WriteFile("./Docs/Layers/" + layer.id + ".md", element, [source], {noTableOfContents: true})
WriteFile("./Docs/Layers/" + layer.id + ".md", element, [source], { noTableOfContents: true })
})
WriteFile("./Docs/SpecialRenderings.md", SpecialVisualizations.HelpMessage(), ["UI/SpecialVisualizations.ts"])
WriteFile("./Docs/CalculatedTags.md", new Combine([new Title("Metatags", 1),
SimpleMetaTaggers.HelpText(), ExtraFunctions.HelpText()]).SetClass("flex-col"),
["Logic/SimpleMetaTagger.ts", "Logic/ExtraFunctions.ts"])
WriteFile("./Docs/SpecialInputElements.md", ValidatedTextField.HelpText(), ["UI/Input/ValidatedTextField.ts"]);
WriteFile("./Docs/BuiltinLayers.md", AllKnownLayouts.GenLayerOverviewText(), ["Customizations/AllKnownLayouts.ts"])
WriteFile("./Docs/BuiltinQuestions.md", SharedTagRenderings.HelpText(), ["Customizations/SharedTagRenderings.ts","assets/tagRenderings/questions.json"])
WriteFile("./Docs/SpecialRenderings.md", SpecialVisualizations.HelpMessage(), [
"UI/SpecialVisualizations.ts",
])
WriteFile(
"./Docs/CalculatedTags.md",
new Combine([
new Title("Metatags", 1),
SimpleMetaTaggers.HelpText(),
ExtraFunctions.HelpText(),
]).SetClass("flex-col"),
["Logic/SimpleMetaTagger.ts", "Logic/ExtraFunctions.ts"]
)
WriteFile("./Docs/SpecialInputElements.md", ValidatedTextField.HelpText(), [
"UI/Input/ValidatedTextField.ts",
])
WriteFile("./Docs/BuiltinLayers.md", AllKnownLayouts.GenLayerOverviewText(), [
"Customizations/AllKnownLayouts.ts",
])
WriteFile("./Docs/BuiltinQuestions.md", SharedTagRenderings.HelpText(), [
"Customizations/SharedTagRenderings.ts",
"assets/tagRenderings/questions.json",
])
{
// Generate the builtinIndex which shows interlayer dependencies
var layers = ScriptUtils.getLayerFiles().map(f => f.parsed)
var builtinsPerLayer= new Map<string, string[]>();
var layersUsingBuiltin = new Map<string /* Builtin */, string[]>();
var layers = ScriptUtils.getLayerFiles().map((f) => f.parsed)
var builtinsPerLayer = new Map<string, string[]>()
var layersUsingBuiltin = new Map<string /* Builtin */, string[]>()
for (const layer of layers) {
if(layer.tagRenderings === undefined){
if (layer.tagRenderings === undefined) {
continue
}
const usedBuiltins : string[] = []
const usedBuiltins: string[] = []
for (const tagRendering of layer.tagRenderings) {
if(typeof tagRendering === "string"){
if (typeof tagRendering === "string") {
usedBuiltins.push(tagRendering)
continue
}
if(tagRendering["builtin"] !== undefined){
if (tagRendering["builtin"] !== undefined) {
const builtins = tagRendering["builtin"]
if(typeof builtins === "string"){
if (typeof builtins === "string") {
usedBuiltins.push(builtins)
}else{
} else {
usedBuiltins.push(...builtins)
}
}
}
for (const usedBuiltin of usedBuiltins) {
var using = layersUsingBuiltin.get(usedBuiltin)
if(using === undefined){
if (using === undefined) {
layersUsingBuiltin.set(usedBuiltin, [layer.id])
}else{
} else {
using.push(layer.id)
}
}
builtinsPerLayer.set(layer.id, usedBuiltins)
}
const docs = new Combine([
new Title("Index of builtin TagRendering" ,1),
new Title("Index of builtin TagRendering", 1),
new Title("Existing builtin tagrenderings", 2),
... Array.from(layersUsingBuiltin.entries()).map(([builtin, usedByLayers]) =>
new Combine([
new Title(builtin),
new List(usedByLayers)
]).SetClass("flex flex-col")
)
...Array.from(layersUsingBuiltin.entries()).map(([builtin, usedByLayers]) =>
new Combine([new Title(builtin), new List(usedByLayers)]).SetClass("flex flex-col")
),
]).SetClass("flex flex-col")
WriteFile("./Docs/BuiltinIndex.md", docs, ["assets/layers/*.json"])
}
Minimap.createMiniMap = _ => {
console.log("Not creating a minimap, it is disabled");
Minimap.createMiniMap = (_) => {
console.log("Not creating a minimap, it is disabled")
return undefined
}
WriteFile("./Docs/URL_Parameters.md", QueryParameterDocumentation.GenerateQueryParameterDocs(), ["Logic/Web/QueryParameters.ts", "UI/QueryParameterDocumentation.ts"])
WriteFile("./Docs/URL_Parameters.md", QueryParameterDocumentation.GenerateQueryParameterDocs(), [
"Logic/Web/QueryParameters.ts",
"UI/QueryParameterDocumentation.ts",
])
console.log("Generated docs")

37
scripts/generateIncludedImages.ts
View file

@ -1,44 +1,43 @@
import * as fs from "fs";
import * as fs from "fs"
function genImages(dryrun = false) {
console.log("Generating images")
const dir = fs.readdirSync("./assets/svg")
let module = "import Img from \"./UI/Base/Img\";\nimport {FixedUiElement} from \"./UI/Base/FixedUiElement\";\n\nexport default class Svg {\n\n\n";
const allNames: string[] = [];
let module =
'import Img from "./UI/Base/Img";\nimport {FixedUiElement} from "./UI/Base/FixedUiElement";\n\nexport default class Svg {\n\n\n'
const allNames: string[] = []
for (const path of dir) {
if (path.endsWith("license_info.json")) {
continue;
continue
}
if (!path.endsWith(".svg")) {
throw "Non-svg file detected in the svg files: " + path;
throw "Non-svg file detected in the svg files: " + path
}
let svg : string = fs.readFileSync("./assets/svg/" + path, "utf-8")
let svg: string = fs
.readFileSync("./assets/svg/" + path, "utf-8")
.replace(/<\?xml.*?>/, "")
.replace(/fill: ?none;/g, "fill: none !important;") // This is such a brittle hack...
.replace(/\n/g, " ")
.replace(/\r/g, "")
.replace(/\\/g, "\\")
.replace(/"/g, "\\\"")
.replace(/"/g, '\\"')
let hasNonAsciiChars = Array.from(svg).some(char => char.charCodeAt(0) > 127);
if(hasNonAsciiChars){
throw "The svg '"+path+"' has non-ascii characters";
let hasNonAsciiChars = Array.from(svg).some((char) => char.charCodeAt(0) > 127)
if (hasNonAsciiChars) {
throw "The svg '" + path + "' has non-ascii characters"
}
const name = path.substr(0, path.length - 4)
.replace(/[ -]/g, "_");
const name = path.substr(0, path.length - 4).replace(/[ -]/g, "_")
if (dryrun) {
svg = "xxx"
}
let rawName = name;
let rawName = name
if (dryrun) {
rawName = "add";
rawName = "add"
}
module += ` public static ${name} = "${svg}"\n`
@ -50,9 +49,9 @@ function genImages(dryrun = false) {
}
}
module += `public static All = {${allNames.join(",")}};`
module += "}\n";
fs.writeFileSync("Svg.ts", module);
module += "}\n"
fs.writeFileSync("Svg.ts", module)
console.log("Done")
}
genImages()
genImages()

352
scripts/generateLayerOverview.ts
View file

@ -1,43 +1,44 @@
import ScriptUtils from "./ScriptUtils";
import {existsSync, mkdirSync, readFileSync, statSync, writeFileSync} from "fs";
import ScriptUtils from "./ScriptUtils"
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "fs"
import * as licenses from "../assets/generated/license_info.json"
import {LayoutConfigJson} from "../Models/ThemeConfig/Json/LayoutConfigJson";
import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
import Constants from "../Models/Constants";
import { LayoutConfigJson } from "../Models/ThemeConfig/Json/LayoutConfigJson"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import Constants from "../Models/Constants"
import {
DoesImageExist,
PrevalidateTheme,
ValidateLayer,
ValidateTagRenderings,
ValidateThemeAndLayers
} from "../Models/ThemeConfig/Conversion/Validation";
import {Translation} from "../UI/i18n/Translation";
import {TagRenderingConfigJson} from "../Models/ThemeConfig/Json/TagRenderingConfigJson";
import * as questions from "../assets/tagRenderings/questions.json";
import * as icons from "../assets/tagRenderings/icons.json";
import PointRenderingConfigJson from "../Models/ThemeConfig/Json/PointRenderingConfigJson";
import {PrepareLayer} from "../Models/ThemeConfig/Conversion/PrepareLayer";
import {PrepareTheme} from "../Models/ThemeConfig/Conversion/PrepareTheme";
import {DesugaringContext} from "../Models/ThemeConfig/Conversion/Conversion";
import {Utils} from "../Utils";
import {AllKnownLayouts} from "../Customizations/AllKnownLayouts";
ValidateThemeAndLayers,
} from "../Models/ThemeConfig/Conversion/Validation"
import { Translation } from "../UI/i18n/Translation"
import { TagRenderingConfigJson } from "../Models/ThemeConfig/Json/TagRenderingConfigJson"
import * as questions from "../assets/tagRenderings/questions.json"
import * as icons from "../assets/tagRenderings/icons.json"
import PointRenderingConfigJson from "../Models/ThemeConfig/Json/PointRenderingConfigJson"
import { PrepareLayer } from "../Models/ThemeConfig/Conversion/PrepareLayer"
import { PrepareTheme } from "../Models/ThemeConfig/Conversion/PrepareTheme"
import { DesugaringContext } from "../Models/ThemeConfig/Conversion/Conversion"
import { Utils } from "../Utils"
import { AllKnownLayouts } from "../Customizations/AllKnownLayouts"
// This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files.
// It spits out an overview of those to be used to load them
class LayerOverviewUtils {
public static readonly layerPath = "./assets/generated/layers/"
public static readonly themePath = "./assets/generated/themes/"
private static publicLayerIdsFrom(themefiles: LayoutConfigJson[]): Set<string> {
const publicThemes = [].concat(...themefiles
.filter(th => !th.hideFromOverview))
const publicThemes = [].concat(...themefiles.filter((th) => !th.hideFromOverview))
return new Set([].concat(...publicThemes.map(th => this.extractLayerIdsFrom(th))))
return new Set([].concat(...publicThemes.map((th) => this.extractLayerIdsFrom(th))))
}
private static extractLayerIdsFrom(themeFile: LayoutConfigJson, includeInlineLayers = true): string[] {
private static extractLayerIdsFrom(
themeFile: LayoutConfigJson,
includeInlineLayers = true
): string[] {
const publicLayerIds = []
for (const publicLayer of themeFile.layers) {
if (typeof publicLayer === "string") {
@ -50,7 +51,7 @@ class LayerOverviewUtils {
publicLayerIds.push(bi)
continue
}
bi.forEach(id => publicLayerIds.push(id))
bi.forEach((id) => publicLayerIds.push(id))
continue
}
if (includeInlineLayers) {
@ -62,24 +63,33 @@ class LayerOverviewUtils {
shouldBeUpdated(sourcefile: string | string[], targetfile: string): boolean {
if (!existsSync(targetfile)) {
return true;
return true
}
const targetModified = statSync(targetfile).mtime
if (typeof sourcefile === "string") {
sourcefile = [sourcefile]
}
return sourcefile.some(sourcefile => statSync(sourcefile).mtime > targetModified)
return sourcefile.some((sourcefile) => statSync(sourcefile).mtime > targetModified)
}
writeSmallOverview(themes: { id: string, title: any, shortDescription: any, icon: string, hideFromOverview: boolean, mustHaveLanguage: boolean, layers: (LayerConfigJson | string | { builtin })[] }[]) {
const perId = new Map<string, any>();
writeSmallOverview(
themes: {
id: string
title: any
shortDescription: any
icon: string
hideFromOverview: boolean
mustHaveLanguage: boolean
layers: (LayerConfigJson | string | { builtin })[]
}[]
) {
const perId = new Map<string, any>()
for (const theme of themes) {
const keywords: {}[] = []
for (const layer of (theme.layers ?? [])) {
const l = <LayerConfigJson>layer;
keywords.push({"*": l.id})
for (const layer of theme.layers ?? []) {
const l = <LayerConfigJson>layer
keywords.push({ "*": l.id })
keywords.push(l.title)
keywords.push(l.description)
}
@ -91,56 +101,69 @@ class LayerOverviewUtils {
icon: theme.icon,
hideFromOverview: theme.hideFromOverview,
mustHaveLanguage: theme.mustHaveLanguage,
keywords: Utils.NoNull(keywords)
keywords: Utils.NoNull(keywords),
}
perId.set(theme.id, data);
perId.set(theme.id, data)
}
const sorted = Constants.themeOrder.map(id => {
const sorted = Constants.themeOrder.map((id) => {
if (!perId.has(id)) {
throw "Ordered theme id " + id + " not found"
}
return perId.get(id);
});
return perId.get(id)
})
perId.forEach((value) => {
if (Constants.themeOrder.indexOf(value.id) >= 0) {
return; // actually a continue
return // actually a continue
}
sorted.push(value)
})
writeFileSync("./assets/generated/theme_overview.json", JSON.stringify(sorted, null, " "), "UTF8");
writeFileSync(
"./assets/generated/theme_overview.json",
JSON.stringify(sorted, null, " "),
"UTF8"
)
}
writeTheme(theme: LayoutConfigJson) {
if (!existsSync(LayerOverviewUtils.themePath)) {
mkdirSync(LayerOverviewUtils.themePath);
mkdirSync(LayerOverviewUtils.themePath)
}
writeFileSync(`${LayerOverviewUtils.themePath}${theme.id}.json`, JSON.stringify(theme, null, " "), "UTF8");
writeFileSync(
`${LayerOverviewUtils.themePath}${theme.id}.json`,
JSON.stringify(theme, null, " "),
"UTF8"
)
}
writeLayer(layer: LayerConfigJson) {
if (!existsSync(LayerOverviewUtils.layerPath)) {
mkdirSync(LayerOverviewUtils.layerPath);
mkdirSync(LayerOverviewUtils.layerPath)
}
writeFileSync(`${LayerOverviewUtils.layerPath}${layer.id}.json`, JSON.stringify(layer, null, " "), "UTF8");
writeFileSync(
`${LayerOverviewUtils.layerPath}${layer.id}.json`,
JSON.stringify(layer, null, " "),
"UTF8"
)
}
getSharedTagRenderings(doesImageExist: DoesImageExist): Map<string, TagRenderingConfigJson> {
const dict = new Map<string, TagRenderingConfigJson>();
const dict = new Map<string, TagRenderingConfigJson>()
const validator = new ValidateTagRenderings(undefined, doesImageExist);
const validator = new ValidateTagRenderings(undefined, doesImageExist)
for (const key in questions["default"]) {
if (key === "id") {
continue
}
questions[key].id = key;
questions[key].id = key
questions[key]["source"] = "shared-questions"
const config = <TagRenderingConfigJson>questions[key]
validator.convertStrict(config, "generate-layer-overview:tagRenderings/questions.json:" + key)
validator.convertStrict(
config,
"generate-layer-overview:tagRenderings/questions.json:" + key
)
dict.set(key, config)
}
for (const key in icons["default"]) {
@ -150,9 +173,12 @@ class LayerOverviewUtils {
if (typeof icons[key] !== "object") {
continue
}
icons[key].id = key;
icons[key].id = key
const config = <TagRenderingConfigJson>icons[key]
validator.convertStrict(config, "generate-layer-overview:tagRenderings/icons.json:" + key)
validator.convertStrict(
config,
"generate-layer-overview:tagRenderings/icons.json:" + key
)
dict.set(key, config)
}
@ -160,35 +186,43 @@ class LayerOverviewUtils {
if (key === "id") {
return
}
value.id = value.id ?? key;
value.id = value.id ?? key
})
return dict;
return dict
}
checkAllSvgs() {
const allSvgs = ScriptUtils.readDirRecSync("./assets")
.filter(path => path.endsWith(".svg"))
.filter(path => !path.startsWith("./assets/generated"))
let errCount = 0;
const exempt = ["assets/SocialImageTemplate.svg", "assets/SocialImageTemplateWide.svg", "assets/SocialImageBanner.svg", "assets/svg/osm-logo.svg"];
.filter((path) => path.endsWith(".svg"))
.filter((path) => !path.startsWith("./assets/generated"))
let errCount = 0
const exempt = [
"assets/SocialImageTemplate.svg",
"assets/SocialImageTemplateWide.svg",
"assets/SocialImageBanner.svg",
"assets/svg/osm-logo.svg",
]
for (const path of allSvgs) {
if (exempt.some(p => "./" + p === path)) {
if (exempt.some((p) => "./" + p === path)) {
continue
}
const contents = readFileSync(path, "UTF8")
if (contents.indexOf("data:image/png;") >= 0) {
console.warn("The SVG at " + path + " is a fake SVG: it contains PNG data!")
errCount++;
errCount++
if (path.startsWith("./assets/svg")) {
throw "A core SVG is actually a PNG. Don't do this!"
}
}
if (contents.indexOf("<text") > 0) {
console.warn("The SVG at " + path + " contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path")
errCount++;
console.warn(
"The SVG at " +
path +
" contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path"
)
errCount++
}
}
if (errCount > 0) {
@ -197,91 +231,115 @@ class LayerOverviewUtils {
}
main(args: string[]) {
const forceReload = args.some(a => a == "--force")
const forceReload = args.some((a) => a == "--force")
const licensePaths = new Set<string>()
for (const i in licenses) {
licensePaths.add(licenses[i].path)
}
const doesImageExist = new DoesImageExist(licensePaths, existsSync)
const sharedLayers = this.buildLayerIndex(doesImageExist, forceReload);
const sharedLayers = this.buildLayerIndex(doesImageExist, forceReload)
const recompiledThemes: string[] = []
const sharedThemes = this.buildThemeIndex(doesImageExist, sharedLayers, recompiledThemes, forceReload)
const sharedThemes = this.buildThemeIndex(
doesImageExist,
sharedLayers,
recompiledThemes,
forceReload
)
writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
"layers": Array.from(sharedLayers.values()),
"themes": Array.from(sharedThemes.values())
}))
writeFileSync(
"./assets/generated/known_layers_and_themes.json",
JSON.stringify({
layers: Array.from(sharedLayers.values()),
themes: Array.from(sharedThemes.values()),
})
)
writeFileSync("./assets/generated/known_layers.json", JSON.stringify({layers: Array.from(sharedLayers.values())}))
writeFileSync(
"./assets/generated/known_layers.json",
JSON.stringify({ layers: Array.from(sharedLayers.values()) })
)
if (recompiledThemes.length > 0 && !(recompiledThemes.length === 1 && recompiledThemes[0] === "mapcomplete-changes")) {
if (
recompiledThemes.length > 0 &&
!(recompiledThemes.length === 1 && recompiledThemes[0] === "mapcomplete-changes")
) {
// mapcomplete-changes shows an icon for each corresponding mapcomplete-theme
const iconsPerTheme =
Array.from(sharedThemes.values()).map(th => ({
if: "theme=" + th.id,
then: th.icon
}))
const proto: LayoutConfigJson = JSON.parse(readFileSync("./assets/themes/mapcomplete-changes/mapcomplete-changes.proto.json", "UTF8"));
const protolayer = <LayerConfigJson>(proto.layers.filter(l => l["id"] === "mapcomplete-changes")[0])
const rendering = (<PointRenderingConfigJson>protolayer.mapRendering[0])
const iconsPerTheme = Array.from(sharedThemes.values()).map((th) => ({
if: "theme=" + th.id,
then: th.icon,
}))
const proto: LayoutConfigJson = JSON.parse(
readFileSync(
"./assets/themes/mapcomplete-changes/mapcomplete-changes.proto.json",
"UTF8"
)
)
const protolayer = <LayerConfigJson>(
proto.layers.filter((l) => l["id"] === "mapcomplete-changes")[0]
)
const rendering = <PointRenderingConfigJson>protolayer.mapRendering[0]
rendering.icon["mappings"] = iconsPerTheme
writeFileSync('./assets/themes/mapcomplete-changes/mapcomplete-changes.json', JSON.stringify(proto, null, " "))
writeFileSync(
"./assets/themes/mapcomplete-changes/mapcomplete-changes.json",
JSON.stringify(proto, null, " ")
)
}
this.checkAllSvgs()
if(AllKnownLayouts.getSharedLayersConfigs().size == 0){
console.error( "This was a bootstrapping-run. Run generate layeroverview again!")
}else{
const green = s => '\x1b[92m' + s + '\x1b[0m'
if (AllKnownLayouts.getSharedLayersConfigs().size == 0) {
console.error("This was a bootstrapping-run. Run generate layeroverview again!")
} else {
const green = (s) => "\x1b[92m" + s + "\x1b[0m"
console.log(green("All done!"))
}
}
private buildLayerIndex(doesImageExist: DoesImageExist, forceReload: boolean): Map<string, LayerConfigJson> {
private buildLayerIndex(
doesImageExist: DoesImageExist,
forceReload: boolean
): Map<string, LayerConfigJson> {
// First, we expand and validate all builtin layers. These are written to assets/generated/layers
// At the same time, an index of available layers is built.
console.log(" ---------- VALIDATING BUILTIN LAYERS ---------")
const sharedTagRenderings = this.getSharedTagRenderings(doesImageExist);
const sharedTagRenderings = this.getSharedTagRenderings(doesImageExist)
const state: DesugaringContext = {
tagRenderings: sharedTagRenderings,
sharedLayers: AllKnownLayouts.getSharedLayersConfigs()
sharedLayers: AllKnownLayouts.getSharedLayersConfigs(),
}
const sharedLayers = new Map<string, LayerConfigJson>()
const prepLayer = new PrepareLayer(state);
const prepLayer = new PrepareLayer(state)
const skippedLayers: string[] = []
const recompiledLayers: string[] = []
for (const sharedLayerPath of ScriptUtils.getLayerPaths()) {
{
const targetPath = LayerOverviewUtils.layerPath + sharedLayerPath.substring(sharedLayerPath.lastIndexOf("/"))
const targetPath =
LayerOverviewUtils.layerPath +
sharedLayerPath.substring(sharedLayerPath.lastIndexOf("/"))
if (!forceReload && !this.shouldBeUpdated(sharedLayerPath, targetPath)) {
const sharedLayer = JSON.parse(readFileSync(targetPath, "utf8"))
sharedLayers.set(sharedLayer.id, sharedLayer)
skippedLayers.push(sharedLayer.id)
console.log("Loaded "+sharedLayer.id)
continue;
console.log("Loaded " + sharedLayer.id)
continue
}
}
let parsed;
let parsed
try {
parsed = JSON.parse(readFileSync(sharedLayerPath, "utf8"))
} catch (e) {
throw ("Could not parse or read file " + sharedLayerPath)
throw "Could not parse or read file " + sharedLayerPath
}
const context = "While building builtin layer " + sharedLayerPath
const fixed = prepLayer.convertStrict(parsed, context)
if (fixed.source.osmTags["and"] === undefined) {
fixed.source.osmTags = {"and": [fixed.source.osmTags]}
fixed.source.osmTags = { and: [fixed.source.osmTags] }
}
const validator = new ValidateLayer(sharedLayerPath, true, doesImageExist);
const validator = new ValidateLayer(sharedLayerPath, true, doesImageExist)
validator.convertStrict(fixed, context)
if (sharedLayers.has(fixed.id)) {
@ -292,74 +350,108 @@ class LayerOverviewUtils {
recompiledLayers.push(fixed.id)
this.writeLayer(fixed)
}
console.log("Recompiled layers " + recompiledLayers.join(", ") + " and skipped " + skippedLayers.length + " layers")
console.log(
"Recompiled layers " +
recompiledLayers.join(", ") +
" and skipped " +
skippedLayers.length +
" layers"
)
return sharedLayers;
return sharedLayers
}
private buildThemeIndex(doesImageExist: DoesImageExist, sharedLayers: Map<string, LayerConfigJson>, recompiledThemes: string[], forceReload: boolean): Map<string, LayoutConfigJson> {
private buildThemeIndex(
doesImageExist: DoesImageExist,
sharedLayers: Map<string, LayerConfigJson>,
recompiledThemes: string[],
forceReload: boolean
): Map<string, LayoutConfigJson> {
console.log(" ---------- VALIDATING BUILTIN THEMES ---------")
const themeFiles = ScriptUtils.getThemeFiles();
const fixed = new Map<string, LayoutConfigJson>();
const themeFiles = ScriptUtils.getThemeFiles()
const fixed = new Map<string, LayoutConfigJson>()
const publicLayers = LayerOverviewUtils.publicLayerIdsFrom(themeFiles.map(th => th.parsed))
const publicLayers = LayerOverviewUtils.publicLayerIdsFrom(
themeFiles.map((th) => th.parsed)
)
const convertState: DesugaringContext = {
sharedLayers,
tagRenderings: this.getSharedTagRenderings(doesImageExist),
publicLayers
publicLayers,
}
const skippedThemes: string[] = []
for (const themeInfo of themeFiles) {
const themePath = themeInfo.path;
const themePath = themeInfo.path
let themeFile = themeInfo.parsed
{
const targetPath = LayerOverviewUtils.themePath + "/" + themePath.substring(themePath.lastIndexOf("/"))
const usedLayers = Array.from(LayerOverviewUtils.extractLayerIdsFrom(themeFile, false))
.map(id => LayerOverviewUtils.layerPath + id + ".json")
const targetPath =
LayerOverviewUtils.themePath +
"/" +
themePath.substring(themePath.lastIndexOf("/"))
const usedLayers = Array.from(
LayerOverviewUtils.extractLayerIdsFrom(themeFile, false)
).map((id) => LayerOverviewUtils.layerPath + id + ".json")
if (!forceReload && !this.shouldBeUpdated([themePath, ...usedLayers], targetPath)) {
fixed.set(themeFile.id, JSON.parse(readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", 'utf8')))
fixed.set(
themeFile.id,
JSON.parse(
readFileSync(
LayerOverviewUtils.themePath + themeFile.id + ".json",
"utf8"
)
)
)
skippedThemes.push(themeFile.id)
continue;
continue
}
recompiledThemes.push(themeFile.id)
}
new PrevalidateTheme().convertStrict(themeFile, themePath)
try {
themeFile = new PrepareTheme(convertState).convertStrict(themeFile, themePath)
new ValidateThemeAndLayers(doesImageExist, themePath, true, convertState.tagRenderings)
.convertStrict(themeFile, themePath)
new ValidateThemeAndLayers(
doesImageExist,
themePath,
true,
convertState.tagRenderings
).convertStrict(themeFile, themePath)
this.writeTheme(themeFile)
fixed.set(themeFile.id, themeFile)
} catch (e) {
console.error("ERROR: could not prepare theme " + themePath + " due to " + e)
throw e;
throw e
}
}
this.writeSmallOverview(Array.from(fixed.values()).map(t => {
return {
...t,
hideFromOverview: t.hideFromOverview ?? false,
shortDescription: t.shortDescription ?? new Translation(t.description).FirstSentence().translations,
mustHaveLanguage: t.mustHaveLanguage?.length > 0,
}
}));
this.writeSmallOverview(
Array.from(fixed.values()).map((t) => {
return {
...t,
hideFromOverview: t.hideFromOverview ?? false,
shortDescription:
t.shortDescription ??
new Translation(t.description).FirstSentence().translations,
mustHaveLanguage: t.mustHaveLanguage?.length > 0,
}
})
)
console.log("Recompiled themes " + recompiledThemes.join(", ") + " and skipped " + skippedThemes.length + " themes")
return fixed;
console.log(
"Recompiled themes " +
recompiledThemes.join(", ") +
" and skipped " +
skippedThemes.length +
" themes"
)
return fixed
}
}

249
scripts/generateLayouts.ts
View file

@ -1,35 +1,34 @@
import {appendFileSync, existsSync, mkdirSync, readFileSync, writeFile, writeFileSync} from "fs";
import Locale from "../UI/i18n/Locale";
import Translations from "../UI/i18n/Translations";
import {Translation} from "../UI/i18n/Translation";
import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFile, writeFileSync } from "fs"
import Locale from "../UI/i18n/Locale"
import Translations from "../UI/i18n/Translations"
import { Translation } from "../UI/i18n/Translation"
import * as all_known_layouts from "../assets/generated/known_layers_and_themes.json"
import {LayoutConfigJson} from "../Models/ThemeConfig/Json/LayoutConfigJson";
import LayoutConfig from "../Models/ThemeConfig/LayoutConfig";
import xml2js from 'xml2js';
import ScriptUtils from "./ScriptUtils";
import {Utils} from "../Utils";
const sharp = require('sharp');
const template = readFileSync("theme.html", "utf8");
const codeTemplate = readFileSync("index_theme.ts.template", "utf8");
import { LayoutConfigJson } from "../Models/ThemeConfig/Json/LayoutConfigJson"
import LayoutConfig from "../Models/ThemeConfig/LayoutConfig"
import xml2js from "xml2js"
import ScriptUtils from "./ScriptUtils"
import { Utils } from "../Utils"
const sharp = require("sharp")
const template = readFileSync("theme.html", "utf8")
const codeTemplate = readFileSync("index_theme.ts.template", "utf8")
function enc(str: string): string {
return encodeURIComponent(str.toLowerCase());
return encodeURIComponent(str.toLowerCase())
}
async function createIcon(iconPath: string, size: number, alreadyWritten: string[]) {
let name = iconPath.split(".").slice(0, -1).join("."); // drop svg suffix
let name = iconPath.split(".").slice(0, -1).join(".") // drop svg suffix
if (name.startsWith("./")) {
name = name.substr(2)
}
const newname = `assets/generated/images/${name.replace(/\//g,"_")}${size}.png`;
const newname = `assets/generated/images/${name.replace(/\//g, "_")}${size}.png`
if (alreadyWritten.indexOf(newname) >= 0) {
return newname;
return newname
}
alreadyWritten.push(newname);
alreadyWritten.push(newname)
if (existsSync(newname)) {
return newname
}
@ -48,20 +47,25 @@ async function createIcon(iconPath: string, size: number, alreadyWritten: string
console.error("Could not read icon", iconPath, " to create a PNG due to", e)
}
return newname;
return newname
}
async function createSocialImage(layout: LayoutConfig, template: "" | "Wide"): Promise<string> {
if (!layout.icon.endsWith(".svg")) {
console.warn("Not creating a social image for " + layout.id + " as it is _not_ a .svg: " + layout.icon)
console.warn(
"Not creating a social image for " +
layout.id +
" as it is _not_ a .svg: " +
layout.icon
)
return undefined
}
const path = `./assets/generated/images/social_image_${layout.id}_${template}.svg`
if(existsSync(path)){
return path;
if (existsSync(path)) {
return path
}
const svg = await ScriptUtils.ReadSvg(layout.icon)
let width: string = svg.$.width;
let width: string = svg.$.width
if (width === undefined) {
throw "The logo at " + layout.icon + " does not have a defined width"
}
@ -74,15 +78,16 @@ async function createSocialImage(layout: LayoutConfig, template: "" | "Wide"): P
delete svg["defs"]
delete svg["$"]
let templateSvg = await ScriptUtils.ReadSvg("./assets/SocialImageTemplate" + template + ".svg")
templateSvg = Utils.WalkJson(templateSvg,
templateSvg = Utils.WalkJson(
templateSvg,
(leaf) => {
const {cx, cy, r} = leaf["circle"][0].$
const { cx, cy, r } = leaf["circle"][0].$
return {
$: {
id: "icon",
transform: `translate(${cx - r},${cy - r}) scale(${(r * 2) / Number(width)}) `
transform: `translate(${cx - r},${cy - r}) scale(${(r * 2) / Number(width)}) `,
},
g: [svg]
g: [svg],
}
},
(mightBeTokenToReplace) => {
@ -93,73 +98,76 @@ async function createSocialImage(layout: LayoutConfig, template: "" | "Wide"): P
}
)
const builder = new xml2js.Builder();
const xml = builder.buildObject({svg: templateSvg});
const builder = new xml2js.Builder()
const xml = builder.buildObject({ svg: templateSvg })
writeFileSync(path, xml)
console.log("Created social image at ", path)
return path
}
async function createManifest(layout: LayoutConfig, alreadyWritten: string[]): Promise<{
manifest: any,
async function createManifest(
layout: LayoutConfig,
alreadyWritten: string[]
): Promise<{
manifest: any
whiteIcons: string[]
}> {
Translation.forcedLanguage = "en"
const icons = [];
const icons = []
const whiteIcons: string[] = []
let icon = layout.icon;
let icon = layout.icon
if (icon.endsWith(".svg") || icon.startsWith("<svg") || icon.startsWith("<?xml")) {
// This is an svg. Lets create the needed pngs and do some checkes!
const whiteBackgroundPath = "./assets/generated/images/theme_" + layout.id + "_white_background.svg"
const whiteBackgroundPath =
"./assets/generated/images/theme_" + layout.id + "_white_background.svg"
{
const svg = await ScriptUtils.ReadSvg(icon)
const width: string = svg.$.width;
const height: string = svg.$.height;
const width: string = svg.$.width
const height: string = svg.$.height
const builder = new xml2js.Builder();
const withRect = {rect: {"$": {width, height, style: "fill:#ffffff;"}}, ...svg}
const xml = builder.buildObject({svg: withRect});
const builder = new xml2js.Builder()
const withRect = { rect: { $: { width, height, style: "fill:#ffffff;" } }, ...svg }
const xml = builder.buildObject({ svg: withRect })
writeFileSync(whiteBackgroundPath, xml)
}
let path = layout.icon;
let path = layout.icon
if (layout.icon.startsWith("<")) {
// THis is already the svg
path = "./assets/generated/images/" + layout.id + "_logo.svg"
writeFileSync(path, layout.icon)
}
const sizes = [72, 96, 120, 128, 144, 152, 180, 192, 384, 512];
const sizes = [72, 96, 120, 128, 144, 152, 180, 192, 384, 512]
for (const size of sizes) {
const name = await createIcon(path, size, alreadyWritten);
const name = await createIcon(path, size, alreadyWritten)
const whiteIcon = await createIcon(whiteBackgroundPath, size, alreadyWritten)
whiteIcons.push(whiteIcon)
icons.push({
src: name,
sizes: size + "x" + size,
type: "image/png"
type: "image/png",
})
}
icons.push({
src: path,
sizes: "513x513",
type: "image/svg"
type: "image/svg",
})
} else if (icon.endsWith(".png")) {
icons.push({
src: icon,
sizes: "513x513",
type: "image/png"
type: "image/png",
})
} else {
console.log(icon)
throw "Icon is not an svg for " + layout.id
}
const ogTitle = Translations.T(layout.title).txt;
const ogDescr = Translations.T(layout.description ?? "").txt;
const ogTitle = Translations.T(layout.title).txt
const ogDescr = Translations.T(layout.description ?? "").txt
const manifest = {
name: ogTitle,
@ -171,48 +179,50 @@ async function createManifest(layout: LayoutConfig, alreadyWritten: string[]): P
description: ogDescr,
orientation: "portrait-primary, landscape-primary",
icons: icons,
categories: ["map", "navigation"]
};
categories: ["map", "navigation"],
}
return {
manifest,
whiteIcons
whiteIcons,
}
}
async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alreadyWritten) {
Locale.language.setData(layout.language[0]);
Locale.language.setData(layout.language[0])
const targetLanguage = layout.language[0]
const ogTitle = Translations.T(layout.title).textFor(targetLanguage).replace(/"/g, '\\"');
const ogDescr = Translations.T(layout.shortDescription ?? "Easily add and edit geodata with OpenStreetMap").textFor(targetLanguage).replace(/"/g, '\\"');
let ogImage = layout.socialImage;
const ogTitle = Translations.T(layout.title).textFor(targetLanguage).replace(/"/g, '\\"')
const ogDescr = Translations.T(
layout.shortDescription ?? "Easily add and edit geodata with OpenStreetMap"
)
.textFor(targetLanguage)
.replace(/"/g, '\\"')
let ogImage = layout.socialImage
let twitterImage = ogImage
if (ogImage === LayoutConfig.defaultSocialImage && layout.official) {
ogImage = await createSocialImage(layout, "") ?? layout.socialImage
twitterImage = await createSocialImage(layout, "Wide") ?? layout.socialImage
ogImage = (await createSocialImage(layout, "")) ?? layout.socialImage
twitterImage = (await createSocialImage(layout, "Wide")) ?? layout.socialImage
}
if (twitterImage.endsWith(".svg")) {
// svgs are badly supported as social image, we use a generated svg instead
twitterImage = await createIcon(twitterImage, 512, alreadyWritten);
twitterImage = await createIcon(twitterImage, 512, alreadyWritten)
}
if(ogImage.endsWith(".svg")){
if (ogImage.endsWith(".svg")) {
ogImage = await createIcon(ogImage, 512, alreadyWritten)
}
let customCss = "";
let customCss = ""
if (layout.customCss !== undefined && layout.customCss !== "") {
try {
const cssContent = readFileSync(layout.customCss);
customCss = "<style>" + cssContent + "</style>";
const cssContent = readFileSync(layout.customCss)
customCss = "<style>" + cssContent + "</style>"
} catch (e) {
customCss = `<link rel='stylesheet' href="${layout.customCss}"/>`
}
}
const og = `
<meta property="og:image" content="${ogImage ?? 'assets/SocialImage.png'}">
<meta property="og:image" content="${ogImage ?? "assets/SocialImage.png"}">
<meta property="og:title" content="${ogTitle}">
<meta property="og:description" content="${ogDescr}">
<meta name="twitter:card" content="summary_large_image">
@ -222,11 +232,11 @@ async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alr
<meta name="twitter:description" content="${ogDescr}">
<meta name="twitter:image" content="${twitterImage}">`
let icon = layout.icon;
let icon = layout.icon
if (icon.startsWith("<?xml") || icon.startsWith("<svg")) {
// This already is an svg
icon = `./assets/generated/images/${layout.id}_icon.svg`
writeFileSync(icon, layout.icon);
writeFileSync(icon, layout.icon)
}
const apple_icons = []
@ -244,31 +254,49 @@ async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alr
og,
customCss,
`<link rel="icon" href="${icon}" sizes="any" type="image/svg+xml">`,
...apple_icons
...apple_icons,
].join("\n")
const loadingText = Translations.t.general.loadingTheme.Subs({theme: ogTitle});
const loadingText = Translations.t.general.loadingTheme.Subs({ theme: ogTitle })
let output = template
.replace("Loading MapComplete, hang on...", loadingText.textFor(targetLanguage))
.replace("Powered by OpenStreetMap", Translations.t.general.poweredByOsm.textFor(targetLanguage))
.replace(
"Powered by OpenStreetMap",
Translations.t.general.poweredByOsm.textFor(targetLanguage)
)
.replace(/<!-- THEME-SPECIFIC -->.*<!-- THEME-SPECIFIC-END-->/s, themeSpecific)
.replace(/<!-- DESCRIPTION START -->.*<!-- DESCRIPTION END -->/s, layout.shortDescription.textFor(targetLanguage))
.replace("<script src=\"./index.ts\"></script>", `<script src='./index_${layout.id}.ts'></script>`);
0
.replace(
/<!-- DESCRIPTION START -->.*<!-- DESCRIPTION END -->/s,
layout.shortDescription.textFor(targetLanguage)
)
.replace(
'<script src="./index.ts"></script>',
`<script src='./index_${layout.id}.ts'></script>`
)
0
try {
output = output
.replace(/<!-- DECORATION 0 START -->.*<!-- DECORATION 0 END -->/s, `<img src='${icon}' width="100%" height="100%">`)
.replace(/<!-- DECORATION 1 START -->.*<!-- DECORATION 1 END -->/s, `<img src='${icon}' width="100%" height="100%">`);
.replace(
/<!-- DECORATION 0 START -->.*<!-- DECORATION 0 END -->/s,
`<img src='${icon}' width="100%" height="100%">`
)
.replace(
/<!-- DECORATION 1 START -->.*<!-- DECORATION 1 END -->/s,
`<img src='${icon}' width="100%" height="100%">`
)
} catch (e) {
console.warn("Error while applying logo: ", e)
}
return output;
return output
}
async function createIndexFor(theme: LayoutConfig) {
const filename = "index_" + theme.id + ".ts"
writeFileSync(filename, `import * as themeConfig from "./assets/generated/themes/${theme.id}.json"\n`)
writeFileSync(
filename,
`import * as themeConfig from "./assets/generated/themes/${theme.id}.json"\n`
)
appendFileSync(filename, codeTemplate)
}
@ -279,17 +307,28 @@ function createDir(path) {
}
async function main(): Promise<void> {
const alreadyWritten = []
createDir("./assets/generated")
createDir("./assets/generated/layers")
createDir("./assets/generated/themes")
createDir("./assets/generated/images")
const blacklist = ["", "test", ".", "..", "manifest", "index", "land", "preferences", "account", "openstreetmap", "custom", "theme"]
const blacklist = [
"",
"test",
".",
"..",
"manifest",
"index",
"land",
"preferences",
"account",
"openstreetmap",
"custom",
"theme",
]
// @ts-ignore
const all: LayoutConfigJson[] = all_known_layouts.themes;
const all: LayoutConfigJson[] = all_known_layouts.themes
const args = process.argv
const theme = args[2]
if (theme !== undefined) {
@ -303,18 +342,18 @@ async function main(): Promise<void> {
const layout = new LayoutConfig(layoutConfigJson, true)
const layoutName = layout.id
if (blacklist.indexOf(layoutName.toLowerCase()) >= 0) {
console.log(`Skipping a layout with name${layoutName}, it is on the blacklist`);
continue;
console.log(`Skipping a layout with name${layoutName}, it is on the blacklist`)
continue
}
const err = err => {
const err = (err) => {
if (err !== null) {
console.log("Could not write manifest for ", layoutName, " because ", err)
}
};
const {manifest, whiteIcons} = await createManifest(layout, alreadyWritten)
const manif = JSON.stringify(manifest, undefined, 2);
const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest";
writeFile(manifestLocation, manif, err);
}
const { manifest, whiteIcons } = await createManifest(layout, alreadyWritten)
const manif = JSON.stringify(manifest, undefined, 2)
const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest"
writeFile(manifestLocation, manif, err)
// Create a landing page for the given theme
const landing = await createLandingPage(layout, manifest, whiteIcons, alreadyWritten)
@ -322,23 +361,25 @@ async function main(): Promise<void> {
await createIndexFor(layout)
}
const { manifest } = await createManifest(
new LayoutConfig({
icon: "./assets/svg/mapcomplete_logo.svg",
id: "index",
layers: [],
socialImage: "assets/SocialImage.png",
startLat: 0,
startLon: 0,
startZoom: 0,
title: { en: "MapComplete" },
description: { en: "A thematic map viewer and editor based on OpenStreetMap" },
}),
alreadyWritten
)
const {manifest} = await createManifest(new LayoutConfig({
icon: "./assets/svg/mapcomplete_logo.svg",
id: "index",
layers: [],
socialImage: "assets/SocialImage.png",
startLat: 0,
startLon: 0,
startZoom: 0,
title: {en: "MapComplete"},
description: {en: "A thematic map viewer and editor based on OpenStreetMap"}
}), alreadyWritten);
const manif = JSON.stringify(manifest, undefined, 2);
const manif = JSON.stringify(manifest, undefined, 2)
writeFileSync("index.manifest", manif)
}
main().then(() => {
console.log("All done!")
})
})

150
scripts/generateLicenseInfo.ts
View file

@ -1,18 +1,17 @@
import {existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync} from "fs";
import SmallLicense from "../Models/smallLicense";
import ScriptUtils from "./ScriptUtils";
import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
import SmallLicense from "../Models/smallLicense"
import ScriptUtils from "./ScriptUtils"
const prompt = require('prompt-sync')();
const prompt = require("prompt-sync")()
function validateLicenseInfo(l : SmallLicense){
l.sources.map(s => {
try{
return new URL(s);
}catch (e) {
throw "Could not parse URL "+s+" for a license for "+l.path+" due to "+ e
}
})
function validateLicenseInfo(l: SmallLicense) {
l.sources.map((s) => {
try {
return new URL(s)
} catch (e) {
throw "Could not parse URL " + s + " for a license for " + l.path + " due to " + e
}
})
}
/**
* Sweeps the entire 'assets/' (except assets/generated) directory for image files and any 'license_info.json'-file.
@ -27,18 +26,19 @@ function generateLicenseInfos(paths: string[]): SmallLicense[] {
if (Array.isArray(parsed)) {
const l: SmallLicense[] = parsed
for (const smallLicens of l) {
smallLicens.path = path.substring(0, path.length - "license_info.json".length) + smallLicens.path
smallLicens.path =
path.substring(0, path.length - "license_info.json".length) +
smallLicens.path
}
licenses.push(...l)
} else {
const smallLicens: SmallLicense = parsed;
const smallLicens: SmallLicense = parsed
smallLicens.path = path.substring(0, 1 + path.lastIndexOf("/")) + smallLicens.path
licenses.push(smallLicens)
}
} catch (e) {
console.error("Error: ", e, "while handling", path)
}
}
return licenses
}
@ -53,77 +53,81 @@ function missingLicenseInfos(licenseInfos: SmallLicense[], allIcons: string[]) {
for (const iconPath of allIcons) {
if (iconPath.indexOf("license_info.json") >= 0) {
continue;
continue
}
if (knownPaths.has(iconPath)) {
continue;
continue
}
missing.push(iconPath)
}
return missing;
return missing
}
const knownLicenses = new Map<string, SmallLicense>()
knownLicenses.set("me", {
authors: ["Pieter Vander Vennet"],
path: undefined,
license: "CC0",
sources: []
sources: [],
})
knownLicenses.set("streetcomplete", {
authors: ["Tobias Zwick (westnordost)"],
path: undefined,
license: "CC0",
sources: ["https://github.com/streetcomplete/StreetComplete/tree/master/res/graphics", "https://f-droid.org/packages/de.westnordost.streetcomplete/"]
sources: [
"https://github.com/streetcomplete/StreetComplete/tree/master/res/graphics",
"https://f-droid.org/packages/de.westnordost.streetcomplete/",
],
})
knownLicenses.set("temaki", {
authors: ["Temaki"],
path: undefined,
license: "CC0",
sources: ["https://github.com/ideditor/temaki","https://ideditor.github.io/temaki/docs/"]
sources: ["https://github.com/ideditor/temaki", "https://ideditor.github.io/temaki/docs/"],
})
knownLicenses.set("maki", {
authors: ["Maki"],
path: undefined,
license: "CC0",
sources: ["https://labs.mapbox.com/maki-icons/"]
sources: ["https://labs.mapbox.com/maki-icons/"],
})
knownLicenses.set("t", {
authors: [],
path: undefined,
license: "CC0; trivial",
sources: []
sources: [],
})
knownLicenses.set("na", {
authors: [],
path: undefined,
license: "CC0",
sources: []
sources: [],
})
knownLicenses.set("tv", {
authors: ["Toerisme Vlaanderen"],
path: undefined,
license: "CC0",
sources: ["https://toerismevlaanderen.be/pinjepunt","https://mapcomplete.osm.be/toerisme_vlaanderenn"]
sources: [
"https://toerismevlaanderen.be/pinjepunt",
"https://mapcomplete.osm.be/toerisme_vlaanderenn",
],
})
knownLicenses.set("tvf", {
authors: ["Jo De Baerdemaeker "],
path: undefined,
license: "All rights reserved",
sources: ["https://www.studiotype.be/fonts/flandersart"]
sources: ["https://www.studiotype.be/fonts/flandersart"],
})
knownLicenses.set("twemoji", {
authors: ["Twemoji"],
path: undefined,
license: "CC-BY 4.0",
sources: ["https://github.com/twitter/twemoji"]
sources: ["https://github.com/twitter/twemoji"],
})
function promptLicenseFor(path): SmallLicense {
console.log("License abbreviations:")
knownLicenses.forEach((value, key) => {
@ -133,13 +137,13 @@ function promptLicenseFor(path): SmallLicense {
path = path.substring(path.lastIndexOf("/") + 1)
if (knownLicenses.has(author)) {
const license = knownLicenses.get(author);
license.path = path;
return license;
const license = knownLicenses.get(author)
license.path = path
return license
}
if (author == "s") {
return null;
return null
}
if (author == "Q" || author == "q" || author == "") {
throw "Quitting now!"
@ -152,14 +156,14 @@ function promptLicenseFor(path): SmallLicense {
authors: author.split(";"),
path: path,
license: prompt("What is the license for artwork " + path + "? > "),
sources: prompt("Where was this artwork found? > ").split(";")
sources: prompt("Where was this artwork found? > ").split(";"),
}
}
function createLicenseInfoFor(path): void {
const li = promptLicenseFor(path);
const li = promptLicenseFor(path)
if (li == null) {
return;
return
}
writeFileSync(path + ".license_info.json", JSON.stringify(li, null, " "))
}
@ -185,39 +189,40 @@ function cleanLicenseInfo(allPaths: string[], allLicenseInfos: SmallLicense[]) {
path: license.path,
license: license.license,
authors: license.authors,
sources: license.sources
sources: license.sources,
}
perDirectory.get(dir).push(cloned)
}
perDirectory.forEach((licenses, dir) => {
for (let i = licenses.length - 1; i >= 0; i--) {
const license = licenses[i];
const license = licenses[i]
const path = dir + "/" + license.path
if (!existsSync(path)) {
console.log("Found license for now missing file: ", path, " - removing this license")
console.log(
"Found license for now missing file: ",
path,
" - removing this license"
)
licenses.splice(i, 1)
}
}
licenses.sort((a, b) => a.path < b.path ? -1 : 1)
licenses.sort((a, b) => (a.path < b.path ? -1 : 1))
writeFileSync(dir + "/license_info.json", JSON.stringify(licenses, null, 2))
})
}
function queryMissingLicenses(missingLicenses: string[]) {
process.on('SIGINT', function () {
console.log("Aborting... Bye!");
process.exit();
});
process.on("SIGINT", function () {
console.log("Aborting... Bye!")
process.exit()
})
let i = 1;
let i = 1
for (const missingLicens of missingLicenses) {
console.log(i + " / " + missingLicenses.length)
i++;
i++
if (i < missingLicenses.length - 5) {
// continue
}
@ -227,16 +232,14 @@ function queryMissingLicenses(missingLicenses: string[]) {
console.log("You're through!")
}
/**
* Creates the humongous license_info in the generated assets, containing all licenses with a path relative to the root
* @param licensePaths
*/
function createFullLicenseOverview(licensePaths: string[]) {
const allLicenses: SmallLicense[] = []
for (const licensePath of licensePaths) {
if(!existsSync(licensePath)){
if (!existsSync(licensePath)) {
continue
}
const licenses = <SmallLicense[]>JSON.parse(readFileSync(licensePath, "UTF-8"))
@ -251,43 +254,46 @@ function createFullLicenseOverview(licensePaths: string[]) {
writeFileSync("./assets/generated/license_info.json", JSON.stringify(allLicenses, null, " "))
}
function main(args: string[]){
function main(args: string[]) {
console.log("Checking and compiling license info")
if (!existsSync("./assets/generated")) {
mkdirSync("./assets/generated")
}
let contents = ScriptUtils.readDirRecSync("./assets")
.filter(entry => entry.indexOf("./assets/generated") != 0)
let licensePaths = contents.filter(entry => entry.indexOf("license_info.json") >= 0)
let licenseInfos = generateLicenseInfos(licensePaths);
const artwork = contents.filter(pth => pth.match(/(.svg|.png|.jpg|.ttf|.otf|.woff)$/i) != null)
let contents = ScriptUtils.readDirRecSync("./assets").filter(
(entry) => entry.indexOf("./assets/generated") != 0
)
let licensePaths = contents.filter((entry) => entry.indexOf("license_info.json") >= 0)
let licenseInfos = generateLicenseInfos(licensePaths)
const artwork = contents.filter(
(pth) => pth.match(/(.svg|.png|.jpg|.ttf|.otf|.woff)$/i) != null
)
const missingLicenses = missingLicenseInfos(licenseInfos, artwork)
if (args.indexOf("--prompt") >= 0 || args.indexOf("--query") >= 0) {
queryMissingLicenses(missingLicenses)
return main([])
}
const invalidLicenses = licenseInfos.filter(l => (l.license ?? "") === "").map(l => `License for artwork ${l.path} is empty string or undefined`)
const invalidLicenses = licenseInfos
.filter((l) => (l.license ?? "") === "")
.map((l) => `License for artwork ${l.path} is empty string or undefined`)
for (const licenseInfo of licenseInfos) {
for (const source of licenseInfo.sources) {
if (source == "") {
invalidLicenses.push("Invalid license: empty string in " + JSON.stringify(licenseInfo))
invalidLicenses.push(
"Invalid license: empty string in " + JSON.stringify(licenseInfo)
)
}
try {
new URL(source);
new URL(source)
} catch {
invalidLicenses.push("Not a valid URL: " + source)
}
}
}
if (missingLicenses.length > 0) {
const msg = `There are ${missingLicenses.length} licenses missing and ${invalidLicenses.length} invalid licenses.`
console.log(missingLicenses.concat(invalidLicenses).join("\n"))
@ -296,7 +302,7 @@ function main(args: string[]){
throw msg
}
}
cleanLicenseInfo(licensePaths, licenseInfos)
createFullLicenseOverview(licensePaths)
}

68
scripts/generateStats.ts
View file

@ -1,10 +1,10 @@
import * as known_layers from "../assets/generated/known_layers.json"
import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
import {TagUtils} from "../Logic/Tags/TagUtils";
import {Utils} from "../Utils";
import {writeFileSync} from "fs";
import ScriptUtils from "./ScriptUtils";
import Constants from "../Models/Constants";
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import { TagUtils } from "../Logic/Tags/TagUtils"
import { Utils } from "../Utils"
import { writeFileSync } from "fs"
import ScriptUtils from "./ScriptUtils"
import Constants from "../Models/Constants"
/* Downloads stats on osmSource-tags and keys from tagInfo */
@ -15,7 +15,6 @@ async function main(includeTags = true) {
const keysAndTags = new Map<string, Set<string>>()
for (const layer of layers) {
if (layer.source["geoJson"] !== undefined && !layer.source["isOsmCache"]) {
continue
}
@ -41,37 +40,42 @@ async function main(includeTags = true) {
const keyTotal = new Map<string, number>()
const tagTotal = new Map<string, Map<string, number>>()
await Promise.all(Array.from(keysAndTags.keys()).map(async key => {
const values = keysAndTags.get(key)
const data = await Utils.downloadJson(`https://taginfo.openstreetmap.org/api/4/key/stats?key=${key}`)
const count = data.data.find(item => item.type === "all").count
keyTotal.set(key, count)
console.log(key, "-->", count)
if (values.size > 0) {
tagTotal.set(key, new Map<string, number>())
await Promise.all(
Array.from(values).map(async value => {
const tagData = await Utils.downloadJson(`https://taginfo.openstreetmap.org/api/4/tag/stats?key=${key}&value=${value}`)
const count = tagData.data.find(item => item.type === "all").count
tagTotal.get(key).set(value, count)
console.log(key + "=" + value, "-->", count)
})
await Promise.all(
Array.from(keysAndTags.keys()).map(async (key) => {
const values = keysAndTags.get(key)
const data = await Utils.downloadJson(
`https://taginfo.openstreetmap.org/api/4/key/stats?key=${key}`
)
const count = data.data.find((item) => item.type === "all").count
keyTotal.set(key, count)
console.log(key, "-->", count)
}
}))
writeFileSync("./assets/key_totals.json",
if (values.size > 0) {
tagTotal.set(key, new Map<string, number>())
await Promise.all(
Array.from(values).map(async (value) => {
const tagData = await Utils.downloadJson(
`https://taginfo.openstreetmap.org/api/4/tag/stats?key=${key}&value=${value}`
)
const count = tagData.data.find((item) => item.type === "all").count
tagTotal.get(key).set(value, count)
console.log(key + "=" + value, "-->", count)
})
)
}
})
)
writeFileSync(
"./assets/key_totals.json",
JSON.stringify(
{
keys: Utils.MapToObj(keyTotal, t => t),
tags: Utils.MapToObj(tagTotal, v => Utils.MapToObj(v, t => t))
keys: Utils.MapToObj(keyTotal, (t) => t),
tags: Utils.MapToObj(tagTotal, (v) => Utils.MapToObj(v, (t) => t)),
},
null, " "
null,
" "
)
)
}
main().then(() => console.log("All done"))
main().then(() => console.log("All done"))

162
scripts/generateTaginfoProjectFiles.ts
View file

@ -1,11 +1,11 @@
import {AllKnownLayouts} from "../Customizations/AllKnownLayouts";
import Locale from "../UI/i18n/Locale";
import {Translation} from "../UI/i18n/Translation";
import {readFileSync, writeFileSync} from "fs";
import LayoutConfig from "../Models/ThemeConfig/LayoutConfig";
import LayerConfig from "../Models/ThemeConfig/LayerConfig";
import Constants from "../Models/Constants";
import {Utils} from "../Utils";
import { AllKnownLayouts } from "../Customizations/AllKnownLayouts"
import Locale from "../UI/i18n/Locale"
import { Translation } from "../UI/i18n/Translation"
import { readFileSync, writeFileSync } from "fs"
import LayoutConfig from "../Models/ThemeConfig/LayoutConfig"
import LayerConfig from "../Models/ThemeConfig/LayerConfig"
import Constants from "../Models/Constants"
import { Utils } from "../Utils"
/**
* Generates all the files in "Docs/TagInfo". These are picked up by the taginfo project, showing a link to the mapcomplete theme if the key is used
@ -13,17 +13,20 @@ import {Utils} from "../Utils";
const outputDirectory = "Docs/TagInfo"
function generateTagOverview(kv: { k: string, v: string }, description: string): {
key: string,
description: string,
function generateTagOverview(
kv: { k: string; v: string },
description: string
): {
key: string
description: string
value?: string
} {
const overview = {
// OSM tag key (required)
key: kv.k,
description: description,
value: undefined
};
value: undefined,
}
if (kv.v !== undefined) {
// OSM tag value (optional, if not supplied it means "all values")
overview.value = kv.v
@ -31,20 +34,24 @@ function generateTagOverview(kv: { k: string, v: string }, description: string):
return overview
}
function generateLayerUsage(layer: LayerConfig, layout: LayoutConfig): any [] {
function generateLayerUsage(layer: LayerConfig, layout: LayoutConfig): any[] {
if (layer.name === undefined) {
return [] // Probably a duplicate or irrelevant layer
}
const usedTags = layer.source.osmTags.asChange({})
const result: {
key: string,
description: string,
key: string
description: string
value?: string
}[] = []
for (const kv of usedTags) {
const description = "The MapComplete theme " + layout.title.txt + " has a layer " + layer.name.txt + " showing features with this tag"
const description =
"The MapComplete theme " +
layout.title.txt +
" has a layer " +
layer.name.txt +
" showing features with this tag"
result.push(generateTagOverview(kv, description))
}
@ -54,54 +61,57 @@ function generateLayerUsage(layer: LayerConfig, layout: LayoutConfig): any [] {
const usesImageUpload = (tr.render?.txt?.indexOf("image_upload") ?? -2) > 0
if (usesImageCarousel || usesImageUpload) {
const descrNoUpload = `The layer '${layer.name.txt} shows images based on the keys image, image:0, image:1,... and wikidata, wikipedia, wikimedia_commons and mapillary`
const descrUpload = `The layer '${layer.name.txt} allows to upload images and adds them under the 'image'-tag (and image:0, image:1, ... for multiple images). Furhtermore, this layer shows images based on the keys image, wikidata, wikipedia, wikimedia_commons and mapillary`
const descrNoUpload = `The layer '${layer.name.txt} shows images based on the keys image, image:0, image:1,... and wikidata, wikipedia, wikimedia_commons and mapillary`;
const descrUpload = `The layer '${layer.name.txt} allows to upload images and adds them under the 'image'-tag (and image:0, image:1, ... for multiple images). Furhtermore, this layer shows images based on the keys image, wikidata, wikipedia, wikimedia_commons and mapillary`;
const descr = usesImageUpload ? descrUpload : descrNoUpload;
result.push(generateTagOverview({k: "image", v: undefined}, descr));
result.push(generateTagOverview({k: "mapillary", v: undefined}, descr));
result.push(generateTagOverview({k: "wikidata", v: undefined}, descr));
result.push(generateTagOverview({k: "wikipedia", v: undefined}, descr));
const descr = usesImageUpload ? descrUpload : descrNoUpload
result.push(generateTagOverview({ k: "image", v: undefined }, descr))
result.push(generateTagOverview({ k: "mapillary", v: undefined }, descr))
result.push(generateTagOverview({ k: "wikidata", v: undefined }, descr))
result.push(generateTagOverview({ k: "wikipedia", v: undefined }, descr))
}
}
const q = tr.question?.txt;
const key = tr.freeform?.key;
const q = tr.question?.txt
const key = tr.freeform?.key
if (key != undefined) {
let descr = `Layer '${layer.name.txt}'`;
let descr = `Layer '${layer.name.txt}'`
if (q == undefined) {
descr += " shows values with";
descr += " shows values with"
} else {
descr += " shows and asks freeform values for"
}
descr += ` key '${key}' (in the MapComplete.osm.be theme '${layout.title.txt}')`
result.push(generateTagOverview({k: key, v: undefined}, descr))
result.push(generateTagOverview({ k: key, v: undefined }, descr))
}
const mappings = tr.mappings ?? []
for (const mapping of mappings) {
let descr = "Layer '" + layer.name.txt + "'";
descr += " shows " + mapping.if.asHumanString(false, false, {}) + " with a fixed text, namely '" + mapping.then.txt + "'";
if (q != undefined
&& mapping.hideInAnswer != true // != true will also match if a
let descr = "Layer '" + layer.name.txt + "'"
descr +=
" shows " +
mapping.if.asHumanString(false, false, {}) +
" with a fixed text, namely '" +
mapping.then.txt +
"'"
if (
q != undefined &&
mapping.hideInAnswer != true // != true will also match if a
) {
descr += " and allows to pick this as a default answer"
}
descr += ` (in the MapComplete.osm.be theme '${layout.title.txt}')`
for (const kv of mapping.if.asChange({})) {
let d = descr;
let d = descr
if (q != undefined && kv.v == "") {
d = `${descr} Picking this answer will delete the key ${kv.k}.`
}
result.push(generateTagOverview(kv, d))
}
}
}
return result.filter(result => !result.key.startsWith("_"))
return result.filter((result) => !result.key.startsWith("_"))
}
/**
@ -111,39 +121,38 @@ function generateLayerUsage(layer: LayerConfig, layout: LayoutConfig): any [] {
function generateTagInfoEntry(layout: LayoutConfig): any {
const usedTags = []
for (const layer of layout.layers) {
if(Constants.priviliged_layers.indexOf(layer.id) >= 0){
if (Constants.priviliged_layers.indexOf(layer.id) >= 0) {
continue
}
if(layer.source.geojsonSource !== undefined && layer.source.isOsmCacheLayer !== true){
if (layer.source.geojsonSource !== undefined && layer.source.isOsmCacheLayer !== true) {
continue
}
usedTags.push(...generateLayerUsage(layer, layout))
}
if(usedTags.length == 0){
if (usedTags.length == 0) {
return undefined
}
let icon = layout.icon;
let icon = layout.icon
if (icon.startsWith("./")) {
icon = icon.substring(2)
}
const themeInfo = {
// data format version, currently always 1, will get updated if there are incompatible changes to the format (required)
"data_format": 1,
data_format: 1,
// timestamp when project file was updated is not given as it pollutes the github history
"project": {
"name": "MapComplete " + layout.title.txt, // name of the project (required)
"description": layout.shortDescription.txt, // short description of the project (required)
"project_url": "https://mapcomplete.osm.be/" + layout.id, // home page of the project with general information (required)
"doc_url": "https://github.com/pietervdvn/MapComplete/tree/master/assets/themes/", // documentation of the project and especially the tags used (optional)
"icon_url": "https://mapcomplete.osm.be/" + icon, // project logo, should work in 16x16 pixels on white and light gray backgrounds (optional)
"contact_name": "Pieter Vander Vennet", // contact name, needed for taginfo maintainer (required)
"contact_email": "pietervdvn@posteo.net" // contact email, needed for taginfo maintainer (required)
project: {
name: "MapComplete " + layout.title.txt, // name of the project (required)
description: layout.shortDescription.txt, // short description of the project (required)
project_url: "https://mapcomplete.osm.be/" + layout.id, // home page of the project with general information (required)
doc_url: "https://github.com/pietervdvn/MapComplete/tree/master/assets/themes/", // documentation of the project and especially the tags used (optional)
icon_url: "https://mapcomplete.osm.be/" + icon, // project logo, should work in 16x16 pixels on white and light gray backgrounds (optional)
contact_name: "Pieter Vander Vennet", // contact name, needed for taginfo maintainer (required)
contact_email: "pietervdvn@posteo.net", // contact email, needed for taginfo maintainer (required)
},
tags: usedTags
tags: usedTags,
}
const filename = "mapcomplete_" + layout.id
@ -158,39 +167,40 @@ function generateProjectsOverview(files: string[]) {
const tagInfoList = "../taginfo-projects/project_list.txt"
let projectList = readFileSync(tagInfoList, "UTF8")
.split("\n")
.filter(entry => entry.indexOf("mapcomplete_") < 0)
.concat(files.map(f => `${f} https://raw.githubusercontent.com/pietervdvn/MapComplete/develop/Docs/TagInfo/${f}.json`))
.filter((entry) => entry.indexOf("mapcomplete_") < 0)
.concat(
files.map(
(f) =>
`${f} https://raw.githubusercontent.com/pietervdvn/MapComplete/develop/Docs/TagInfo/${f}.json`
)
)
.sort()
.filter(entry => entry != "")
console.log("Writing taginfo project filelist");
writeFileSync(tagInfoList, projectList.join("\n") + "\n");
.filter((entry) => entry != "")
console.log("Writing taginfo project filelist")
writeFileSync(tagInfoList, projectList.join("\n") + "\n")
} catch (e) {
console.warn("Could not write the taginfo-projects list - the repository is probably not checked out. Are you creating a fork? Ignore this message then.")
console.warn(
"Could not write the taginfo-projects list - the repository is probably not checked out. Are you creating a fork? Ignore this message then."
)
}
}
function main() {
console.log("Creating taginfo project files")
function main(){
console.log("Creating taginfo project files")
Locale.language.setData("en")
Translation.forcedLanguage = "en"
Locale.language.setData("en")
Translation.forcedLanguage = "en"
const files = []
for (const layout of AllKnownLayouts.layoutsList) {
if (layout.hideFromOverview) {
continue;
continue
}
files.push(generateTagInfoEntry(layout));
files.push(generateTagInfoEntry(layout))
}
generateProjectsOverview(Utils.NoNull(files));
generateProjectsOverview(Utils.NoNull(files))
}
main()
main()

19
scripts/generateTileOverview.ts
View file

@ -1,20 +1,17 @@
/**
* Generates an overview for which tiles exist and which don't
*/
import ScriptUtils from "./ScriptUtils";
import {writeFileSync} from "fs";
import ScriptUtils from "./ScriptUtils"
import { writeFileSync } from "fs"
function main(args: string[]) {
const directory = args[0]
let zoomLevel = args[1]
const files = ScriptUtils.readDirRecSync(directory, 1)
.filter(f => f.endsWith(".geojson"))
const files = ScriptUtils.readDirRecSync(directory, 1).filter((f) => f.endsWith(".geojson"))
const indices /* Map<string, number[]>*/ = {}
for (const path of files) {
const match = path.match(".*_\([0-9]*\)_\([0-9]*\)_\([0-9]*\).geojson")
const match = path.match(".*_([0-9]*)_([0-9]*)_([0-9]*).geojson")
if (match === null) {
continue
}
@ -32,8 +29,8 @@ function main(args: string[]) {
}
indices[x].push(Number(y))
}
indices["zoom"] = zoomLevel;
const match = files[0].match("\(.*\)_\([0-9]*\)_\([0-9]*\)_\([0-9]*\).geojson")
indices["zoom"] = zoomLevel
const match = files[0].match("(.*)_([0-9]*)_([0-9]*)_([0-9]*).geojson")
const path = match[1] + "_" + zoomLevel + "_overview.json"
writeFileSync(path, JSON.stringify(indices))
console.log("Written overview for", files.length, " tiles at", path)
@ -41,10 +38,10 @@ function main(args: string[]) {
let args = [...process.argv]
args.splice(0, 2)
args.forEach(file => {
args.forEach((file) => {
try {
main(args)
} catch (e) {
console.error("Could not handle file ", file, " due to ", e)
}
})
})

341
scripts/generateTranslations.ts
View file

@ -1,16 +1,15 @@
import * as fs from "fs";
import {existsSync, mkdirSync, readFileSync, writeFileSync} from "fs";
import {Utils} from "../Utils";
import ScriptUtils from "./ScriptUtils";
import * as fs from "fs"
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
import { Utils } from "../Utils"
import ScriptUtils from "./ScriptUtils"
const knownLanguages = ["en", "nl", "de", "fr", "es", "gl", "ca"];
const knownLanguages = ["en", "nl", "de", "fr", "es", "gl", "ca"]
class TranslationPart {
contents: Map<string, TranslationPart | string> = new Map<string, TranslationPart | string>()
static fromDirectory(path): TranslationPart {
const files = ScriptUtils.readDirRecSync(path, 1).filter(file => file.endsWith(".json"))
const files = ScriptUtils.readDirRecSync(path, 1).filter((file) => file.endsWith(".json"))
const rootTranslation = new TranslationPart()
for (const file of files) {
const content = JSON.parse(readFileSync(file, "UTF8"))
@ -37,36 +36,49 @@ class TranslationPart {
} else {
subpart.add(language, v)
}
}
}
addTranslationObject(translations: any, context?: string) {
if (translations["#"] === "no-translations") {
console.log("Ignoring object at ", context, "which has '#':'no-translations'")
return;
return
}
for (const translationsKey in translations) {
if (!translations.hasOwnProperty(translationsKey)) {
continue;
continue
}
const v = translations[translationsKey]
if (typeof (v) != "string") {
console.error(`Non-string object at ${context} in translation while trying to add the translation ` + JSON.stringify(v) + ` to '` + translationsKey + "'. The offending object which _should_ be a translation is: ", v, "\n\nThe current object is (only showing en):", this.toJson(), "and has translations for", Array.from(this.contents.keys()))
throw "Error in an object depicting a translation: a non-string object was found. (" + context + ")\n You probably put some other section accidentally in the translation"
if (typeof v != "string") {
console.error(
`Non-string object at ${context} in translation while trying to add the translation ` +
JSON.stringify(v) +
` to '` +
translationsKey +
"'. The offending object which _should_ be a translation is: ",
v,
"\n\nThe current object is (only showing en):",
this.toJson(),
"and has translations for",
Array.from(this.contents.keys())
)
throw (
"Error in an object depicting a translation: a non-string object was found. (" +
context +
")\n You probably put some other section accidentally in the translation"
)
}
this.contents.set(translationsKey, v)
}
}
recursiveAdd(object: any, context: string) {
const isProbablyTranslationObject = knownLanguages.some(l => object.hasOwnProperty(l)); // TODO FIXME ID
const isProbablyTranslationObject = knownLanguages.some((l) => object.hasOwnProperty(l)) // TODO FIXME ID
if (isProbablyTranslationObject) {
this.addTranslationObject(object, context)
return;
return
}
let dontTranslateKeys: string[] = undefined
{
const noTranslate = <string | string[]>object["#dont-translate"]
@ -80,13 +92,18 @@ class TranslationPart {
dontTranslateKeys = noTranslate
}
if (noTranslate !== undefined) {
console.log("Ignoring some translations for " + context + ": " + dontTranslateKeys.join(", "))
console.log(
"Ignoring some translations for " +
context +
": " +
dontTranslateKeys.join(", ")
)
}
}
for (let key in object) {
if (!object.hasOwnProperty(key)) {
continue;
continue
}
if (dontTranslateKeys?.indexOf(key) >= 0) {
@ -99,7 +116,7 @@ class TranslationPart {
continue
}
if (typeof v !== "object") {
continue;
continue
}
if (context.endsWith(".tagRenderings")) {
@ -107,17 +124,21 @@ class TranslationPart {
if (v["builtin"] !== undefined && typeof v["builtin"] === "string") {
key = v["builtin"]
} else {
throw "At " + context + ": every object within a tagRenderings-list should have an id. " + JSON.stringify(v) + " has no id"
throw (
"At " +
context +
": every object within a tagRenderings-list should have an id. " +
JSON.stringify(v) +
" has no id"
)
}
} else {
// We use the embedded id as key instead of the index as this is more stable
// Note: indonesian is shortened as 'id' as well!
if (v["en"] !== undefined || v["nl"] !== undefined) {
// This is probably a translation already!
// pass
} else {
key = v["id"]
if (typeof key !== "string") {
throw "Panic: found a non-string ID at" + context
@ -126,31 +147,29 @@ class TranslationPart {
}
}
if (!this.contents.get(key)) {
this.contents.set(key, new TranslationPart())
}
(this.contents.get(key) as TranslationPart).recursiveAdd(v, context + "." + key);
;(this.contents.get(key) as TranslationPart).recursiveAdd(v, context + "." + key)
}
}
knownLanguages(): string[] {
const languages = []
for (let key of Array.from(this.contents.keys())) {
const value = this.contents.get(key);
const value = this.contents.get(key)
if (typeof value === "string") {
if (key === "#") {
continue;
continue
}
languages.push(key)
} else {
languages.push(...(value as TranslationPart).knownLanguages())
}
}
return Utils.Dedup(languages);
return Utils.Dedup(languages)
}
toJson(neededLanguage?: string): string {
@ -158,35 +177,34 @@ class TranslationPart {
let keys = Array.from(this.contents.keys())
keys = keys.sort()
for (let key of keys) {
let value = this.contents.get(key);
let value = this.contents.get(key)
if (typeof value === "string") {
value = value.replace(/"/g, "\\\"")
.replace(/\n/g, "\\n")
value = value.replace(/"/g, '\\"').replace(/\n/g, "\\n")
if (neededLanguage === undefined) {
parts.push(`\"${key}\": \"${value}\"`)
} else if (key === neededLanguage) {
return `"${value}"`
}
} else {
const sub = (value as TranslationPart).toJson(neededLanguage)
if (sub !== "") {
parts.push(`\"${key}\": ${sub}`);
parts.push(`\"${key}\": ${sub}`)
}
}
}
if (parts.length === 0) {
return "";
return ""
}
return `{${parts.join(",")}}`;
return `{${parts.join(",")}}`
}
validateStrict(ctx?: string): void {
const errors = this.validate()
for (const err of errors) {
console.error("ERROR in " + (ctx ?? "") + " " + err.path.join(".") + "\n " + err.error)
console.error(
"ERROR in " + (ctx ?? "") + " " + err.path.join(".") + "\n " + err.error
)
}
if (errors.length > 0) {
throw ctx + " has " + errors.length + " inconsistencies in the translation"
@ -196,21 +214,24 @@ class TranslationPart {
/**
* Checks the leaf objects: special values must be present and identical in every leaf
*/
validate(path = []): { error: string, path: string[] } [] {
const errors: { error: string, path: string[] } [] = []
const neededSubparts = new Set<{ part: string, usedByLanguage: string }>()
validate(path = []): { error: string; path: string[] }[] {
const errors: { error: string; path: string[] }[] = []
const neededSubparts = new Set<{ part: string; usedByLanguage: string }>()
let isLeaf: boolean = undefined
this.contents.forEach((value, key) => {
if (typeof value !== "string") {
const recErrors = value.validate([...path, key])
errors.push(...recErrors)
return;
return
}
if (isLeaf === undefined) {
isLeaf = true
} else if (!isLeaf) {
errors.push({error: "Mixed node: non-leaf node has translation strings", path: path})
errors.push({
error: "Mixed node: non-leaf node has translation strings",
path: path,
})
}
let subparts: string[] = value.match(/{[^}]*}/g)
@ -220,19 +241,18 @@ class TranslationPart {
// This is a core translation, it has one less path segment
lang = weblatepart
}
subparts = subparts.map(p => p.split(/\(.*\)/)[0])
subparts = subparts.map((p) => p.split(/\(.*\)/)[0])
for (const subpart of subparts) {
neededSubparts.add({part: subpart, usedByLanguage: lang})
neededSubparts.add({ part: subpart, usedByLanguage: lang })
}
}
})
// Actually check for the needed sub-parts, e.g. that {key} isn't translated into {sleutel}
this.contents.forEach((value, key) => {
neededSubparts.forEach(({part, usedByLanguage}) => {
neededSubparts.forEach(({ part, usedByLanguage }) => {
if (typeof value !== "string") {
return;
return
}
let [_, __, weblatepart, lang] = key.split("/")
if (lang === undefined) {
@ -240,26 +260,40 @@ class TranslationPart {
lang = weblatepart
weblatepart = "core"
}
const fixLink = `Fix it on https://hosted.weblate.org/translate/mapcomplete/${weblatepart}/${lang}/?offset=1&q=context%3A%3D%22${encodeURIComponent(path.join("."))}%22`;
const fixLink = `Fix it on https://hosted.weblate.org/translate/mapcomplete/${weblatepart}/${lang}/?offset=1&q=context%3A%3D%22${encodeURIComponent(
path.join(".")
)}%22`
let subparts: string[] = value.match(/{[^}]*}/g)
if (subparts === null) {
if (neededSubparts.size > 0) {
errors.push({
error: "The translation for " + key + " does not have any subparts, but expected " + Array.from(neededSubparts).map(part => part.part + " (used in " + part.usedByLanguage + ")").join(",") + " . The full translation is " + value + "\n" + fixLink,
path: path
error:
"The translation for " +
key +
" does not have any subparts, but expected " +
Array.from(neededSubparts)
.map(
(part) =>
part.part + " (used in " + part.usedByLanguage + ")"
)
.join(",") +
" . The full translation is " +
value +
"\n" +
fixLink,
path: path,
})
}
return
}
subparts = subparts.map(p => p.split(/\(.*\)/)[0])
subparts = subparts.map((p) => p.split(/\(.*\)/)[0])
if (subparts.indexOf(part) < 0) {
if (lang === "en" || usedByLanguage === "en") {
errors.push({
error: `The translation for ${key} does not have the required subpart ${part} (in ${usedByLanguage}).
\tThe full translation is ${value}
\t${fixLink}`,
path: path
path: path,
})
}
}
@ -278,7 +312,7 @@ class TranslationPart {
private addTranslation(language: string, object: any) {
for (const key in object) {
const v = object[key]
if(v === ""){
if (v === "") {
delete object[key]
continue
}
@ -293,9 +327,7 @@ class TranslationPart {
subpart.addTranslation(language, v)
}
}
}
}
/**
@ -308,10 +340,10 @@ function isTranslation(tr: any): boolean {
}
for (const key in tr) {
if (typeof tr[key] !== "string") {
return false;
return false
}
}
return true;
return true
}
/**
@ -319,8 +351,11 @@ function isTranslation(tr: any): boolean {
*
* To debug the 'compiledTranslations', add a languageWhiteList to only generate a single language
*/
function transformTranslation(obj: any, path: string[] = [], languageWhitelist: string[] = undefined) {
function transformTranslation(
obj: any,
path: string[] = [],
languageWhitelist: string[] = undefined
) {
if (isTranslation(obj)) {
return `new Translation( ${JSON.stringify(obj)} )`
}
@ -328,7 +363,7 @@ function transformTranslation(obj: any, path: string[] = [], languageWhitelist:
let values = ""
for (const key in obj) {
if (key === "#") {
continue;
continue
}
if (key.match("^[a-zA-Z0-9_]*$") === null) {
@ -342,33 +377,46 @@ function transformTranslation(obj: any, path: string[] = [], languageWhitelist:
for (const ln of languageWhitelist) {
nv[ln] = value[ln]
}
value = nv;
value = nv
}
if (value["en"] === undefined) {
throw `At ${path.join(".")}: Missing 'en' translation at path ${path.join(".")}.${key}\n\tThe translations in other languages are ${JSON.stringify(value)}`
throw `At ${path.join(".")}: Missing 'en' translation at path ${path.join(
"."
)}.${key}\n\tThe translations in other languages are ${JSON.stringify(value)}`
}
const subParts: string[] = value["en"].match(/{[^}]*}/g)
let expr = `return new Translation(${JSON.stringify(value)}, "core:${path.join(".")}.${key}")`
let expr = `return new Translation(${JSON.stringify(value)}, "core:${path.join(
"."
)}.${key}")`
if (subParts !== null) {
// convert '{to_substitute}' into 'to_substitute'
const types = Utils.Dedup(subParts.map(tp => tp.substring(1, tp.length - 1)))
const invalid = types.filter(part => part.match(/^[a-z0-9A-Z_]+(\(.*\))?$/) == null)
const types = Utils.Dedup(subParts.map((tp) => tp.substring(1, tp.length - 1)))
const invalid = types.filter(
(part) => part.match(/^[a-z0-9A-Z_]+(\(.*\))?$/) == null
)
if (invalid.length > 0) {
throw `At ${path.join(".")}: A subpart contains invalid characters: ${subParts.join(', ')}`
throw `At ${path.join(
"."
)}: A subpart contains invalid characters: ${subParts.join(", ")}`
}
expr = `return new TypedTranslation<{ ${types.join(", ")} }>(${JSON.stringify(value)}, "core:${path.join(".")}.${key}")`
expr = `return new TypedTranslation<{ ${types.join(", ")} }>(${JSON.stringify(
value
)}, "core:${path.join(".")}.${key}")`
}
values += `${Utils.Times((_) => " ", path.length + 1)}get ${key}() { ${expr} },
`
} else {
values += (Utils.Times((_) => " ", path.length + 1)) + key + ": " + transformTranslation(value, [...path, key], languageWhitelist) + ",\n"
values +=
Utils.Times((_) => " ", path.length + 1) +
key +
": " +
transformTranslation(value, [...path, key], languageWhitelist) +
",\n"
}
}
return `{${values}}`;
return `{${values}}`
}
function sortKeys(o: object): object {
@ -386,14 +434,13 @@ function sortKeys(o: object): object {
return nw
}
function removeEmptyString(object: object) {
for (const k in object) {
if(object[k] === ""){
if (object[k] === "") {
delete object[k]
continue
}
if(typeof object[k] === "object"){
if (typeof object[k] === "object") {
removeEmptyString(object[k])
}
}
@ -415,15 +462,16 @@ function formatFile(path) {
* Generates the big compiledTranslations file
*/
function genTranslations() {
const translations = JSON.parse(fs.readFileSync("./assets/generated/translations.json", "utf-8"))
const transformed = transformTranslation(translations);
const translations = JSON.parse(
fs.readFileSync("./assets/generated/translations.json", "utf-8")
)
const transformed = transformTranslation(translations)
let module = `import {Translation, TypedTranslation} from "../../UI/i18n/Translation"\n\nexport default class CompiledTranslations {\n\n`;
module += " public static t = " + transformed;
let module = `import {Translation, TypedTranslation} from "../../UI/i18n/Translation"\n\nexport default class CompiledTranslations {\n\n`
module += " public static t = " + transformed
module += "\n }"
fs.writeFileSync("./assets/generated/CompiledTranslations.ts", module);
fs.writeFileSync("./assets/generated/CompiledTranslations.ts", module)
}
/**
@ -431,18 +479,17 @@ function genTranslations() {
* This is only for the core translations
*/
function compileTranslationsFromWeblate() {
const translations = ScriptUtils.readDirRecSync("./langs", 1)
.filter(path => path.indexOf(".json") > 0)
const translations = ScriptUtils.readDirRecSync("./langs", 1).filter(
(path) => path.indexOf(".json") > 0
)
const allTranslations = new TranslationPart()
allTranslations.validateStrict()
for (const translationFile of translations) {
try {
const contents = JSON.parse(readFileSync(translationFile, "utf-8"));
const contents = JSON.parse(readFileSync(translationFile, "utf-8"))
let language = translationFile.substring(translationFile.lastIndexOf("/") + 1)
language = language.substring(0, language.length - 5)
allTranslations.add(language, contents)
@ -451,8 +498,10 @@ function compileTranslationsFromWeblate() {
}
}
writeFileSync("./assets/generated/translations.json", JSON.stringify(JSON.parse(allTranslations.toJson()), null, " "))
writeFileSync(
"./assets/generated/translations.json",
JSON.stringify(JSON.parse(allTranslations.toJson()), null, " ")
)
}
/**
@ -460,12 +509,15 @@ function compileTranslationsFromWeblate() {
* @param objects
* @param target
*/
function generateTranslationsObjectFrom(objects: { path: string, parsed: { id: string } }[], target: string): string[] {
const tr = new TranslationPart();
function generateTranslationsObjectFrom(
objects: { path: string; parsed: { id: string } }[],
target: string
): string[] {
const tr = new TranslationPart()
for (const layerFile of objects) {
const config: { id: string } = layerFile.parsed;
const layerTr = new TranslationPart();
const config: { id: string } = layerFile.parsed
const layerTr = new TranslationPart()
if (config === undefined) {
throw "Got something not parsed! Path is " + layerFile.path
}
@ -473,16 +525,15 @@ function generateTranslationsObjectFrom(objects: { path: string, parsed: { id: s
tr.contents.set(config.id, layerTr)
}
const langs = tr.knownLanguages();
const langs = tr.knownLanguages()
for (const lang of langs) {
if (lang === "#" || lang === "*") {
// Lets not export our comments or non-translated stuff
continue;
continue
}
let json = tr.toJson(lang)
try {
json = JSON.stringify(JSON.parse(json), null, " "); // MUST BE FOUR SPACES
json = JSON.stringify(JSON.parse(json), null, " ") // MUST BE FOUR SPACES
} catch (e) {
console.error(e)
}
@ -501,7 +552,6 @@ function generateTranslationsObjectFrom(objects: { path: string, parsed: { id: s
* @constructor
*/
function MergeTranslation(source: any, target: any, language: string, context: string = "") {
let keyRemapping: Map<string, string> = undefined
if (context.endsWith(".tagRenderings")) {
keyRemapping = new Map<string, string>()
@ -515,29 +565,36 @@ function MergeTranslation(source: any, target: any, language: string, context: s
continue
}
const sourceV = source[key];
const sourceV = source[key]
const targetV = target[keyRemapping?.get(key) ?? key]
if (typeof sourceV === "string") {
// Add the translation
if (targetV === undefined) {
if (typeof target === "string") {
throw "Trying to merge a translation into a fixed string at " + context + " for key " + key;
throw (
"Trying to merge a translation into a fixed string at " +
context +
" for key " +
key
)
}
target[key] = source[key];
continue;
target[key] = source[key]
continue
}
if (targetV[language] === sourceV) {
// Already the same
continue;
continue
}
if (typeof targetV === "string") {
throw `At context ${context}: Could not add a translation in language ${language}. The target object has a string at the given path, whereas the translation contains an object.\n String at target: ${targetV}\n Object at translation source: ${JSON.stringify(sourceV)}`
throw `At context ${context}: Could not add a translation in language ${language}. The target object has a string at the given path, whereas the translation contains an object.\n String at target: ${targetV}\n Object at translation source: ${JSON.stringify(
sourceV
)}`
}
targetV[language] = sourceV;
targetV[language] = sourceV
let was = ""
if (targetV[language] !== undefined && targetV[language] !== sourceV) {
was = " (overwritten " + targetV[language] + ")"
@ -548,35 +605,38 @@ function MergeTranslation(source: any, target: any, language: string, context: s
if (typeof sourceV === "object") {
if (targetV === undefined) {
try {
target[language] = sourceV;
target[language] = sourceV
} catch (e) {
throw `At context${context}: Could not add a translation in language ${language} due to ${e}`
}
} else {
MergeTranslation(sourceV, targetV, language, context + "." + key);
MergeTranslation(sourceV, targetV, language, context + "." + key)
}
continue;
continue
}
throw "Case fallthrough"
}
return target;
return target
}
function mergeLayerTranslation(layerConfig: { id: string }, path: string, translationFiles: Map<string, any>) {
const id = layerConfig.id;
function mergeLayerTranslation(
layerConfig: { id: string },
path: string,
translationFiles: Map<string, any>
) {
const id = layerConfig.id
translationFiles.forEach((translations, lang) => {
const translationsForLayer = translations[id]
MergeTranslation(translationsForLayer, layerConfig, lang, path + ":" + id)
})
}
function loadTranslationFilesFrom(target: string): Map<string, any> {
const translationFilePaths = ScriptUtils.readDirRecSync("./langs/" + target)
.filter(path => path.endsWith(".json"))
const translationFilePaths = ScriptUtils.readDirRecSync("./langs/" + target).filter((path) =>
path.endsWith(".json")
)
const translationFiles = new Map<string, any>();
const translationFiles = new Map<string, any>()
for (const translationFilePath of translationFilePaths) {
let language = translationFilePath.substr(translationFilePath.lastIndexOf("/") + 1)
language = language.substr(0, language.length - 5)
@ -584,18 +644,17 @@ function loadTranslationFilesFrom(target: string): Map<string, any> {
translationFiles.set(language, JSON.parse(readFileSync(translationFilePath, "utf8")))
} catch (e) {
console.error("Invalid JSON file or file does not exist", translationFilePath)
throw e;
throw e
}
}
return translationFiles;
return translationFiles
}
/**
* Load the translations from the weblate files back into the layers
*/
function mergeLayerTranslations() {
const layerFiles = ScriptUtils.getLayerFiles();
const layerFiles = ScriptUtils.getLayerFiles()
for (const layerFile of layerFiles) {
mergeLayerTranslation(layerFile.parsed, layerFile.path, loadTranslationFilesFrom("layers"))
writeFileSync(layerFile.path, JSON.stringify(layerFile.parsed, null, " ")) // layers use 2 spaces
@ -606,12 +665,12 @@ function mergeLayerTranslations() {
* Load the translations into the theme files
*/
function mergeThemeTranslations() {
const themeFiles = ScriptUtils.getThemeFiles();
const themeFiles = ScriptUtils.getThemeFiles()
for (const themeFile of themeFiles) {
const config = themeFile.parsed;
const config = themeFile.parsed
mergeLayerTranslation(config, themeFile.path, loadTranslationFilesFrom("themes"))
const allTranslations = new TranslationPart();
const allTranslations = new TranslationPart()
allTranslations.recursiveAdd(config, themeFile.path)
writeFileSync(themeFile.path, JSON.stringify(config, null, " ")) // Themefiles use 2 spaces
}
@ -622,32 +681,46 @@ if (!existsSync("./langs/themes")) {
}
const themeOverwritesWeblate = process.argv[2] === "--ignore-weblate"
const questionsPath = "assets/tagRenderings/questions.json"
const questionsParsed = JSON.parse(readFileSync(questionsPath, 'utf8'))
const questionsParsed = JSON.parse(readFileSync(questionsPath, "utf8"))
if (!themeOverwritesWeblate) {
mergeLayerTranslations();
mergeThemeTranslations();
mergeLayerTranslations()
mergeThemeTranslations()
mergeLayerTranslation(questionsParsed, questionsPath, loadTranslationFilesFrom("shared-questions"))
mergeLayerTranslation(
questionsParsed,
questionsPath,
loadTranslationFilesFrom("shared-questions")
)
writeFileSync(questionsPath, JSON.stringify(questionsParsed, null, " "))
} else {
console.log("Ignore weblate")
}
const l1 = generateTranslationsObjectFrom(ScriptUtils.getLayerFiles(), "layers")
const l2 = generateTranslationsObjectFrom(ScriptUtils.getThemeFiles().filter(th => th.parsed.mustHaveLanguage === undefined), "themes")
const l3 = generateTranslationsObjectFrom([{path: questionsPath, parsed: questionsParsed}], "shared-questions")
const l2 = generateTranslationsObjectFrom(
ScriptUtils.getThemeFiles().filter((th) => th.parsed.mustHaveLanguage === undefined),
"themes"
)
const l3 = generateTranslationsObjectFrom(
[{ path: questionsPath, parsed: questionsParsed }],
"shared-questions"
)
const usedLanguages: string[] = Utils.Dedup(l1.concat(l2).concat(l3)).filter(v => v !== "*")
const usedLanguages: string[] = Utils.Dedup(l1.concat(l2).concat(l3)).filter((v) => v !== "*")
usedLanguages.sort()
fs.writeFileSync("./assets/generated/used_languages.json", JSON.stringify({languages: usedLanguages}))
fs.writeFileSync(
"./assets/generated/used_languages.json",
JSON.stringify({ languages: usedLanguages })
)
if (!themeOverwritesWeblate) {
// Generates the core translations
compileTranslationsFromWeblate();
// Generates the core translations
compileTranslationsFromWeblate()
}
genTranslations()
const allTranslationFiles = ScriptUtils.readDirRecSync("langs").filter(path => path.endsWith(".json"))
const allTranslationFiles = ScriptUtils.readDirRecSync("langs").filter((path) =>
path.endsWith(".json")
)
for (const path of allTranslationFiles) {
formatFile(path)
}

32
scripts/generateWikiPage.ts
View file

@ -1,10 +1,14 @@
import {writeFile} from "fs";
import Translations from "../UI/i18n/Translations";
import { writeFile } from "fs"
import Translations from "../UI/i18n/Translations"
import * as themeOverview from "../assets/generated/theme_overview.json"
function generateWikiEntry(layout: { hideFromOverview: boolean, id: string, shortDescription: any }) {
function generateWikiEntry(layout: {
hideFromOverview: boolean
id: string
shortDescription: any
}) {
if (layout.hideFromOverview) {
return "";
return ""
}
const languagesInDescr = []
@ -12,8 +16,8 @@ function generateWikiEntry(layout: { hideFromOverview: boolean, id: string, shor
languagesInDescr.push(shortDescriptionKey)
}
const languages = languagesInDescr.map(ln => `{{#language:${ln}|en}}`).join(", ")
let auth = "Yes";
const languages = languagesInDescr.map((ln) => `{{#language:${ln}|en}}`).join(", ")
let auth = "Yes"
return `{{service_item
|name= [https://mapcomplete.osm.be/${layout.id} ${layout.id}]
|region= Worldwide
@ -21,29 +25,29 @@ function generateWikiEntry(layout: { hideFromOverview: boolean, id: string, shor
|descr= A MapComplete theme: ${Translations.T(layout.shortDescription)
.textFor("en")
.replace("<a href='", "[[")
.replace(/'>.*<\/a>/, "]]")
}
.replace(/'>.*<\/a>/, "]]")}
|material= {{yes|[https://mapcomplete.osm.be/ ${auth}]}}
|image= MapComplete_Screenshot.png
|genre= POI, editor, ${layout.id}
}}`
}
let wikiPage = "{|class=\"wikitable sortable\"\n" +
let wikiPage =
'{|class="wikitable sortable"\n' +
"! Name, link !! Genre !! Covered region !! Language !! Description !! Free materials !! Image\n" +
"|-";
"|-"
for (const layout of themeOverview) {
if (layout.hideFromOverview) {
continue;
continue
}
wikiPage += "\n" + generateWikiEntry(layout);
wikiPage += "\n" + generateWikiEntry(layout)
}
wikiPage += "\n|}"
writeFile("Docs/wikiIndex.txt", wikiPage, (err) => {
if (err !== null) {
console.log("Could not save wikiindex", err);
console.log("Could not save wikiindex", err)
}
});
})

41
scripts/lint.ts
View file

@ -1,23 +1,26 @@
import ScriptUtils from "./ScriptUtils";
import {writeFileSync} from "fs";
import {FixLegacyTheme, UpdateLegacyLayer} from "../Models/ThemeConfig/Conversion/LegacyJsonConvert";
import Translations from "../UI/i18n/Translations";
import {Translation} from "../UI/i18n/Translation";
import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
import ScriptUtils from "./ScriptUtils"
import { writeFileSync } from "fs"
import {
FixLegacyTheme,
UpdateLegacyLayer,
} from "../Models/ThemeConfig/Conversion/LegacyJsonConvert"
import Translations from "../UI/i18n/Translations"
import { Translation } from "../UI/i18n/Translation"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
/*
* This script reads all theme and layer files and reformats them inplace
* Use with caution, make a commit beforehand!
*/
const t : Translation = Translations.t.general.add.addNew
const t: Translation = Translations.t.general.add.addNew
t.OnEveryLanguage((txt, ln) => {
console.log(ln, txt)
return txt
})
const articles = {
/* de: "eine",
/* de: "eine",
es: 'una',
fr: 'une',
it: 'una',
@ -27,7 +30,7 @@ const articles = {
pt_BR : 'uma',//*/
}
function addArticleToPresets(layerConfig: {presets?: {title: any}[]}){
function addArticleToPresets(layerConfig: { presets?: { title: any }[] }) {
/*
if(layerConfig.presets === undefined){
return
@ -59,10 +62,15 @@ function addArticleToPresets(layerConfig: {presets?: {title: any}[]}){
//*/
}
const layerFiles = ScriptUtils.getLayerFiles();
const layerFiles = ScriptUtils.getLayerFiles()
for (const layerFile of layerFiles) {
try {
const fixed =<LayerConfigJson> new UpdateLegacyLayer().convertStrict(layerFile.parsed, "While linting " + layerFile.path);
const fixed = <LayerConfigJson>(
new UpdateLegacyLayer().convertStrict(
layerFile.parsed,
"While linting " + layerFile.path
)
)
addArticleToPresets(fixed)
writeFileSync(layerFile.path, JSON.stringify(fixed, null, " "))
} catch (e) {
@ -73,13 +81,16 @@ for (const layerFile of layerFiles) {
const themeFiles = ScriptUtils.getThemeFiles()
for (const themeFile of themeFiles) {
try {
const fixed = new FixLegacyTheme().convertStrict(themeFile.parsed, "While linting " + themeFile.path);
const fixed = new FixLegacyTheme().convertStrict(
themeFile.parsed,
"While linting " + themeFile.path
)
for (const layer of fixed.layers) {
if(layer["presets"] !== undefined){
addArticleToPresets(<any> layer)
if (layer["presets"] !== undefined) {
addArticleToPresets(<any>layer)
}
}
// extractInlineLayer(fixed)
// extractInlineLayer(fixed)
writeFileSync(themeFile.path, JSON.stringify(fixed, null, " "))
} catch (e) {
console.error("COULD NOT LINT THEME" + themeFile.path + ":\n\t" + e)

14
scripts/makeConvex.ts
View file

@ -1,6 +1,6 @@
import fs from "fs";
import {GeoOperations} from "../Logic/GeoOperations";
import ScriptUtils from "./ScriptUtils";
import fs from "fs"
import { GeoOperations } from "../Logic/GeoOperations"
import ScriptUtils from "./ScriptUtils"
/**
* Given one of more files, calculates a somewhat convex hull for them
@ -10,21 +10,19 @@ import ScriptUtils from "./ScriptUtils";
function makeConvex(file) {
ScriptUtils.erasableLog("Handling", file)
const geoJson = JSON.parse(fs.readFileSync(file, "UTF8"))
const convex = GeoOperations.convexHull(geoJson, {concavity: 2})
const convex = GeoOperations.convexHull(geoJson, { concavity: 2 })
if (convex.properties === undefined) {
convex.properties = {}
}
fs.writeFileSync(file + ".convex.geojson", JSON.stringify(convex))
}
let args = [...process.argv]
args.splice(0, 2)
args.forEach(file => {
args.forEach((file) => {
try {
makeConvex(file)
} catch (e) {
console.error("Could not handle file ", file, " due to ", e)
}
})
})

7
scripts/mergeJsonFiles.ts
View file

@ -1,8 +1,7 @@
import {readFileSync, writeFileSync} from "fs";
import {Utils} from "../Utils";
import { readFileSync, writeFileSync } from "fs"
import { Utils } from "../Utils"
function main(args: string[]) {
console.log("File Merge")
if (args.length != 3) {
@ -17,4 +16,4 @@ function main(args: string[]) {
writeFileSync(output, JSON.stringify(f2, null, " "))
}
main(process.argv.slice(2))
main(process.argv.slice(2))

186
scripts/onwheels/constants.ts
View file

@ -1,102 +1,102 @@
/**
* Class containing all constants and tables used in the script
*
*
* @class Constants
*/
export default class Constants {
/**
* Table used to determine tags for the category
*
* Keys are the original category names,
* values are an object containing the tags
*/
public static categories = {
restaurant: {
amenity: "restaurant",
},
parking: {
amenity: "parking",
},
hotel: {
tourism: "hotel",
},
wc: {
amenity: "toilets",
},
winkel: {
shop: "yes",
},
apotheek: {
amenity: "pharmacy",
healthcare: "pharmacy",
},
ziekenhuis: {
amenity: "hospital",
healthcare: "hospital",
},
bezienswaardigheid: {
tourism: "attraction",
},
ontspanning: {
fixme: "Needs proper tags",
},
cafe: {
amenity: "cafe",
},
dienst: {
fixme: "Needs proper tags",
},
bank: {
amenity: "bank",
},
gas: {
amenity: "fuel",
},
medical: {
fixme: "Needs proper tags",
},
obstacle: {
fixme: "Needs proper tags",
},
};
/**
* Table used to determine tags for the category
*
* Keys are the original category names,
* values are an object containing the tags
*/
public static categories = {
restaurant: {
amenity: "restaurant",
},
parking: {
amenity: "parking",
},
hotel: {
tourism: "hotel",
},
wc: {
amenity: "toilets",
},
winkel: {
shop: "yes",
},
apotheek: {
amenity: "pharmacy",
healthcare: "pharmacy",
},
ziekenhuis: {
amenity: "hospital",
healthcare: "hospital",
},
bezienswaardigheid: {
tourism: "attraction",
},
ontspanning: {
fixme: "Needs proper tags",
},
cafe: {
amenity: "cafe",
},
dienst: {
fixme: "Needs proper tags",
},
bank: {
amenity: "bank",
},
gas: {
amenity: "fuel",
},
medical: {
fixme: "Needs proper tags",
},
obstacle: {
fixme: "Needs proper tags",
},
}
/**
* Table used to rename original Onwheels properties to their corresponding OSM properties
*
* Keys are the original Onwheels properties, values are the corresponding OSM properties
*/
public static names = {
ID: "id",
Naam: "name",
Straat: "addr:street",
Nummer: "addr:housenumber",
Postcode: "addr:postcode",
Plaats: "addr:city",
Website: "website",
Email: "email",
"Aantal aangepaste parkeerplaatsen": "capacity:disabled",
"Aantal treden": "step_count",
"Hellend vlak aanwezig": "ramp",
"Baby verzorging aanwezig": "changing_table",
"Totale hoogte van de treden": "kerb:height",
"Deurbreedte": "door:width",
};
/**
* Table used to rename original Onwheels properties to their corresponding OSM properties
*
* Keys are the original Onwheels properties, values are the corresponding OSM properties
*/
public static names = {
ID: "id",
Naam: "name",
Straat: "addr:street",
Nummer: "addr:housenumber",
Postcode: "addr:postcode",
Plaats: "addr:city",
Website: "website",
Email: "email",
"Aantal aangepaste parkeerplaatsen": "capacity:disabled",
"Aantal treden": "step_count",
"Hellend vlak aanwezig": "ramp",
"Baby verzorging aanwezig": "changing_table",
"Totale hoogte van de treden": "kerb:height",
Deurbreedte: "door:width",
}
/**
* In some cases types might need to be converted as well
*
* Keys are the OSM properties, values are the wanted type
*/
public static types = {
"Hellend vlak aanwezig": "boolean",
"Baby verzorging aanwezig": "boolean",
};
/**
* In some cases types might need to be converted as well
*
* Keys are the OSM properties, values are the wanted type
*/
public static types = {
"Hellend vlak aanwezig": "boolean",
"Baby verzorging aanwezig": "boolean",
}
/**
* Some tags also need to have units added
*/
public static units = {
"Totale hoogte van de treden": "cm",
"Deurbreedte": "cm",
};
/**
* Some tags also need to have units added
*/
public static units = {
"Totale hoogte van de treden": "cm",
Deurbreedte: "cm",
}
}

305
scripts/onwheels/convertData.ts
View file

@ -1,7 +1,7 @@
import { parse } from "csv-parse/sync";
import { readFileSync, writeFileSync } from "fs";
import { Feature, FeatureCollection, GeoJsonProperties } from "geojson";
import Constants from "./constants";
import { parse } from "csv-parse/sync"
import { readFileSync, writeFileSync } from "fs"
import { Feature, FeatureCollection, GeoJsonProperties } from "geojson"
import Constants from "./constants"
/**
* Function to determine the tags for a category
@ -10,15 +10,15 @@ import Constants from "./constants";
* @returns List of tags for the category
*/
function categoryTags(category: string): GeoJsonProperties {
const tags = {
tags: Object.keys(Constants.categories[category]).map((tag) => {
return `${tag}=${Constants.categories[category][tag]}`;
}),
};
if (!tags) {
throw `Unknown category: ${category}`;
}
return tags;
const tags = {
tags: Object.keys(Constants.categories[category]).map((tag) => {
return `${tag}=${Constants.categories[category][tag]}`
}),
}
if (!tags) {
throw `Unknown category: ${category}`
}
return tags
}
/**
@ -28,70 +28,68 @@ function categoryTags(category: string): GeoJsonProperties {
* @returns GeoJsonProperties for the item
*/
function renameTags(item): GeoJsonProperties {
const properties: GeoJsonProperties = {};
properties.tags = [];
// Loop through the original item tags
for (const key in item) {
// Check if we need it and it's not a null value
if (Constants.names[key] && item[key]) {
// Name and id tags need to be in the properties
if (Constants.names[key] == "name" || Constants.names[key] == "id") {
properties[Constants.names[key]] = item[key];
}
// Other tags need to be in the tags variable
if (Constants.names[key] !== "id") {
// Website needs to have at least any = encoded
if(Constants.names[key] == "website") {
let website = item[key];
// Encode URL
website = website.replace("=", "%3D");
item[key] = website;
const properties: GeoJsonProperties = {}
properties.tags = []
// Loop through the original item tags
for (const key in item) {
// Check if we need it and it's not a null value
if (Constants.names[key] && item[key]) {
// Name and id tags need to be in the properties
if (Constants.names[key] == "name" || Constants.names[key] == "id") {
properties[Constants.names[key]] = item[key]
}
// Other tags need to be in the tags variable
if (Constants.names[key] !== "id") {
// Website needs to have at least any = encoded
if (Constants.names[key] == "website") {
let website = item[key]
// Encode URL
website = website.replace("=", "%3D")
item[key] = website
}
properties.tags.push(Constants.names[key] + "=" + item[key])
}
}
properties.tags.push(Constants.names[key] + "=" + item[key]);
}
}
}
return properties;
return properties
}
/**
* Convert types to match the OSM standard
*
*
* @param properties The properties to convert
* @returns The converted properties
*/
function convertTypes(properties: GeoJsonProperties): GeoJsonProperties {
// Split the tags into a list
let tags = properties.tags.split(";");
// Split the tags into a list
let tags = properties.tags.split(";")
for (const tag in tags) {
// Split the tag into key and value
const key = tags[tag].split("=")[0];
const value = tags[tag].split("=")[1];
const originalKey = Object.keys(Constants.names).find(
(tag) => Constants.names[tag] === key
);
for (const tag in tags) {
// Split the tag into key and value
const key = tags[tag].split("=")[0]
const value = tags[tag].split("=")[1]
const originalKey = Object.keys(Constants.names).find((tag) => Constants.names[tag] === key)
if (Constants.types[originalKey]) {
// We need to convert the value to the correct type
let newValue;
switch (Constants.types[originalKey]) {
case "boolean":
newValue = value === "1" ? "yes" : "no";
break;
default:
newValue = value;
break;
}
tags[tag] = `${key}=${newValue}`;
if (Constants.types[originalKey]) {
// We need to convert the value to the correct type
let newValue
switch (Constants.types[originalKey]) {
case "boolean":
newValue = value === "1" ? "yes" : "no"
break
default:
newValue = value
break
}
tags[tag] = `${key}=${newValue}`
}
}
}
// Rejoin the tags
properties.tags = tags.join(";");
// Rejoin the tags
properties.tags = tags.join(";")
// Return the properties
return properties;
// Return the properties
return properties
}
/**
@ -101,27 +99,25 @@ function convertTypes(properties: GeoJsonProperties): GeoJsonProperties {
* @returns The properties with units added
*/
function addUnits(properties: GeoJsonProperties): GeoJsonProperties {
// Split the tags into a list
let tags = properties.tags.split(";");
// Split the tags into a list
let tags = properties.tags.split(";")
for (const tag in tags) {
const key = tags[tag].split("=")[0];
const value = tags[tag].split("=")[1];
const originalKey = Object.keys(Constants.names).find(
(tag) => Constants.names[tag] === key
);
for (const tag in tags) {
const key = tags[tag].split("=")[0]
const value = tags[tag].split("=")[1]
const originalKey = Object.keys(Constants.names).find((tag) => Constants.names[tag] === key)
// Check if the property needs units, and doesn't already have them
if (Constants.units[originalKey] && value.match(/.*([A-z]).*/gi) === null) {
tags[tag] = `${key}=${value} ${Constants.units[originalKey]}`;
// Check if the property needs units, and doesn't already have them
if (Constants.units[originalKey] && value.match(/.*([A-z]).*/gi) === null) {
tags[tag] = `${key}=${value} ${Constants.units[originalKey]}`
}
}
}
// Rejoin the tags
properties.tags = tags.join(";");
// Rejoin the tags
properties.tags = tags.join(";")
// Return the properties
return properties;
// Return the properties
return properties
}
/**
@ -131,15 +127,15 @@ function addUnits(properties: GeoJsonProperties): GeoJsonProperties {
* @param item The original CSV item
*/
function addMaprouletteTags(properties: GeoJsonProperties, item: any): GeoJsonProperties {
properties[
"blurb"
] = `This is feature out of the ${item["Categorie"]} category.
properties["blurb"] = `This is feature out of the ${item["Categorie"]} category.
It may match another OSM item, if so, you can add any missing tags to it.
If it doesn't match any other OSM item, you can create a new one.
Here is a list of tags that can be added:
${properties["tags"].split(";").join("\n")}
You can also easily import this item using MapComplete: https://mapcomplete.osm.be/onwheels.html#${properties["id"]}`;
return properties;
You can also easily import this item using MapComplete: https://mapcomplete.osm.be/onwheels.html#${
properties["id"]
}`
return properties
}
/**
@ -148,87 +144,84 @@ function addMaprouletteTags(properties: GeoJsonProperties, item: any): GeoJsonPr
* @param args List of arguments [input.csv]
*/
function main(args: string[]): void {
const csvOptions = {
columns: true,
skip_empty_lines: true,
trim: true,
};
const file = args[0];
const output = args[1];
const csvOptions = {
columns: true,
skip_empty_lines: true,
trim: true,
}
const file = args[0]
const output = args[1]
// Create an empty list to store the converted features
var items: Feature[] = [];
// Create an empty list to store the converted features
var items: Feature[] = []
// Read CSV file
const csv: Record<any, string>[] = parse(readFileSync(file), csvOptions);
// Read CSV file
const csv: Record<any, string>[] = parse(readFileSync(file), csvOptions)
// Loop through all the entries
for (var i = 0; i < csv.length; i++) {
const item = csv[i];
// Loop through all the entries
for (var i = 0; i < csv.length; i++) {
const item = csv[i]
// Determine coordinates
const lat = Number(item["Latitude"]);
const lon = Number(item["Longitude"]);
// Determine coordinates
const lat = Number(item["Latitude"])
const lon = Number(item["Longitude"])
// Check if coordinates are valid
if (isNaN(lat) || isNaN(lon)) {
throw `Not a valid lat or lon for entry ${i}: ${JSON.stringify(item)}`;
// Check if coordinates are valid
if (isNaN(lat) || isNaN(lon)) {
throw `Not a valid lat or lon for entry ${i}: ${JSON.stringify(item)}`
}
// Create a new collection to store the converted properties
var properties: GeoJsonProperties = {}
// Add standard tags for category
const category = item["Categorie"]
const tagsCategory = categoryTags(category)
// Add the rest of the needed tags
properties = { ...properties, ...renameTags(item) }
// Merge them together
properties.tags = [...tagsCategory.tags, ...properties.tags]
properties.tags = properties.tags.join(";")
// Convert types
properties = convertTypes(properties)
// Add units if necessary
properties = addUnits(properties)
// Add Maproulette tags
properties = addMaprouletteTags(properties, item)
// Create the new feature
const feature: Feature = {
type: "Feature",
id: item["ID"],
geometry: {
type: "Point",
coordinates: [lon, lat],
},
properties,
}
// Push it to the list we created earlier
items.push(feature)
}
// Create a new collection to store the converted properties
var properties: GeoJsonProperties = {};
// Make a FeatureCollection out of it
const featureCollection: FeatureCollection = {
type: "FeatureCollection",
features: items,
}
// Add standard tags for category
const category = item["Categorie"];
const tagsCategory = categoryTags(category);
// Add the rest of the needed tags
properties = { ...properties, ...renameTags(item) };
// Merge them together
properties.tags = [...tagsCategory.tags, ...properties.tags];
properties.tags = properties.tags.join(";");
// Convert types
properties = convertTypes(properties);
// Add units if necessary
properties = addUnits(properties);
// Add Maproulette tags
properties = addMaprouletteTags(properties, item);
// Create the new feature
const feature: Feature = {
type: "Feature",
id: item["ID"],
geometry: {
type: "Point",
coordinates: [lon, lat],
},
properties,
};
// Push it to the list we created earlier
items.push(feature);
}
// Make a FeatureCollection out of it
const featureCollection: FeatureCollection = {
type: "FeatureCollection",
features: items,
};
// Write the data to a file or output to the console
if (output) {
writeFileSync(
`${output}.geojson`,
JSON.stringify(featureCollection, null, 2)
);
} else {
console.log(JSON.stringify(featureCollection));
}
// Write the data to a file or output to the console
if (output) {
writeFileSync(`${output}.geojson`, JSON.stringify(featureCollection, null, 2))
} else {
console.log(JSON.stringify(featureCollection))
}
}
// Execute the main function, with the stripped arguments
main(process.argv.slice(2));
main(process.argv.slice(2))

17
scripts/perProperty.ts
View file

@ -1,8 +1,10 @@
import * as fs from "fs";
import * as fs from "fs"
function main(args) {
if (args.length < 2) {
console.log("Given a single geojson file and an attribute-key, will generate a new file for every value of the partition.")
console.log(
"Given a single geojson file and an attribute-key, will generate a new file for every value of the partition."
)
console.log("USAGE: perProperty `file.geojson` `property-key`")
return
}
@ -24,15 +26,14 @@ function main(args) {
const stripped = path.substr(0, path.length - ".geojson".length)
perProperty.forEach((features, v) => {
fs.writeFileSync(stripped + "." + v.replace(/[^a-zA-Z0-9_]/g, "_") + ".geojson",
fs.writeFileSync(
stripped + "." + v.replace(/[^a-zA-Z0-9_]/g, "_") + ".geojson",
JSON.stringify({
type: "FeatureCollection",
features
}))
features,
})
)
})
}
main(process.argv.slice(2))

85
scripts/postal_code_tools/createRoutablePoint.ts
View file

@ -1,11 +1,10 @@
import {appendFileSync, existsSync, readFileSync, writeFileSync} from "fs";
import {GeoOperations} from "../../Logic/GeoOperations";
import ScriptUtils from "../ScriptUtils";
import {Utils} from "../../Utils";
import { appendFileSync, existsSync, readFileSync, writeFileSync } from "fs"
import { GeoOperations } from "../../Logic/GeoOperations"
import ScriptUtils from "../ScriptUtils"
import { Utils } from "../../Utils"
async function main(args: string[]) {
ScriptUtils.fixUtils()
ScriptUtils.fixUtils()
const pointCandidates = JSON.parse(readFileSync(args[0], "utf8"))
const postcodes = JSON.parse(readFileSync(args[1], "utf8"))
const output = args[2] ?? "centralCoordinates.csv"
@ -16,7 +15,7 @@ ScriptUtils.fixUtils()
if (existsSync(output)) {
const lines = readFileSync(output, "UTF8").split("\n")
lines.shift()
lines.forEach(line => {
lines.forEach((line) => {
const postalCode = Number(line.split(",")[0])
alreadyLoaded.add(postalCode)
})
@ -35,7 +34,6 @@ ScriptUtils.fixUtils()
} else {
perPostCode.set(postcode, [boundary])
}
}
for (const postcode of Array.from(perPostCode.keys())) {
@ -48,45 +46,68 @@ ScriptUtils.fixUtils()
continue
}
candidates.push(candidate.geometry.coordinates)
}
}
if (candidates.length === 0) {
console.log("Postcode ", postcode, "has", candidates.length, "candidates, using centerpoint instead")
candidates.push(...boundaries.map(boundary => GeoOperations.centerpointCoordinates(boundary)))
console.log(
"Postcode ",
postcode,
"has",
candidates.length,
"candidates, using centerpoint instead"
)
candidates.push(
...boundaries.map((boundary) => GeoOperations.centerpointCoordinates(boundary))
)
}
const url =
"https://staging.anyways.eu/routing-api/v1/routes?access_token=postal_code_script&turn_by_turn=false&format=geojson&language=en"
const depPoints: [number, number][] = Utils.NoNull(
await Promise.all(
candidates.map(async (candidate) => {
try {
const result = await Utils.downloadJson(
url +
"&loc=" +
candidate.join("%2C") +
"&loc=3.22000%2C51.21577&profile=car.short"
)
const depPoint = result.features.filter(
(f) => f.geometry.type === "LineString"
)[0].geometry.coordinates[0]
return <[number, number]>[depPoint[0], depPoint[1]] // Drop elevation
} catch (e) {
console.error("No result or could not calculate a route")
}
})
)
)
const url = "https://staging.anyways.eu/routing-api/v1/routes?access_token=postal_code_script&turn_by_turn=false&format=geojson&language=en"
const depPoints: [number, number][] = Utils.NoNull(await Promise.all(candidates.map(async candidate => {
try {
const result = await Utils.downloadJson(url + "&loc=" + candidate.join("%2C") + "&loc=3.22000%2C51.21577&profile=car.short")
const depPoint = result.features.filter(f => f.geometry.type === "LineString")[0].geometry.coordinates[0]
return <[number, number]>[depPoint[0], depPoint[1]] // Drop elevation
} catch (e) {
console.error("No result or could not calculate a route")
}
})))
const centers = boundaries.map(b => GeoOperations.centerpointCoordinates(b))
const centers = boundaries.map((b) => GeoOperations.centerpointCoordinates(b))
const center = GeoOperations.centerpointCoordinates({
type: "Feature",
geometry: {
type: "LineString",
coordinates: centers
}
coordinates: centers,
},
})
depPoints.sort((c0, c1) => GeoOperations.distanceBetween(c0, center) - GeoOperations.distanceBetween(c1, center))
console.log("Sorted departure point candidates for ", postcode, " are ", JSON.stringify(depPoints))
depPoints.sort(
(c0, c1) =>
GeoOperations.distanceBetween(c0, center) -
GeoOperations.distanceBetween(c1, center)
)
console.log(
"Sorted departure point candidates for ",
postcode,
" are ",
JSON.stringify(depPoints)
)
appendFileSync(output, [postcode, ...depPoints[0]].join(", ") + "\n", "UTF-8")
}
}
let args = [...process.argv]
args.splice(0, 2)
main(args).then(_ => console.log("Done!"))
main(args).then((_) => console.log("Done!"))

111
scripts/postal_code_tools/openaddressestogeojson.ts
View file

@ -1,49 +1,70 @@
import * as fs from "fs";
import {existsSync, writeFileSync} from "fs";
import * as readline from "readline";
import ScriptUtils from "../ScriptUtils";
import * as fs from "fs"
import { existsSync, writeFileSync } from "fs"
import * as readline from "readline"
import ScriptUtils from "../ScriptUtils"
/**
* Converts an open-address CSV file into a big geojson file
*/
async function main(args: string[]) {
const inputFile = args[0]
const outputFile = args[1]
const fileStream = fs.createReadStream(inputFile);
const fileStream = fs.createReadStream(inputFile)
const perPostalCode = args[2] == "--per-postal-code"
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
crlfDelay: Infinity,
})
// Note: we use the crlfDelay option to recognize all instances of CR LF
// ('\r\n') in input.txt as a single line break.
const fields = [
"EPSG:31370_x", "EPSG:31370_y", "EPSG:4326_lat", "EPSG:4326_lon",
"address_id", "box_number",
"house_number", "municipality_id", "municipality_name_de", "municipality_name_fr", "municipality_name_nl", "postcode", "postname_fr",
"postname_nl", "street_id", "streetname_de", "streetname_fr", "streetname_nl", "region_code", "status"
"EPSG:31370_x",
"EPSG:31370_y",
"EPSG:4326_lat",
"EPSG:4326_lon",
"address_id",
"box_number",
"house_number",
"municipality_id",
"municipality_name_de",
"municipality_name_fr",
"municipality_name_nl",
"postcode",
"postname_fr",
"postname_nl",
"street_id",
"streetname_de",
"streetname_fr",
"streetname_nl",
"region_code",
"status",
]
let i = 0;
let i = 0
let failed = 0
let createdFiles: string [] = []
let createdFiles: string[] = []
if (!perPostalCode) {
fs.writeFileSync(outputFile, "")
}
// @ts-ignore
for await (const line of rl) {
i++;
i++
if (i % 10000 == 0) {
ScriptUtils.erasableLog("Converted ", i, "features (of which ", failed, "features don't have a coordinate)")
ScriptUtils.erasableLog(
"Converted ",
i,
"features (of which ",
failed,
"features don't have a coordinate)"
)
}
const data = line.split(",")
const parsed: any = {}
for (let i = 0; i < fields.length; i++) {
const field = fields[i];
const field = fields[i]
parsed[field] = data[i]
}
const lat = Number(parsed["EPSG:4326_lat"])
@ -67,7 +88,7 @@ async function main(args: string[]) {
continue
}
if (isNaN(Number(parsed["postcode"]))) {
continue;
continue
}
targetFile = outputFile + "-" + parsed["postcode"] + ".geojson"
let isFirst = false
@ -81,29 +102,30 @@ async function main(args: string[]) {
fs.appendFileSync(targetFile, ",\n")
}
fs.appendFileSync(targetFile, JSON.stringify({
type: "Feature",
properties: parsed,
geometry: {
type: "Point",
coordinates: [lon, lat]
}
}))
fs.appendFileSync(
targetFile,
JSON.stringify({
type: "Feature",
properties: parsed,
geometry: {
type: "Point",
coordinates: [lon, lat],
},
})
)
} else {
fs.appendFileSync(outputFile, JSON.stringify({
type: "Feature",
properties: parsed,
geometry: {
type: "Point",
coordinates: [lon, lat]
}
}) + "\n")
fs.appendFileSync(
outputFile,
JSON.stringify({
type: "Feature",
properties: parsed,
geometry: {
type: "Point",
coordinates: [lon, lat],
},
}) + "\n"
)
}
}
console.log("Closing files...")
@ -113,8 +135,13 @@ async function main(args: string[]) {
fs.appendFileSync(createdFile, "]}")
}
console.log("Done! Converted ", i, "features (of which ", failed, "features don't have a coordinate)")
console.log(
"Done! Converted ",
i,
"features (of which ",
failed,
"features don't have a coordinate)"
)
}
let args = [...process.argv]
@ -123,5 +150,5 @@ args.splice(0, 2)
if (args.length == 0) {
console.log("USAGE: input-csv-file output.newline-delimited-geojson.json [--per-postal-code]")
} else {
main(args).catch(e => console.error(e))
main(args).catch((e) => console.error(e))
}

45
scripts/postal_code_tools/prepPostalCodesHulls.ts
View file

@ -1,22 +1,22 @@
import * as fs from "fs";
import {writeFileSync} from "fs";
import ScriptUtils from "../ScriptUtils";
import * as fs from "fs"
import { writeFileSync } from "fs"
import ScriptUtils from "../ScriptUtils"
function handleFile(file: string, postalCode: number) {
const geojson = JSON.parse(fs.readFileSync(file, "UTF8"))
geojson.properties = {
type: "boundary",
"boundary": "postal_code",
"postal_code": postalCode + ""
boundary: "postal_code",
postal_code: postalCode + "",
}
return geojson
}
function getKnownPostalCodes(): number[] {
return fs.readFileSync("./scripts/postal_code_tools/knownPostalCodes.csv", "UTF8").split("\n")
.map(line => Number(line.split(",")[1]))
return fs
.readFileSync("./scripts/postal_code_tools/knownPostalCodes.csv", "UTF8")
.split("\n")
.map((line) => Number(line.split(",")[1]))
}
function main(args: string[]) {
@ -28,27 +28,36 @@ function main(args: string[]) {
for (const file of files) {
const nameParts = file.split("-")
const postalCodeStr = nameParts[nameParts.length - 1]
const postalCode = Number(postalCodeStr.substr(0, postalCodeStr.length - ".geojson.convex.geojson".length))
const postalCode = Number(
postalCodeStr.substr(0, postalCodeStr.length - ".geojson.convex.geojson".length)
)
if (isNaN(postalCode)) {
console.error("Not a number: ", postalCodeStr)
continue
}
if (knownPostals.has(postalCode)) {
skipped.push(postalCode)
ScriptUtils.erasableLog("Skipping boundary for ", postalCode, "as it is already known - skipped ", skipped.length, "already")
ScriptUtils.erasableLog(
"Skipping boundary for ",
postalCode,
"as it is already known - skipped ",
skipped.length,
"already"
)
continue
}
allFiles.push(handleFile(file, postalCode))
}
writeFileSync("all_postal_codes_filtered.geojson", JSON.stringify({
type: "FeatureCollection",
features: allFiles
}))
writeFileSync(
"all_postal_codes_filtered.geojson",
JSON.stringify({
type: "FeatureCollection",
features: allFiles,
})
)
}
let args = [...process.argv]
args.splice(0, 2)
main(args)
main(args)

4
scripts/printVersion.ts
View file

@ -1,3 +1,3 @@
import Constants from "../Models/Constants";
import Constants from "../Models/Constants"
console.log("git tag -a", Constants.vNumber, `-m "Deployed on ${new Date()}"`)
console.log("git tag -a", Constants.vNumber, `-m "Deployed on ${new Date()}"`)

297
scripts/schools/amendSchoolData.ts
View file

@ -1,24 +1,23 @@
import {parse} from 'csv-parse/sync';
import {readFileSync, writeFileSync} from "fs";
import {Utils} from "../../Utils";
import {GeoJSONObject, geometry} from "@turf/turf";
import { parse } from "csv-parse/sync"
import { readFileSync, writeFileSync } from "fs"
import { Utils } from "../../Utils"
import { GeoJSONObject, geometry } from "@turf/turf"
function parseAndClean(filename: string): Record<any, string>[] {
const csvOptions = {
columns: true,
skip_empty_lines: true,
trim: true
trim: true,
}
const records: Record<any, string>[] = parse(readFileSync(filename), csvOptions)
return records.map(r => {
return records.map((r) => {
for (const key of Object.keys(r)) {
if (r[key].endsWith("niet van toepassing")) {
delete r[key]
}
}
return r;
return r
})
}
@ -26,52 +25,82 @@ const structuren = {
"Voltijds Gewoon Secundair Onderwijs": "secondary",
"Gewoon Lager Onderwijs": "primary",
"Gewoon Kleuteronderwijs": "kindergarten",
"Kleuteronderwijs": "kindergarten",
Kleuteronderwijs: "kindergarten",
"Buitengewoon Lager Onderwijs": "primary",
"Buitengewoon Secundair Onderwijs": "secondary",
"Buitengewoon Kleuteronderwijs": "kindergarten",
"Deeltijds Beroepssecundair Onderwijs": "secondary"
"Deeltijds Beroepssecundair Onderwijs": "secondary",
}
const degreesMapping = {
"Derde graad":"upper_secondary",
"Tweede graad":"middle_secondary",
"Eerste graad" :"lower_secondary"
"Derde graad": "upper_secondary",
"Tweede graad": "middle_secondary",
"Eerste graad": "lower_secondary",
}
const classificationOrder = ["kindergarten","primary","secondary","lower_secondary","middle_secondary","upper_secondary"]
const classificationOrder = [
"kindergarten",
"primary",
"secondary",
"lower_secondary",
"middle_secondary",
"upper_secondary",
]
const stelselsMapping = {
"Beide stelsels":"linear_courses;modular_courses",
"Lineair stelsel":"linear_courses",
"Modulair stelsel" :"modular_courses"
"Beide stelsels": "linear_courses;modular_courses",
"Lineair stelsel": "linear_courses",
"Modulair stelsel": "modular_courses",
}
const rmKeys = ["schoolnummer", "instellingstype",
"adres", "begindatum","hoofdzetel","huisnummer","kbo-nummer",
"beheerder(s)", "bestuur", "clb", "ingerichte hoofdstructuren", "busnummer", "crab-code", "crab-huisnr",
"einddatum", "fax", "gemeente", "intern_vplnummer", "kbo_nummer", "lx", "ly", "niscode",
"onderwijsniveau","onderwijsvorm","scholengemeenschap",
"postcode", "provincie",
"provinciecode", "soort instelling", "status erkenning", "straat", "VWO-vestigingsplaatscode", "taalstelsel",
"net"]
const rmKeys = [
"schoolnummer",
"instellingstype",
"adres",
"begindatum",
"hoofdzetel",
"huisnummer",
"kbo-nummer",
"beheerder(s)",
"bestuur",
"clb",
"ingerichte hoofdstructuren",
"busnummer",
"crab-code",
"crab-huisnr",
"einddatum",
"fax",
"gemeente",
"intern_vplnummer",
"kbo_nummer",
"lx",
"ly",
"niscode",
"onderwijsniveau",
"onderwijsvorm",
"scholengemeenschap",
"postcode",
"provincie",
"provinciecode",
"soort instelling",
"status erkenning",
"straat",
"VWO-vestigingsplaatscode",
"taalstelsel",
"net",
]
const rename = {
"e-mail":"email",
"naam":"name",
"telefoon":"phone"
"e-mail": "email",
naam: "name",
telefoon: "phone",
}
function fuzzIdenticals(features: {geometry: {coordinates: [number,number]}}[]){
function fuzzIdenticals(features: { geometry: { coordinates: [number, number] } }[]) {
var seen = new Set<string>()
for (const feature of features) {
var coors = feature.geometry.coordinates;
var coors = feature.geometry.coordinates
let k = coors[0] + "," + coors[1]
while(seen.has(k)){
while (seen.has(k)) {
coors[0] += 0.00025
k = coors[0] + "," + coors[1]
}
@ -81,60 +110,158 @@ function fuzzIdenticals(features: {geometry: {coordinates: [number,number]}}[]){
/**
* Sorts classifications in order
*
*
* sortClassifications(["primary","secondary","kindergarten"] // => ["kindergarten", "primary", "secondary"]
*/
function sortClassifications(classification: string[]){
return classification.sort((a, b) => classificationOrder.indexOf(a) - classificationOrder.indexOf(b))
function sortClassifications(classification: string[]) {
return classification.sort(
(a, b) => classificationOrder.indexOf(a) - classificationOrder.indexOf(b)
)
}
function main() {
console.log("Parsing schools...")
const aantallen = "/home/pietervdvn/Downloads/Scholen/aantallen.csv"
const perSchool = "/home/pietervdvn/Downloads/Scholen/perschool.csv"
const schoolfields = ["schoolnummer", "intern_vplnummer", "net", "naam", "hoofdzetel", "adres", "straat", "huisnummer", "busnummer", "postcode", "gemeente", "niscode", "provinciecode", "provincie", "VWO-vestigingsplaatscode", "crab-code", "crab-huisnr", "lx", "ly", "kbo-nummer", "telefoon", "fax", "e-mail", "website", "beheerder(s)", "soort instelling", "onderwijsniveau", "instellingstype", "begindatum", "einddatum", "status erkenning", "clb", "bestuur", "scholengemeenschap", "taalstelsel", "ingerichte hoofdstructuren"] as const
const schoolfields = [
"schoolnummer",
"intern_vplnummer",
"net",
"naam",
"hoofdzetel",
"adres",
"straat",
"huisnummer",
"busnummer",
"postcode",
"gemeente",
"niscode",
"provinciecode",
"provincie",
"VWO-vestigingsplaatscode",
"crab-code",
"crab-huisnr",
"lx",
"ly",
"kbo-nummer",
"telefoon",
"fax",
"e-mail",
"website",
"beheerder(s)",
"soort instelling",
"onderwijsniveau",
"instellingstype",
"begindatum",
"einddatum",
"status erkenning",
"clb",
"bestuur",
"scholengemeenschap",
"taalstelsel",
"ingerichte hoofdstructuren",
] as const
const schoolGeojson: {
features: {
properties: Record<(typeof schoolfields)[number], string>,
geometry:{
type: "Point",
coordinates: [number,number]
properties: Record<typeof schoolfields[number], string>
geometry: {
type: "Point"
coordinates: [number, number]
}
}[]
} = JSON.parse(readFileSync("scripts/schools/scholen.geojson", "utf8"))
fuzzIdenticals(schoolGeojson.features)
const aantallenFields = ["schooljaar", "nr koepel", "koepel", "instellingscode", "intern volgnr vpl", "volgnr vpl", "naam instelling", "GON-school", "GOK-school", "instellingsnummer scholengemeenschap", "scholengemeenschap", "code schoolbestuur", "schoolbestuur", "type vestigingsplaats", "fusiegemeente hoofdvestigingsplaats", "straatnaam vestigingsplaats", "huisnr vestigingsplaats", "bus vestigingsplaats", "postcode vestigingsplaats", "deelgemeente vestigingsplaats", "fusiegemeente vestigingsplaats", "hoofdstructuur (code)", "hoofdstructuur", "administratieve groep (code)", "administratieve groep", "graad lager onderwijs", "pedagogische methode", "graad secundair onderwijs", "leerjaar", "A of B-stroom", "basisopties", "beroepenveld", "onderwijsvorm", "studiegebied", "studierichting", "stelsel", "okan cluster", "type buitengewoon onderwijs", "opleidingsvorm (code)", "opleidingsvorm", "fase", "opleidingen", "geslacht", "aantal inschrijvingen"] as const
const aantallenParsed: Record<(typeof aantallenFields)[number], string>[] = parseAndClean(aantallen)
const perschoolFields = ["schooljaar", "nr koepel", "koepel", "instellingscode", "naam instelling", "straatnaam", "huisnr", "bus", "postcode", "deelgemeente", "fusiegemeente", "aantal inschrijvingen"] as const
const perschoolParsed: Record<(typeof perschoolFields)[number], string>[] = parseAndClean(perSchool)
const aantallenFields = [
"schooljaar",
"nr koepel",
"koepel",
"instellingscode",
"intern volgnr vpl",
"volgnr vpl",
"naam instelling",
"GON-school",
"GOK-school",
"instellingsnummer scholengemeenschap",
"scholengemeenschap",
"code schoolbestuur",
"schoolbestuur",
"type vestigingsplaats",
"fusiegemeente hoofdvestigingsplaats",
"straatnaam vestigingsplaats",
"huisnr vestigingsplaats",
"bus vestigingsplaats",
"postcode vestigingsplaats",
"deelgemeente vestigingsplaats",
"fusiegemeente vestigingsplaats",
"hoofdstructuur (code)",
"hoofdstructuur",
"administratieve groep (code)",
"administratieve groep",
"graad lager onderwijs",
"pedagogische methode",
"graad secundair onderwijs",
"leerjaar",
"A of B-stroom",
"basisopties",
"beroepenveld",
"onderwijsvorm",
"studiegebied",
"studierichting",
"stelsel",
"okan cluster",
"type buitengewoon onderwijs",
"opleidingsvorm (code)",
"opleidingsvorm",
"fase",
"opleidingen",
"geslacht",
"aantal inschrijvingen",
] as const
const aantallenParsed: Record<typeof aantallenFields[number], string>[] =
parseAndClean(aantallen)
const perschoolFields = [
"schooljaar",
"nr koepel",
"koepel",
"instellingscode",
"naam instelling",
"straatnaam",
"huisnr",
"bus",
"postcode",
"deelgemeente",
"fusiegemeente",
"aantal inschrijvingen",
] as const
const perschoolParsed: Record<typeof perschoolFields[number], string>[] =
parseAndClean(perSchool)
schoolGeojson.features = schoolGeojson.features
.filter(sch => sch.properties.lx != "0" && sch.properties.ly != "0")
.filter(sch => sch.properties.instellingstype !== "Universiteit")
.filter((sch) => sch.properties.lx != "0" && sch.properties.ly != "0")
.filter((sch) => sch.properties.instellingstype !== "Universiteit")
const c = schoolGeojson.features.length
console.log("Got ", schoolGeojson.features.length, "items after filtering")
let i = 0
let lastWrite = 0;
let lastWrite = 0
for (const feature of schoolGeojson.features) {
i++
const now = Date.now();
if(now - lastWrite > 1000){
lastWrite = now;
console.log("Processing "+i+"/"+c)
const now = Date.now()
if (now - lastWrite > 1000) {
lastWrite = now
console.log("Processing " + i + "/" + c)
}
const props = feature.properties
const aantallen = aantallenParsed.filter(i => i.instellingscode == props.schoolnummer)
const aantallen = aantallenParsed.filter((i) => i.instellingscode == props.schoolnummer)
if (aantallen.length > 0) {
const fetch = (key: (typeof aantallenFields)[number]) => Utils.NoNull(Utils.Dedup(aantallen.map(x => x[key])))
const fetch = (key: typeof aantallenFields[number]) =>
Utils.NoNull(Utils.Dedup(aantallen.map((x) => x[key])))
props["onderwijsvorm"] = fetch("onderwijsvorm").join(";")
@ -148,14 +275,12 @@ function main() {
*/
const hoofdstructuur = fetch("hoofdstructuur")
let specialEducation = false
let classification = hoofdstructuur.map(s => {
let classification = hoofdstructuur.map((s) => {
const v = structuren[s]
if (s.startsWith("Buitengewoon")) {
specialEducation = true;
specialEducation = true
}
if (v === undefined) {
console.error("Type not found: " + s)
@ -164,45 +289,46 @@ function main() {
return v
})
const graden = fetch("graad secundair onderwijs")
if(classification[0] === "secondary"){
if(graden.length !== 3){
classification = graden.map(degree => degreesMapping[degree])
if (classification[0] === "secondary") {
if (graden.length !== 3) {
classification = graden.map((degree) => degreesMapping[degree])
}
}
sortClassifications(classification)
props["school"] = Utils.Dedup(classification).join("; ")
// props["koepel"] = koepel.join(";")
// props["scholengemeenschap"] = scholengemeenschap.join(";")
// props["stelsel"] = stelselsMapping[stelsel]
if (specialEducation) {
props["school:for"] = "special_education"
}
if (props.taalstelsel === "Nederlandstalig") {
props["language:nl"] = "yes"
}
if(props.instellingstype === "Instelling voor deeltijds kunstonderwijs") {
props["amenity"] = "college"
if (props.instellingstype === "Instelling voor deeltijds kunstonderwijs") {
props["amenity"] = "college"
props["school:subject"] = "art"
}
}
const schoolinfo = perschoolParsed.filter(i => i.instellingscode == props.schoolnummer)
const schoolinfo = perschoolParsed.filter((i) => i.instellingscode == props.schoolnummer)
if (schoolinfo.length == 0) {
// pass
} else if (schoolinfo.length == 1) {
props["capacity"] = schoolinfo[0]["aantal inschrijvingen"].split(";").map(i => Number(i)).reduce((sum, i) => sum + i, 0)
props["capacity"] = schoolinfo[0]["aantal inschrijvingen"]
.split(";")
.map((i) => Number(i))
.reduce((sum, i) => sum + i, 0)
} else {
throw "Multiple schoolinfo's found for " + props.schoolnummer
}
//props["source:ref"] = props.schoolnummer
props["amenity"]="school"
if ( props["school"] === "kindergarten" ) {
props["amenity"] = "school"
if (props["school"] === "kindergarten") {
props["amenity"] = "kindergarten"
props["isced:2011:level"] = "early_education"
delete props["school"]
@ -210,28 +336,29 @@ function main() {
for (const renameKey in rename) {
const into = rename[renameKey]
if(props[renameKey] !== undefined){
if (props[renameKey] !== undefined) {
props[into] = props[renameKey]
delete props[renameKey]
}
}
for (const rmKey of rmKeys) {
delete props[rmKey]
}
}
//schoolGeojson.features = schoolGeojson.features.filter(f => f.properties["capacity"] !== undefined)
/*schoolGeojson.features.forEach((f, i) => {
f.properties["id"] = "school/"+i
})*/
schoolGeojson.features = schoolGeojson.features.filter(f => f.properties["amenity"] === "kindergarten")
schoolGeojson.features = schoolGeojson.features.filter(
(f) => f.properties["amenity"] === "kindergarten"
)
writeFileSync("scripts/schools/amended_schools.geojson", JSON.stringify(schoolGeojson), "utf8")
console.log("Done")
}
if(!process.argv[1].endsWith("mocha")){
if (!process.argv[1].endsWith("mocha")) {
main()
}

90
scripts/slice.ts
View file

@ -1,9 +1,9 @@
import * as fs from "fs";
import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource";
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource";
import * as readline from "readline";
import ScriptUtils from "./ScriptUtils";
import {Utils} from "../Utils";
import * as fs from "fs"
import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource"
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource"
import * as readline from "readline"
import ScriptUtils from "./ScriptUtils"
import { Utils } from "../Utils"
/**
* This script slices a big newline-delimeted geojson file into tiled geojson
@ -11,12 +11,12 @@ import {Utils} from "../Utils";
*/
async function readFeaturesFromLineDelimitedJsonFile(inputFile: string): Promise<any[]> {
const fileStream = fs.createReadStream(inputFile);
const fileStream = fs.createReadStream(inputFile)
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
crlfDelay: Infinity,
})
// Note: we use the crlfDelay option to recognize all instances of CR LF
// ('\r\n') in input.txt as a single line break.
@ -37,12 +37,12 @@ async function readFeaturesFromLineDelimitedJsonFile(inputFile: string): Promise
}
async function readGeojsonLineByLine(inputFile: string): Promise<any[]> {
const fileStream = fs.createReadStream(inputFile);
const fileStream = fs.createReadStream(inputFile)
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
crlfDelay: Infinity,
})
// Note: we use the crlfDelay option to recognize all instances of CR LF
// ('\r\n') in input.txt as a single line break.
@ -50,9 +50,9 @@ async function readGeojsonLineByLine(inputFile: string): Promise<any[]> {
let featuresSeen = false
// @ts-ignore
for await (let line: string of rl) {
if (!featuresSeen && line.startsWith("\"features\":")) {
featuresSeen = true;
continue;
if (!featuresSeen && line.startsWith('"features":')) {
featuresSeen = true
continue
}
if (!featuresSeen) {
continue
@ -84,7 +84,6 @@ async function readFeaturesFromGeoJson(inputFile: string): Promise<any[]> {
}
async function main(args: string[]) {
console.log("GeoJSON slicer")
if (args.length < 3) {
console.log("USAGE: <input-file.geojson> <target-zoom-level> <output-directory>")
@ -101,8 +100,7 @@ async function main(args: string[]) {
}
console.log("Using directory ", outputDirectory)
let allFeatures: any [];
let allFeatures: any[]
if (inputFile.endsWith(".geojson")) {
console.log("Detected geojson")
allFeatures = await readFeaturesFromGeoJson(inputFile)
@ -112,7 +110,6 @@ async function main(args: string[]) {
}
allFeatures = Utils.NoNull(allFeatures)
console.log("Loaded all", allFeatures.length, "points")
const keysToRemove = ["STRAATNMID", "GEMEENTE", "POSTCODE"]
@ -126,31 +123,40 @@ async function main(args: string[]) {
}
delete f.bbox
}
TiledFeatureSource.createHierarchy(
StaticFeatureSource.fromGeojson(allFeatures),
{
minZoomLevel: zoomlevel,
maxZoomLevel: zoomlevel,
maxFeatureCount: Number.MAX_VALUE,
registerTile: tile => {
const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson`
const features = tile.features.data.map(ff => ff.feature)
features.forEach(f => {
delete f.bbox
})
fs.writeFileSync(path, JSON.stringify({
"type": "FeatureCollection",
"features": features
}, null, " "))
ScriptUtils.erasableLog("Written ", path, "which has ", tile.features.data.length, "features")
}
}
)
TiledFeatureSource.createHierarchy(StaticFeatureSource.fromGeojson(allFeatures), {
minZoomLevel: zoomlevel,
maxZoomLevel: zoomlevel,
maxFeatureCount: Number.MAX_VALUE,
registerTile: (tile) => {
const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson`
const features = tile.features.data.map((ff) => ff.feature)
features.forEach((f) => {
delete f.bbox
})
fs.writeFileSync(
path,
JSON.stringify(
{
type: "FeatureCollection",
features: features,
},
null,
" "
)
)
ScriptUtils.erasableLog(
"Written ",
path,
"which has ",
tile.features.data.length,
"features"
)
},
})
}
let args = [...process.argv]
args.splice(0, 2)
main(args).then(_ => {
main(args).then((_) => {
console.log("All done!")
});
})

224
scripts/thieves/readIdPresets.ts
View file

@ -1,34 +1,34 @@
/***
* Parses presets from the iD repository and extracts some usefull tags from them
*/
import ScriptUtils from "../ScriptUtils";
import {existsSync, readFileSync, writeFileSync} from "fs";
import ScriptUtils from "../ScriptUtils"
import { existsSync, readFileSync, writeFileSync } from "fs"
import * as known_languages from "../../assets/language_native.json"
import {LayerConfigJson} from "../../Models/ThemeConfig/Json/LayerConfigJson";
import {MappingConfigJson} from "../../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson";
import SmallLicense from "../../Models/smallLicense";
import { LayerConfigJson } from "../../Models/ThemeConfig/Json/LayerConfigJson"
import { MappingConfigJson } from "../../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import SmallLicense from "../../Models/smallLicense"
interface IconThief {
steal(iconName: string): boolean
}
interface IdPresetJson {
icon: string,
icon: string
geometry: ("point" | "line" | "area")[]
/**
* Extra search terms
*/
terms: string []
terms: string[]
tags: Record<string, string>
name: string,
searchable?: boolean,
name: string
searchable?: boolean
}
class IdPreset implements IdPresetJson {
private _preset: IdPresetJson;
private _preset: IdPresetJson
constructor(preset: IdPresetJson) {
this._preset = preset;
this._preset = preset
}
public get searchable(): boolean {
@ -56,36 +56,38 @@ class IdPreset implements IdPresetJson {
}
static fromFile(file: string): IdPreset {
return new IdPreset(JSON.parse(readFileSync(file, 'utf8')))
return new IdPreset(JSON.parse(readFileSync(file, "utf8")))
}
public parseTags(): string | { and: string[] } {
const preset = this._preset;
const preset = this._preset
const tagKeys = Object.keys(preset.tags)
if (tagKeys.length === 1) {
return tagKeys[0] + "=" + preset.tags[tagKeys[0]]
} else {
return {
and: tagKeys.map(key => key + "=" + preset.tags[key])
and: tagKeys.map((key) => key + "=" + preset.tags[key]),
}
}
}
}
class MakiThief implements IconThief {
public readonly _prefix: string;
private readonly _directory: string;
private readonly _license: SmallLicense;
private readonly _targetDir: string;
public readonly _prefix: string
private readonly _directory: string
private readonly _license: SmallLicense
private readonly _targetDir: string
constructor(directory: string, targetDir: string,
license: SmallLicense,
prefix: string = "maki-") {
this._license = license;
this._directory = directory;
this._targetDir = targetDir;
this._prefix = prefix;
constructor(
directory: string,
targetDir: string,
license: SmallLicense,
prefix: string = "maki-"
) {
this._license = license
this._directory = directory
this._targetDir = targetDir
this._prefix = prefix
}
public steal(iconName: string): boolean {
@ -94,13 +96,14 @@ class MakiThief implements IconThief {
return true
}
try {
const file = readFileSync(this._directory + iconName + ".svg", "utf8")
writeFileSync(target, file, 'utf8')
writeFileSync(target, file, "utf8")
writeFileSync(target + ".license_info.json",
JSON.stringify(
{...this._license, path: this._prefix + iconName + ".svg"}), 'utf8')
writeFileSync(
target + ".license_info.json",
JSON.stringify({ ...this._license, path: this._prefix + iconName + ".svg" }),
"utf8"
)
console.log("Successfully stolen " + iconName)
return true
} catch (e) {
@ -108,17 +111,15 @@ class MakiThief implements IconThief {
return false
}
}
}
class AggregateIconThief implements IconThief {
private readonly makiThiefs: MakiThief[];
private readonly makiThiefs: MakiThief[]
constructor(makiThiefs: MakiThief[]) {
this.makiThiefs = makiThiefs;
this.makiThiefs = makiThiefs
}
public steal(iconName: string): boolean {
for (const makiThief1 of this.makiThiefs) {
if (iconName.startsWith(makiThief1._prefix)) {
@ -129,23 +130,31 @@ class AggregateIconThief implements IconThief {
}
}
class IdThief {
private readonly _idPresetsRepository: string;
private readonly _idPresetsRepository: string
private readonly _tranlationFiles: Record<string, object> = {}
private readonly _knownLanguages: string[]
private readonly _iconThief: IconThief;
private readonly _iconThief: IconThief
public constructor(idPresetsRepository: string, iconThief: IconThief) {
this._idPresetsRepository = idPresetsRepository;
this._iconThief = iconThief;
const knownById = ScriptUtils.readDirRecSync(`${this._idPresetsRepository}/dist/translations/`)
.map(pth => pth.substring(pth.lastIndexOf('/') + 1, pth.length - '.json'.length))
.filter(lng => !lng.endsWith('.min'));
const missing = Object.keys(known_languages).filter(lng => knownById.indexOf(lng.replace('-', '_')) < 0)
this._knownLanguages = knownById.filter(lng => known_languages[lng] !== undefined)
console.log("Id knows following languages:", this._knownLanguages.join(", "), "missing:", missing)
this._idPresetsRepository = idPresetsRepository
this._iconThief = iconThief
const knownById = ScriptUtils.readDirRecSync(
`${this._idPresetsRepository}/dist/translations/`
)
.map((pth) => pth.substring(pth.lastIndexOf("/") + 1, pth.length - ".json".length))
.filter((lng) => !lng.endsWith(".min"))
const missing = Object.keys(known_languages).filter(
(lng) => knownById.indexOf(lng.replace("-", "_")) < 0
)
this._knownLanguages = knownById.filter((lng) => known_languages[lng] !== undefined)
console.log(
"Id knows following languages:",
this._knownLanguages.join(", "),
"missing:",
missing
)
}
public getTranslation(language: string, ...path: string[]): string {
@ -153,28 +162,25 @@ class IdThief {
for (const p of path) {
obj = obj[p]
if (obj === undefined) {
return undefined;
return undefined
}
}
return obj
}
/**
* Creates a mapRendering-mapping for the 'shop' theme
*/
public readShopIcons(): { if: string | { and: string[] }, then: string }[] {
public readShopIcons(): { if: string | { and: string[] }; then: string }[] {
const dir = this._idPresetsRepository + "/data/presets/shop"
const mappings:
{
if: string | { and: string[] },
then: string
}[] = []
const files = ScriptUtils.readDirRecSync(dir, 1);
const mappings: {
if: string | { and: string[] }
then: string
}[] = []
const files = ScriptUtils.readDirRecSync(dir, 1)
for (const file of files) {
const preset = IdPreset.fromFile(file);
const preset = IdPreset.fromFile(file)
if (!this._iconThief.steal(preset.icon)) {
continue
@ -182,27 +188,24 @@ class IdThief {
const mapping = {
if: preset.parseTags(),
then: "circle:white;./assets/layers/id_presets/" + preset.icon + ".svg"
then: "circle:white;./assets/layers/id_presets/" + preset.icon + ".svg",
}
mappings.push(mapping)
}
return mappings
}
/**
* Creates a tagRenderingConfigJson for the 'shop' theme
*/
public readShopPresets(): MappingConfigJson[] {
const dir = this._idPresetsRepository + "/data/presets/shop"
const mappings: MappingConfigJson[] = []
const files = ScriptUtils.readDirRecSync(dir, 1);
const files = ScriptUtils.readDirRecSync(dir, 1)
for (const file of files) {
const name = file.substring(file.lastIndexOf('/') + 1, file.length - '.json'.length)
const name = file.substring(file.lastIndexOf("/") + 1, file.length - ".json".length)
const preset = IdPreset.fromFile(file)
if (preset.searchable === false) {
@ -212,30 +215,35 @@ class IdThief {
console.log(` ${name} (shop=${preset.tags["shop"]}), ${preset.icon}`)
const thenClause: Record<string, string> = {
en: preset.name
en: preset.name,
}
const terms: Record<string, string[]> = {
en: preset.terms
en: preset.terms,
}
for (const lng of this._knownLanguages) {
const lngMc = lng.replace('-', '_')
const lngMc = lng.replace("-", "_")
const tr = this.getTranslation(lng, "presets", "presets", "shop/" + name, "name")
if (tr !== undefined) {
thenClause[lngMc] = tr
}
const termsTr = this.getTranslation(lng, "presets", "presets", "shop/" + name, "terms")
const termsTr = this.getTranslation(
lng,
"presets",
"presets",
"shop/" + name,
"terms"
)
if (termsTr !== undefined) {
terms[lngMc] = termsTr.split(",")
}
}
let tag = preset.parseTags();
const mapping : MappingConfigJson= {
let tag = preset.parseTags()
const mapping: MappingConfigJson = {
if: tag,
then: thenClause,
searchTerms: terms
searchTerms: terms,
}
if (preset.tags["shop"] == "yes") {
mapping["hideInAnswer"] = true
@ -245,14 +253,13 @@ class IdThief {
if (this._iconThief.steal(preset.icon)) {
mapping["icon"] = {
path: "./assets/layers/id_presets/" + preset.icon + ".svg",
class: "medium"
class: "medium",
}
} else {
console.log(preset.icon + " could not be stolen :(")
}
mappings.push(mapping)
}
return mappings
@ -263,50 +270,63 @@ class IdThief {
if (cached) {
return cached
}
return this._tranlationFiles[language] = JSON.parse(readFileSync(`${this._idPresetsRepository}/dist/translations/${language}.json`, 'utf8'))
return (this._tranlationFiles[language] = JSON.parse(
readFileSync(`${this._idPresetsRepository}/dist/translations/${language}.json`, "utf8")
))
}
}
const targetDir = "./assets/layers/id_presets/"
const makiThief = new MakiThief('../maki/icons/', targetDir + "maki-", {
authors: ['Maki icon set'],
license: 'CC0',
path: null,
sources: ["https://github.com/mapbox/maki"]
}, 'maki-');
const temakiThief = new MakiThief('../temaki/icons/', targetDir + "temaki-", {
authors: ['Temaki icon set'],
license: 'CC0',
path: null,
sources: ["https://github.com/ideditor/temaki"]
}, 'temaki-');
const fasThief = new MakiThief('../Font-Awesome/svgs/solid/', targetDir + "fas-", {
authors: ['Font-Awesome icon set'],
license: 'CC-BY 4.0',
path: null,
sources: ["https://github.com/FortAwesome/Font-Awesome"]
}, 'fas-');
const iconThief = new AggregateIconThief(
[makiThief, temakiThief, fasThief]
const makiThief = new MakiThief(
"../maki/icons/",
targetDir + "maki-",
{
authors: ["Maki icon set"],
license: "CC0",
path: null,
sources: ["https://github.com/mapbox/maki"],
},
"maki-"
)
const temakiThief = new MakiThief(
"../temaki/icons/",
targetDir + "temaki-",
{
authors: ["Temaki icon set"],
license: "CC0",
path: null,
sources: ["https://github.com/ideditor/temaki"],
},
"temaki-"
)
const fasThief = new MakiThief(
"../Font-Awesome/svgs/solid/",
targetDir + "fas-",
{
authors: ["Font-Awesome icon set"],
license: "CC-BY 4.0",
path: null,
sources: ["https://github.com/FortAwesome/Font-Awesome"],
},
"fas-"
)
const iconThief = new AggregateIconThief([makiThief, temakiThief, fasThief])
const thief = new IdThief("../id-tagging-schema/", iconThief)
const id_presets_path = targetDir + "id_presets.json"
const idPresets = <LayerConfigJson>JSON.parse(readFileSync(id_presets_path, 'utf8'))
const idPresets = <LayerConfigJson>JSON.parse(readFileSync(id_presets_path, "utf8"))
idPresets.tagRenderings = [
{
id: "shop_types",
mappings: thief.readShopPresets()
mappings: thief.readShopPresets(),
},
{
id: "shop_rendering",
mappings: thief.readShopIcons()
}
mappings: thief.readShopIcons(),
},
]
writeFileSync(id_presets_path, JSON.stringify(idPresets, null, " "), 'utf8')
writeFileSync(id_presets_path, JSON.stringify(idPresets, null, " "), "utf8")

99
scripts/thieves/stealLanguages.ts
View file

@ -1,68 +1,75 @@
/*
* Uses the languages in and to every translation from wikidata to generate a language question in wikidata/wikidata
* */
* Uses the languages in and to every translation from wikidata to generate a language question in wikidata/wikidata
* */
import WikidataUtils from "../../Utils/WikidataUtils";
import {existsSync, mkdirSync, readFileSync, writeFileSync} from "fs";
import {LayerConfigJson} from "../../Models/ThemeConfig/Json/LayerConfigJson";
import {MappingConfigJson} from "../../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson";
import LanguageUtils from "../../Utils/LanguageUtils";
import WikidataUtils from "../../Utils/WikidataUtils"
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
import { LayerConfigJson } from "../../Models/ThemeConfig/Json/LayerConfigJson"
import { MappingConfigJson } from "../../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import LanguageUtils from "../../Utils/LanguageUtils"
import * as perCountry from "../../assets/language_in_country.json"
import {Utils} from "../../Utils";
function main(){
const sourcepath = "assets/generated/languages-wd.json";
import { Utils } from "../../Utils"
function main() {
const sourcepath = "assets/generated/languages-wd.json"
console.log(`Converting language data file '${sourcepath}' into a tagMapping`)
const languages = WikidataUtils.extractLanguageData(JSON.parse(readFileSync(sourcepath, "utf8")), {})
const mappings : MappingConfigJson[] = []
const schoolmappings : MappingConfigJson[] = []
const countryToLanguage : Record<string, string[]> = perCountry
const officialLanguagesPerCountry = Utils.TransposeMap(countryToLanguage);
const languages = WikidataUtils.extractLanguageData(
JSON.parse(readFileSync(sourcepath, "utf8")),
{}
)
const mappings: MappingConfigJson[] = []
const schoolmappings: MappingConfigJson[] = []
const countryToLanguage: Record<string, string[]> = perCountry
const officialLanguagesPerCountry = Utils.TransposeMap(countryToLanguage)
languages.forEach((l, code) => {
const then : Record<string, string>= {}
const then: Record<string, string> = {}
l.forEach((tr, lng) => {
const languageCodeWeblate = WikidataUtils.languageRemapping[lng] ?? lng;
if(!LanguageUtils.usedLanguages.has(languageCodeWeblate)){
return;
const languageCodeWeblate = WikidataUtils.languageRemapping[lng] ?? lng
if (!LanguageUtils.usedLanguages.has(languageCodeWeblate)) {
return
}
then[languageCodeWeblate] = tr
})
const officialCountries = Utils.Dedup(officialLanguagesPerCountry[code]?.map(s => s.toLowerCase()) ?? [])
const prioritySearch = officialCountries.length > 0 ? "_country~" + officialCountries.map(c => "((^|;)"+c+"($|;))").join("|") : undefined
const officialCountries = Utils.Dedup(
officialLanguagesPerCountry[code]?.map((s) => s.toLowerCase()) ?? []
)
const prioritySearch =
officialCountries.length > 0
? "_country~" + officialCountries.map((c) => "((^|;)" + c + "($|;))").join("|")
: undefined
mappings.push(<MappingConfigJson>{
if: "language:" + code + "=yes",
ifnot: "language:" + code + "=",
searchTerms: {
"*": [code]
"*": [code],
},
then,
priorityIf: prioritySearch
priorityIf: prioritySearch,
})
schoolmappings.push(<MappingConfigJson>{
schoolmappings.push(<MappingConfigJson>{
if: "school:language=" + code,
then,
priorityIf: prioritySearch,
searchTerms: {
"*":[code]
}
"*": [code],
},
})
})
const wikidataLayer = <LayerConfigJson>{
id: "wikidata",
description: {
en: "Various tagrenderings which are generated from Wikidata. Automatically generated with a script, don't edit manually"
en: "Various tagrenderings which are generated from Wikidata. Automatically generated with a script, don't edit manually",
},
"#dont-translate": "*",
"source": {
"osmTags": "id~*"
source: {
osmTags: "id~*",
},
title: null,
"mapRendering": null,
mapRendering: null,
tagRenderings: [
{
id: "language",
@ -75,27 +82,27 @@ function main(){
override: {
id: "language-multi",
// @ts-ignore
description: "Enables to pick *multiple* 'language:<lng>=yes' within the mappings",
multiAnswer: true
}
description:
"Enables to pick *multiple* 'language:<lng>=yes' within the mappings",
multiAnswer: true,
},
},
{
id:"school-language",
id: "school-language",
// @ts-ignore
description: "Enables to pick a single 'school:language=<lng>' within the mappings",
multiAnswer: true,
mappings: schoolmappings
}
]
mappings: schoolmappings,
},
],
}
const dir = "./assets/layers/wikidata/"
if(!existsSync(dir)){
if (!existsSync(dir)) {
mkdirSync(dir)
}
const path = dir + "wikidata.json"
writeFileSync(path, JSON.stringify(wikidataLayer, null, " "))
console.log("Written "+path)
console.log("Written " + path)
}
main()
main()

28
scripts/translationStatistics.ts
View file

@ -1,22 +1,24 @@
import {Utils} from "../Utils";
import {AllKnownLayouts} from "../Customizations/AllKnownLayouts";
import TranslatorsPanel from "../UI/BigComponents/TranslatorsPanel";
import { Utils } from "../Utils"
import { AllKnownLayouts } from "../Customizations/AllKnownLayouts"
import TranslatorsPanel from "../UI/BigComponents/TranslatorsPanel"
import * as languages from "../assets/generated/used_languages.json"
{
const usedLanguages = languages.languages
// Some statistics
console.log(Utils.FixedLength("", 12) + " " + usedLanguages.map(l => Utils.FixedLength(l, 6)).join(""))
console.log(
Utils.FixedLength("", 12) + " " + usedLanguages.map((l) => Utils.FixedLength(l, 6)).join("")
)
const all = new Map<string, number[]>()
usedLanguages.forEach(ln => all.set(ln, []))
usedLanguages.forEach((ln) => all.set(ln, []))
for (const layoutId of Array.from(AllKnownLayouts.allKnownLayouts.keys())) {
const layout = AllKnownLayouts.allKnownLayouts.get(layoutId)
if(layout.hideFromOverview){
if (layout.hideFromOverview) {
continue
}
const {completeness, total} = TranslatorsPanel.MissingTranslationsFor(layout)
const { completeness, total } = TranslatorsPanel.MissingTranslationsFor(layout)
process.stdout.write(Utils.FixedLength(layout.id, 12) + " ")
for (const language of usedLanguages) {
const compl = completeness.get(language)
@ -25,7 +27,7 @@ import * as languages from "../assets/generated/used_languages.json"
process.stdout.write(" ")
continue
}
const percentage = Math.round(100 * compl / total)
const percentage = Math.round((100 * compl) / total)
process.stdout.write(Utils.FixedLength(percentage + "%", 6))
}
process.stdout.write("\n")
@ -35,10 +37,12 @@ import * as languages from "../assets/generated/used_languages.json"
for (const language of usedLanguages) {
const ratios = all.get(language)
let sum = 0
ratios.forEach(x => sum += x)
ratios.forEach((x) => (sum += x))
const percentage = Math.round(100 * (sum / ratios.length))
process.stdout.write(Utils.FixedLength(percentage + "%", 6))
}
process.stdout.write("\n")
console.log(Utils.FixedLength("", 12) + " " + usedLanguages.map(l => Utils.FixedLength(l, 6)).join(""))
}
console.log(
Utils.FixedLength("", 12) + " " + usedLanguages.map((l) => Utils.FixedLength(l, 6)).join("")
)
}