elgaard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Merge branch 'develop' into feature/traffic-signs

Browse source
This commit is contained in:
Robin van der Linde 2023-05-24 09:06:30 +02:00
commit 70e4b44d76
Signed by untrusted user: Robin-van-der-Linde
GPG key ID: 53956B3252478F0D
845 changed files with 47579 additions and 18878 deletions
Download patch file Download diff file Expand all files Collapse all files

12
scripts/GenerateSeries.ts
View file

@ -1,4 +1,4 @@
import fs, { existsSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
import { existsSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
import ScriptUtils from "./ScriptUtils"
import { Utils } from "../Utils"
import Script from "./Script"
@ -56,9 +56,9 @@ class StatsDownloader {
}.day.json`
writtenFiles.push(path)
if (existsSync(path)) {
let features = JSON.parse(readFileSync(path, { encoding: "utf-8" }))
features = features?.features ?? features
features.push(...features.features) // day-stats are generally a list already, but in some ad-hoc cases might be a geojson-collection too
let loadedFeatures = JSON.parse(readFileSync(path, { encoding: "utf-8" }))
loadedFeatures = loadedFeatures?.features ?? loadedFeatures
features.push(...loadedFeatures) // day-stats are generally a list already, but in some ad-hoc cases might be a geojson-collection too
console.log(
"Loaded ",
path,
@ -199,7 +199,7 @@ class GenerateSeries extends Script {
}
async main(args: string[]): Promise<void> {
const targetDir = args[0] ?? "../MapComplete-data"
const targetDir = args[0] ?? "../../git/MapComplete-data"
await this.downloadStatistics(targetDir + "/changeset-metadata")
await this.generateCenterPoints(
@ -296,7 +296,7 @@ class GenerateSeries extends Script {
features.forEach((f) => {
delete f.bbox
})
fs.writeFileSync(
writeFileSync(
path,
JSON.stringify(
{

42
scripts/ScriptUtils.ts
View file

@ -1,9 +1,9 @@
import * as fs from "fs"
import { existsSync, lstatSync, readdirSync, readFileSync } from "fs"
import { Utils } from "../Utils"
import {existsSync, lstatSync, readdirSync, readFileSync} from "fs"
import {Utils} from "../Utils"
import * as https from "https"
import { LayoutConfigJson } from "../Models/ThemeConfig/Json/LayoutConfigJson"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import {LayoutConfigJson} from "../Models/ThemeConfig/Json/LayoutConfigJson"
import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson"
import xml2js from "xml2js"
export default class ScriptUtils {
@ -37,14 +37,16 @@ export default class ScriptUtils {
return result
}
public static DownloadFileTo(url, targetFilePath: string): void {
console.log("Downloading ", url, "to", targetFilePath)
https.get(url, (res) => {
const filePath = fs.createWriteStream(targetFilePath)
res.pipe(filePath)
filePath.on("finish", () => {
filePath.close()
console.log("Download Completed")
public static DownloadFileTo(url, targetFilePath: string): Promise<void> {
ScriptUtils.erasableLog("Downloading", url, "to", targetFilePath)
return new Promise<void>((resolve, err) => {
https.get(url, (res) => {
const filePath = fs.createWriteStream(targetFilePath)
res.pipe(filePath)
filePath.on("finish", () => {
filePath.close()
resolve()
})
})
})
}
@ -78,13 +80,13 @@ export default class ScriptUtils {
.filter((path) => path.indexOf("license_info.json") < 0)
.map((path) => {
try {
const contents = readFileSync(path, { encoding: "utf8" })
const contents = readFileSync(path, {encoding: "utf8"})
if (contents === "") {
throw "The file " + path + " is empty, did you properly save?"
}
const parsed = JSON.parse(contents)
return { parsed, path }
return {parsed, path}
} catch (e) {
console.error("Could not parse file ", "./assets/layers/" + path, "due to ", e)
throw e
@ -101,12 +103,12 @@ export default class ScriptUtils {
public static getThemeFiles(): { parsed: LayoutConfigJson; path: string }[] {
return this.getThemePaths().map((path) => {
try {
const contents = readFileSync(path, { encoding: "utf8" })
const contents = readFileSync(path, {encoding: "utf8"})
if (contents === "") {
throw "The file " + path + " is empty, did you properly save?"
}
const parsed = JSON.parse(contents)
return { parsed: parsed, path: path }
return {parsed: parsed, path: path}
} catch (e) {
console.error("Could not read file ", path, "due to ", e)
throw e
@ -125,14 +127,14 @@ export default class ScriptUtils {
if (!existsSync(path)) {
throw "File not found: " + path
}
const root = await xml2js.parseStringPromise(readFileSync(path, { encoding: "utf8" }))
const root = await xml2js.parseStringPromise(readFileSync(path, {encoding: "utf8"}))
return root.svg
}
public static ReadSvgSync(path: string, callback: (svg: any) => void): any {
xml2js.parseString(
readFileSync(path, { encoding: "utf8" }),
{ async: false },
readFileSync(path, {encoding: "utf8"}),
{async: false},
(err, root) => {
if (err) {
throw err
@ -171,7 +173,7 @@ export default class ScriptUtils {
})
res.addListener("end", function () {
resolve({ content: parts.join("") })
resolve({content: parts.join("")})
})
}
)

1
scripts/automoveTranslations.ts
View file

@ -1,4 +1,3 @@
import * as languages from "../assets/generated/used_languages.json"
import { readFileSync, writeFileSync } from "fs"
/**

18
scripts/build.sh
View file

@ -9,6 +9,8 @@ rm -rf .cache
mkdir dist 2> /dev/null
mkdir dist/assets 2> /dev/null
export NODE_OPTIONS="--max-old-space-size=8192"
# This script ends every line with '&&' to chain everything. A failure will thus stop the build
npm run generate:editor-layer-index &&
npm run generate &&
@ -33,9 +35,17 @@ then
echo "Source maps are enabled"
fi
ASSET_URL="mc/$BRANCH"
export ASSET_URL
echo "$ASSET_URL"
if [ $BRANCH = "master" ]
then
ASSET_URL="./"
export ASSET_URL
echo "$ASSET_URL"
else
ASSET_URL="mc/$BRANCH"
export ASSET_URL
echo "$ASSET_URL"
fi
export NODE_OPTIONS=--max-old-space-size=6500
vite build $SRC_MAPS
@ -48,3 +58,5 @@ cp -r assets/templates/ dist/assets/templates/
cp -r assets/tagRenderings/ dist/assets/tagRenderings/
cp assets/*.png dist/assets/
cp assets/*.svg dist/assets/
export NODE_OPTIONS=""

96
scripts/downloadEli.ts Normal file
View file

@ -0,0 +1,96 @@
import Script from "./Script"
import { Utils } from "../Utils"
import { FeatureCollection } from "geojson"
import fs from "fs"
class DownloadEli extends Script {
constructor() {
super("Downloads a fresh copy of the editor layer index, removes all unnecessary data.")
}
async main(args: string[]): Promise<void> {
const url = "https://osmlab.github.io/editor-layer-index/imagery.geojson"
// Target should use '.json' instead of '.geojson', as the latter cannot be imported by the build systems
const target = args[0] ?? "assets/editor-layer-index.json"
const eli = <FeatureCollection>await Utils.downloadJson(url)
const keptLayers = []
console.log("Got", eli.features.length, "ELI-entries")
for (let layer of eli.features) {
const props = layer.properties
if (props.type === "bing") {
// A lot of work to implement - see https://github.com/pietervdvn/MapComplete/issues/648
continue
}
if (props.id === "MAPNIK") {
// Already added by default
continue
}
if (props.overlay) {
continue
}
if (props.id === "Mapbox") {
/**
* This token is managed by Martin Reifer on the 'OpenStreetMap'-account on MapBox
*/
const token =
"pk.eyJ1Ijoib3BlbnN0cmVldG1hcCIsImEiOiJjbGZkempiNDkyandvM3lwY3M4MndpdWdzIn0.QnvRv52n3qffVEKmQa9vJA"
props.url = props.url.replace("{apikey}", token)
}
if (props.url.toLowerCase().indexOf("apikey") > 0) {
continue
}
if (props.permission_url === "no") {
continue
}
if (props.max_zoom < 19) {
// We want users to zoom to level 19 when adding a point
// If they are on a layer which hasn't enough precision, they can not zoom far enough. This is confusing, so we don't use this layer
continue
}
if (props.name === undefined) {
console.warn("Editor layer index: name not defined on ", props)
continue
}
const keptKeys = [
"name",
"id",
"url",
"attribution",
"type",
"category",
"min_zoom",
"max_zoom",
"best",
"default",
"tile-size",
]
layer.properties = {}
for (const keptKey of keptKeys) {
if (props[keptKey]) {
layer.properties[keptKey] = props[keptKey]
}
}
layer = { properties: layer.properties, type: layer.type, geometry: layer.geometry }
keptLayers.push(layer)
}
const contents =
'{"type":"FeatureCollection",\n "features": [\n' +
keptLayers.map((l) => JSON.stringify(l)).join(",\n") +
"\n]}"
fs.writeFileSync(target, contents, { encoding: "utf8" })
console.log("Written", keptLayers.length + ", entries to the ELI")
}
}
new DownloadEli().run()

29
scripts/downloadFromOverpass.ts Normal file
View file

@ -0,0 +1,29 @@
import Script from "./Script"
import { TagUtils } from "../Logic/Tags/TagUtils"
import { And } from "../Logic/Tags/And"
import Constants from "../Models/Constants"
import { ImmutableStore } from "../Logic/UIEventSource"
import { BBox } from "../Logic/BBox"
import { Overpass } from "../Logic/Osm/Overpass"
const fs = require("fs")
class DownloadFromOverpass extends Script {
constructor() {
super(
"Downloads data from openstreetmap, will save this as 'export.geojson'. All arguments will be interpreted as key=value pairs"
)
}
async main(args: string[]): Promise<void> {
const tags = new And(args.map((k) => TagUtils.Tag(k)))
const overpass = new Overpass(
tags,
[],
Constants.defaultOverpassUrls[0],
new ImmutableStore(500)
)
const [data, _] = await overpass.queryGeoJson(BBox.global)
fs.writeFileSync("export.geojson", JSON.stringify(data), "utf8")
console.log("Written", data.features.length, "entries")
}
}
new DownloadFromOverpass().run()

2
scripts/extractLayer.ts
View file

@ -15,7 +15,7 @@ function main(args: string[]) {
const layerId = args[1]
const themePath = "./assets/themes/" + themeId + "/" + themeId + ".json"
const contents = <LayoutConfigJson>JSON.parse(readFileSync(themePath, "UTF-8"))
const contents = <LayoutConfigJson>JSON.parse(readFileSync(themePath, { encoding: "utf8" }))
const layers = <LayerConfigJson[]>contents.layers.filter((l) => {
if (typeof l === "string") {
return false

47
scripts/fetchLanguages.ts
View file

@ -7,10 +7,22 @@ import * as wds from "wikidata-sdk"
import { Utils } from "../Utils"
import ScriptUtils from "./ScriptUtils"
import { existsSync, readFileSync, writeFileSync } from "fs"
import { QuestionableTagRenderingConfigJson } from "../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import WikidataUtils from "../Utils/WikidataUtils"
import LanguageUtils from "../Utils/LanguageUtils"
import Wikidata from "../Logic/Web/Wikidata"
interface value<T> {
value: T
type: "uri" | "literal" | string
"xml:lang"?: string
}
interface LanguageSpecResult {
directionalityLabel: value<string | "right-to-left" | "left-to-right">
lang: value<string>
code: value<string>
label: value<string>
}
async function fetch(target: string) {
const regular = await fetchRegularLanguages()
@ -36,33 +48,52 @@ async function fetchRegularLanguages() {
// request the generated URL with your favorite HTTP request library
const result = await Utils.downloadJson(url, { "User-Agent": "MapComplete script" })
const bindings = result.results.bindings
const bindings = <LanguageSpecResult[]>result.results.bindings
const zh_hant = await fetchSpecial(18130932, "zh_Hant")
const zh_hans = await fetchSpecial(13414913, "zh_Hant")
const pt_br = await fetchSpecial(750553, "pt_BR")
const punjabi = await fetchSpecial(58635, "pa_PK")
const Shahmukhi = await Wikidata.LoadWikidataEntryAsync(133800)
punjabi.forEach((item) => {
const neededLanguage = item.label["xml:lang"]
const native = Shahmukhi.labels.get(neededLanguage) ?? Shahmukhi.labels.get("en")
item.label.value = item.label.value + " (" + native + ")"
})
const fil = await fetchSpecial(33298, "fil")
bindings.push(...zh_hant)
bindings.push(...zh_hans)
bindings.push(...pt_br)
bindings.push(...fil)
bindings.push(...punjabi)
return result.results.bindings
}
async function fetchSpecial(id: number, code: string) {
/**
* Fetches the object as is. Sets a 'code' binding as predifined value
* @param id
* @param code
*/
async function fetchSpecial(id: number, code: string): Promise<LanguageSpecResult[]> {
ScriptUtils.fixUtils()
console.log("Fetching languages")
const lang = " wd:Q" + id
const sparql =
"SELECT ?lang ?label ?code \n" +
"SELECT ?label ?directionalityLabel \n" +
"WHERE \n" +
"{ \n" +
" wd:Q" +
id +
" rdfs:label ?label. \n" +
lang +
" rdfs:label ?label." +
lang +
" wdt:P282 ?writing_system. \n" +
" ?writing_system wdt:P1406 ?directionality. \n" +
' SERVICE wikibase:label { bd:serviceParam wikibase:language "en". } \n' +
"} "
console.log("Special sparql:", sparql)
const url = wds.sparqlQuery(sparql)
const result = await Utils.downloadJson(url, { "User-Agent": "MapComplete script" })

117
scripts/fixQuestionHint.ts Normal file
View file

@ -0,0 +1,117 @@
import * as fs from "fs"
import { DesugaringStep } from "../Models/ThemeConfig/Conversion/Conversion"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import { QuestionableTagRenderingConfigJson } from "../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import * as fakedom from "fake-dom"
import Script from "./Script"
import { FixedUiElement } from "../UI/Base/FixedUiElement"
class ExtractQuestionHint extends DesugaringStep<QuestionableTagRenderingConfigJson> {
constructor() {
super(
"Tries to extract a 'questionHint' from the question",
["question", "questionhint"],
"ExtractQuestionHint"
)
}
convert(
json: QuestionableTagRenderingConfigJson,
context: string
): {
result: QuestionableTagRenderingConfigJson
errors?: string[]
warnings?: string[]
information?: string[]
} {
json = { ...json }
if (json.question === undefined || json.questionHint !== undefined) {
return { result: json }
}
if (typeof json.question === "string") {
return { result: json }
}
const hint: Record<string, string> = {}
for (const language in json.question) {
const q = json.question[language]
const parts = q.split(/<br ?\/>/i)
if (parts.length == 2) {
json.question[language] = parts[0]
const txt = new FixedUiElement(parts[1]).ConstructElement().textContent
if (txt.length > 0) {
hint[language] = txt
}
continue
}
const divStart = [q.indexOf("<div "), q.indexOf("<span "), q.indexOf("<p ")].find(
(i) => i > 0
) // note: > 0, not >= : we are not interested in a span starting right away!
if (divStart > 0) {
json.question[language] = q.substring(0, divStart)
const txt = new FixedUiElement(q.substring(divStart)).ConstructElement().textContent
if (txt !== "") {
hint[language] = txt
}
}
}
if (Object.keys(hint).length > 0) {
json.questionHint = hint
}
console.log("Inspecting ", json.question)
return { result: json }
}
}
class FixQuestionHint extends Script {
private fs: any
constructor() {
super("Extracts a 'questionHint' from a question for a given 'layer.json' or 'theme.json'")
if (fakedom === undefined) {
throw "Fakedom not active"
}
}
async main(args: string[]): Promise<void> {
const filepath = args[0]
const contents = JSON.parse(fs.readFileSync(filepath, { encoding: "utf8" }))
const convertor = new ExtractQuestionHint()
if (filepath.endsWith("/questions.json")) {
for (const key in contents) {
const tr = contents[key]
if (typeof tr !== "object") {
continue
}
contents[key] = convertor.convertStrict(
tr,
"While automatically extracting questiondHints of " + filepath
)
}
fs.writeFileSync(filepath, JSON.stringify(contents, null, " "), { encoding: "utf-8" })
return
}
const layers: LayerConfigJson[] = contents["layers"] ?? [contents]
for (const layer of layers) {
for (let i = 0; i < layer.tagRenderings?.length; i++) {
const tagRendering = layer.tagRenderings[i]
if (typeof tagRendering !== "object" || tagRendering["question"] === undefined) {
continue
}
layer.tagRenderings[i] = convertor.convertStrict(
<QuestionableTagRenderingConfigJson>tagRendering,
"While automatically extracting questionHints of " + filepath
)
}
}
// The layer(s) are modified inPlace, so we can simply write to disk
fs.writeFileSync(filepath, JSON.stringify(contents, null, " "), { encoding: "utf8" })
}
}
new FixQuestionHint().run()

6
scripts/fixSchemas.ts
View file

@ -42,7 +42,7 @@ function WalkScheme<T>(
}
const definitionName = ref.substr(prefix.length)
if (isHandlingReference.indexOf(definitionName) >= 0) {
return
return []
}
const loadedScheme = fullScheme.definitions[definitionName]
return WalkScheme(onEach, loadedScheme, fullScheme, path, [
@ -137,7 +137,9 @@ function main() {
def["additionalProperties"] = false
}
}
writeFileSync(dir + "/" + name + ".schema.json", JSON.stringify(parsed, null, " "), { encoding: "utf8" })
writeFileSync(dir + "/" + name + ".schema.json", JSON.stringify(parsed, null, " "), {
encoding: "utf8",
})
}
extractMeta("LayoutConfigJson", "layoutconfigmeta")

44
scripts/generateCache.ts
View file

@ -21,9 +21,12 @@ import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSou
import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource"
import Constants from "../Models/Constants"
import { GeoOperations } from "../Logic/GeoOperations"
import SimpleMetaTaggers from "../Logic/SimpleMetaTagger"
import SimpleMetaTaggers, { ReferencingWaysMetaTagger } from "../Logic/SimpleMetaTagger"
import FilteringFeatureSource from "../Logic/FeatureSource/Sources/FilteringFeatureSource"
import Loc from "../Models/Loc"
import { Feature } from "geojson"
import { BBox } from "../Logic/BBox"
import { bboxClip } from "@turf/turf"
ScriptUtils.fixUtils()
@ -78,13 +81,13 @@ function geoJsonName(targetDir: string, x: number, y: number, z: number): string
return targetDir + "_" + z + "_" + x + "_" + y + ".geojson"
}
/// Downloads the given feature and saves them to disk
/// Downloads the given tilerange from overpass and saves them to disk
async function downloadRaw(
targetdir: string,
r: TileRange,
theme: LayoutConfig,
relationTracker: RelationsTracker
) /* : {failed: number, skipped :number} */ {
): Promise<{ failed: number; skipped: number }> {
let downloaded = 0
let failed = 0
let skipped = 0
@ -232,7 +235,8 @@ function sliceToTiles(
theme: LayoutConfig,
relationsTracker: RelationsTracker,
targetdir: string,
pointsOnlyLayers: string[]
pointsOnlyLayers: string[],
clip: boolean
) {
const skippedLayers = new Set<string>()
@ -310,6 +314,7 @@ function sliceToTiles(
maxFeatureCount: undefined,
registerTile: (tile) => {
const tileIndex = tile.tileIndex
const bbox = BBox.fromTileIndex(tileIndex).asGeoJson({})
console.log("Got tile:", tileIndex, tile.layer.layerDef.id)
if (tile.features.data.length === 0) {
return
@ -343,9 +348,9 @@ function sliceToTiles(
}
let strictlyCalculated = 0
let featureCount = 0
for (const feature of filteredTile.features.data) {
let features: Feature[] = filteredTile.features.data.map((f) => f.feature)
for (const feature of features) {
// Some cleanup
delete feature.feature["bbox"]
if (tile.layer.layerDef.calculatedTags !== undefined) {
// Evaluate all the calculated tags strictly
@ -353,7 +358,7 @@ function sliceToTiles(
(ct) => ct[0]
)
featureCount++
const props = feature.feature.properties
const props = feature.properties
for (const calculatedTagKey of calculatedTagKeys) {
const strict = props[calculatedTagKey]
@ -379,7 +384,16 @@ function sliceToTiles(
}
}
}
delete feature["bbox"]
}
if (clip) {
console.log("Clipping features")
features = [].concat(
...features.map((f: Feature) => GeoOperations.clipWith(<any>f, bbox))
)
}
// Lets save this tile!
const [z, x, y] = Tiles.tile_from_index(tileIndex)
// console.log("Writing tile ", z, x, y, layerId)
@ -391,7 +405,7 @@ function sliceToTiles(
JSON.stringify(
{
type: "FeatureCollection",
features: filteredTile.features.data.map((f) => f.feature),
features,
},
null,
" "
@ -474,10 +488,12 @@ function sliceToTiles(
export async function main(args: string[]) {
console.log("Cache builder started with args ", args.join(", "))
ReferencingWaysMetaTagger.enabled = false
if (args.length < 6) {
console.error(
"Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1 [--generate-point-overview layer-name,layer-name,...] [--force-zoom-level z] \n" +
"Note: a new directory named <theme> will be created in targetdirectory"
"Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1 [--generate-point-overview layer-name,layer-name,...] [--force-zoom-level z] [--clip]" +
"--force-zoom-level causes non-cached-layers to be donwnloaded\n" +
"--clip will erase parts of the feature falling outside of the bounding box"
)
return
}
@ -494,6 +510,7 @@ export async function main(args: string[]) {
const lon0 = Number(args[4])
const lat1 = Number(args[5])
const lon1 = Number(args[6])
const clip = args.indexOf("--clip") >= 0
if (isNaN(lat0)) {
throw "The first number (a latitude) is not a valid number"
@ -523,10 +540,7 @@ export async function main(args: string[]) {
const theme = AllKnownLayouts.allKnownLayouts.get(themeName)
if (theme === undefined) {
const keys = []
AllKnownLayouts.allKnownLayouts.forEach((_, key) => {
keys.push(key)
})
const keys = Array.from(AllKnownLayouts.allKnownLayouts.keys())
console.error("The theme " + theme + " was not found; try one of ", keys)
return
}
@ -570,7 +584,7 @@ export async function main(args: string[]) {
const extraFeatures = await downloadExtraData(theme)
const allFeaturesSource = loadAllTiles(targetdir, tileRange, theme, extraFeatures)
sliceToTiles(allFeaturesSource, theme, relationTracker, targetdir, generatePointLayersFor)
sliceToTiles(allFeaturesSource, theme, relationTracker, targetdir, generatePointLayersFor, clip)
}
let args = [...process.argv]

21
scripts/generateContributors.ts
View file

@ -1,10 +1,23 @@
import { exec } from "child_process"
import { writeFile, writeFileSync } from "fs"
function asList(hist: Map<string, number>): {
contributors: { contributor: string; commits: number }[]
} {
const ls = []
interface Contributor {
/**
* The name of the contributor
*/
contributor: string
/**
* The number of commits
*/
commits: number
}
interface ContributorList {
contributors: Contributor[]
}
function asList(hist: Map<string, number>): ContributorList {
const ls: Contributor[] = []
hist.forEach((commits, contributor) => {
ls.push({ commits, contributor })
})

210
scripts/generateDocs.ts
View file

@ -15,14 +15,20 @@ import List from "../UI/Base/List"
import SharedTagRenderings from "../Customizations/SharedTagRenderings"
import { writeFile } from "fs"
import Translations from "../UI/i18n/Translations"
import * as themeOverview from "../assets/generated/theme_overview.json"
import themeOverview from "../assets/generated/theme_overview.json"
import DefaultGUI from "../UI/DefaultGUI"
import FeaturePipelineState from "../Logic/State/FeaturePipelineState"
import LayoutConfig from "../Models/ThemeConfig/LayoutConfig"
import * as bookcases from "../assets/generated/themes/bookcases.json"
import bookcases from "../assets/generated/themes/bookcases.json"
import { DefaultGuiState } from "../UI/DefaultGuiState"
import * as fakedom from "fake-dom"
import fakedom from "fake-dom"
import Hotkeys from "../UI/Base/Hotkeys"
import { QueryParameters } from "../Logic/Web/QueryParameters"
import Link from "../UI/Base/Link"
import Constants from "../Models/Constants"
import LayerConfig from "../Models/ThemeConfig/LayerConfig"
import DependencyCalculator from "../Models/ThemeConfig/DependencyCalculator"
import { AllSharedLayers } from "../Customizations/AllSharedLayers"
function WriteFile(
filename,
html: BaseUIElement,
@ -63,7 +69,187 @@ function WriteFile(
md.replace(/\n\n\n+/g, "\n\n")
writeFileSync(filename, md)
if (!md.endsWith("\n")) {
md += "\n"
}
const warnAutomated =
"[//]: # (WARNING: this file is automatically generated. Please find the sources at the bottom and edit those sources)"
writeFileSync(filename, warnAutomated + md)
}
function GenerateDocumentationForTheme(theme: LayoutConfig): BaseUIElement {
return new Combine([
new Title(
new Combine([
theme.title,
"(",
new Link(theme.id, "https://mapcomplete.osm.be/" + theme.id),
")",
]),
2
),
theme.description,
"This theme contains the following layers:",
new List(
theme.layers
.filter((l) => !l.id.startsWith("note_import_"))
.map((l) => new Link(l.id, "../Layers/" + l.id + ".md"))
),
"Available languages:",
new List(theme.language.filter((ln) => ln !== "_context")),
]).SetClass("flex flex-col")
}
/**
* Generates the documentation for the layers overview page
* @constructor
*/
function GenLayerOverviewText(): BaseUIElement {
for (const id of Constants.priviliged_layers) {
if (!AllSharedLayers.sharedLayers.has(id)) {
throw "Priviliged layer definition not found: " + id
}
}
const allLayers: LayerConfig[] = Array.from(AllSharedLayers.sharedLayers.values()).filter(
(layer) => Constants.priviliged_layers.indexOf(layer.id) < 0
)
const builtinLayerIds: Set<string> = new Set<string>()
allLayers.forEach((l) => builtinLayerIds.add(l.id))
const themesPerLayer = new Map<string, string[]>()
for (const layout of Array.from(AllKnownLayouts.allKnownLayouts.values())) {
for (const layer of layout.layers) {
if (!builtinLayerIds.has(layer.id)) {
continue
}
if (!themesPerLayer.has(layer.id)) {
themesPerLayer.set(layer.id, [])
}
themesPerLayer.get(layer.id).push(layout.id)
}
}
// Determine the cross-dependencies
const layerIsNeededBy: Map<string, string[]> = new Map<string, string[]>()
for (const layer of allLayers) {
for (const dep of DependencyCalculator.getLayerDependencies(layer)) {
const dependency = dep.neededLayer
if (!layerIsNeededBy.has(dependency)) {
layerIsNeededBy.set(dependency, [])
}
layerIsNeededBy.get(dependency).push(layer.id)
}
}
return new Combine([
new Title("Special and other useful layers", 1),
"MapComplete has a few data layers available in the theme which have special properties through builtin-hooks. Furthermore, there are some normal layers (which are built from normal Theme-config files) but are so general that they get a mention here.",
new Title("Priviliged layers", 1),
new List(Constants.priviliged_layers.map((id) => "[" + id + "](#" + id + ")")),
...Constants.priviliged_layers
.map((id) => AllSharedLayers.sharedLayers.get(id))
.map((l) =>
l.GenerateDocumentation(
themesPerLayer.get(l.id),
layerIsNeededBy,
DependencyCalculator.getLayerDependencies(l),
Constants.added_by_default.indexOf(l.id) >= 0,
Constants.no_include.indexOf(l.id) < 0
)
),
new Title("Normal layers", 1),
"The following layers are included in MapComplete:",
new List(
Array.from(AllSharedLayers.sharedLayers.keys()).map(
(id) => new Link(id, "./Layers/" + id + ".md")
)
),
])
}
/**
* Generates documentation for the layers.
* Inline layers are included (if the theme is public)
* @param callback
* @constructor
*/
function GenOverviewsForSingleLayer(
callback: (layer: LayerConfig, element: BaseUIElement, inlineSource: string) => void
): void {
const allLayers: LayerConfig[] = Array.from(AllSharedLayers.sharedLayers.values()).filter(
(layer) => Constants.priviliged_layers.indexOf(layer.id) < 0
)
const builtinLayerIds: Set<string> = new Set<string>()
allLayers.forEach((l) => builtinLayerIds.add(l.id))
const inlineLayers = new Map<string, string>()
for (const layout of Array.from(AllKnownLayouts.allKnownLayouts.values())) {
if (layout.hideFromOverview) {
continue
}
for (const layer of layout.layers) {
if (Constants.priviliged_layers.indexOf(layer.id) >= 0) {
continue
}
if (builtinLayerIds.has(layer.id)) {
continue
}
if (layer.source.geojsonSource !== undefined) {
// Not an OSM-source
continue
}
allLayers.push(layer)
builtinLayerIds.add(layer.id)
inlineLayers.set(layer.id, layout.id)
}
}
const themesPerLayer = new Map<string, string[]>()
for (const layout of Array.from(AllKnownLayouts.allKnownLayouts.values())) {
if (layout.hideFromOverview) {
continue
}
for (const layer of layout.layers) {
if (!builtinLayerIds.has(layer.id)) {
// This is an inline layer
continue
}
if (!themesPerLayer.has(layer.id)) {
themesPerLayer.set(layer.id, [])
}
themesPerLayer.get(layer.id).push(layout.id)
}
}
// Determine the cross-dependencies
const layerIsNeededBy: Map<string, string[]> = new Map<string, string[]>()
for (const layer of allLayers) {
for (const dep of DependencyCalculator.getLayerDependencies(layer)) {
const dependency = dep.neededLayer
if (!layerIsNeededBy.has(dependency)) {
layerIsNeededBy.set(dependency, [])
}
layerIsNeededBy.get(dependency).push(layer.id)
}
}
allLayers.forEach((layer) => {
const element = layer.GenerateDocumentation(
themesPerLayer.get(layer.id),
layerIsNeededBy,
DependencyCalculator.getLayerDependencies(layer)
)
callback(layer, element, inlineLayers.get(layer.id))
})
}
/**
@ -103,7 +289,7 @@ function generateWikipage() {
"! Name, link !! Genre !! Covered region !! Language !! Description !! Free materials !! Image\n" +
"|-"
for (const layout of themeOverview["default"] ?? themeOverview) {
for (const layout of themeOverview) {
if (layout.hideFromOverview) {
continue
}
@ -123,7 +309,7 @@ console.log("Starting documentation generation...")
ScriptUtils.fixUtils()
generateWikipage()
AllKnownLayouts.GenOverviewsForSingleLayer((layer, element, inlineSource) => {
GenOverviewsForSingleLayer((layer, element, inlineSource) => {
console.log("Exporting ", layer.id)
if (!existsSync("./Docs/Layers")) {
mkdirSync("./Docs/Layers")
@ -136,7 +322,7 @@ AllKnownLayouts.GenOverviewsForSingleLayer((layer, element, inlineSource) => {
})
Array.from(AllKnownLayouts.allKnownLayouts.values()).map((theme) => {
const docs = AllKnownLayouts.GenerateDocumentationForTheme(theme)
const docs = GenerateDocumentationForTheme(theme)
WriteFile(
"./Docs/Themes/" + theme.id + ".md",
docs,
@ -159,9 +345,7 @@ WriteFile(
WriteFile("./Docs/SpecialInputElements.md", ValidatedTextField.HelpText(), [
"UI/Input/ValidatedTextField.ts",
])
WriteFile("./Docs/BuiltinLayers.md", AllKnownLayouts.GenLayerOverviewText(), [
"Customizations/AllKnownLayouts.ts",
])
WriteFile("./Docs/BuiltinLayers.md", GenLayerOverviewText(), ["Customizations/AllKnownLayouts.ts"])
WriteFile("./Docs/BuiltinQuestions.md", SharedTagRenderings.HelpText(), [
"Customizations/SharedTagRenderings.ts",
"assets/tagRenderings/questions.json",
@ -225,6 +409,12 @@ WriteFile("./Docs/URL_Parameters.md", QueryParameterDocumentation.GenerateQueryP
if (fakedom === undefined || window === undefined) {
throw "FakeDom not initialized"
}
QueryParameters.GetQueryParameter(
"mode",
"map",
"The mode the application starts in, e.g. 'map', 'dashboard' or 'statistics'"
)
new DefaultGUI(
new FeaturePipelineState(new LayoutConfig(<any>bookcases)),
new DefaultGuiState()

169
scripts/generateImageAnalysis.ts
View file

@ -1,14 +1,15 @@
import Script from "./Script"
import { Overpass } from "../Logic/Osm/Overpass"
import { RegexTag } from "../Logic/Tags/RegexTag"
import { ImmutableStore } from "../Logic/UIEventSource"
import { BBox } from "../Logic/BBox"
import {Overpass} from "../Logic/Osm/Overpass"
import {RegexTag} from "../Logic/Tags/RegexTag"
import {ImmutableStore} from "../Logic/UIEventSource"
import {BBox} from "../Logic/BBox"
import * as fs from "fs"
import { Feature } from "geojson"
import {Feature} from "geojson"
import ScriptUtils from "./ScriptUtils"
import { Imgur } from "../Logic/ImageProviders/Imgur"
import { LicenseInfo } from "../Logic/ImageProviders/LicenseInfo"
import { Utils } from "../Utils"
import {Imgur} from "../Logic/ImageProviders/Imgur"
import {LicenseInfo} from "../Logic/ImageProviders/LicenseInfo"
import {Utils} from "../Utils"
import Constants from "../Models/Constants";
export default class GenerateImageAnalysis extends Script {
constructor() {
@ -17,9 +18,9 @@ export default class GenerateImageAnalysis extends Script {
)
}
async fetchImages(key: string, datapath: string): Promise<void> {
async fetchImages(key: string, datapath: string, refresh: boolean): Promise<void> {
const targetPath = `${datapath}/features_with_${key.replace(/[:\/]/, "_")}.geojson`
if (fs.existsSync(targetPath)) {
if (fs.existsSync(targetPath) && !refresh) {
console.log("Skipping", key)
return
}
@ -27,27 +28,26 @@ export default class GenerateImageAnalysis extends Script {
const overpass = new Overpass(
tag,
[],
"https://overpass.kumi.systems/api/interpreter",
new ImmutableStore(180),
Constants.defaultOverpassUrls[0], //"https://overpass.kumi.systems/api/interpreter",
new ImmutableStore(500),
undefined,
false
)
console.log("Starting query...")
const data = await overpass.queryGeoJson(BBox.global)
console.log("Got data: ", data[0].features.length)
console.log("Got data:", data[0].features.length, "items; timestamp:", data[1].toISOString())
fs.writeFileSync(targetPath, JSON.stringify(data[0]), "utf8")
console.log("Written", targetPath)
}
async downloadData(datapath: string): Promise<void> {
async downloadData(datapath: string, refresh: boolean): Promise<void> {
if (!fs.existsSync(datapath)) {
fs.mkdirSync(datapath)
}
await this.fetchImages("image", datapath)
await this.fetchImages("image:streetsign", datapath)
await this.fetchImages("image", datapath, refresh)
await this.fetchImages("image:streetsign", datapath, refresh)
for (let i = 0; i < 5; i++) {
await this.fetchImages("image:" + i, datapath)
await this.fetchImages("image:" + i, datapath, refresh)
}
}
@ -73,25 +73,55 @@ export default class GenerateImageAnalysis extends Script {
if (!image.match(/https:\/\/i\.imgur\.com\/[a-zA-Z0-9]+\.jpg/)) {
return false
}
const targetPath = datapath + "/" + image.replace(/[\/:.\-%]/g, "_") + ".json"
const filename = image.replace(/[\/:.\-%]/g, "_") + ".json"
const targetPath = datapath + "/" + filename
if (fs.existsSync(targetPath)) {
return false
}
const attribution = await Imgur.singleton.DownloadAttribution(image)
if ((attribution.artist ?? "") === "") {
// This is an invalid attribution. We save the raw response as well
const hash = image.substr("https://i.imgur.com/".length).split(".jpg")[0]
const apiUrl = "https://api.imgur.com/3/image/" + hash
const response = await Utils.downloadJsonCached(apiUrl, 365 * 24 * 60 * 60, {
Authorization: "Client-ID " + Constants.ImgurApiKey,
})
const rawTarget = datapath + "/raw/" + filename
console.log("Also storing the raw response to", rawTarget)
await fs.writeFileSync(rawTarget, JSON.stringify(response, null, " "))
}
await fs.writeFileSync(targetPath, JSON.stringify(attribution, null, " "))
return true
}
async downloadMetadata(datapath: string): Promise<void> {
const features = this.loadData(datapath)
loadImageUrls(datapath: string): { allImages: Set<string>, imageSource: Map<string, string> } {
let allImages = new Set<string>()
const features = this.loadData(datapath)
let imageSource: Map<string, string> = new Map<string, string>()
for (const feature of features) {
allImages.add(feature.properties["image"])
imageSource[feature.properties["image"]] = feature.properties.id
allImages.add(feature.properties["image:streetsign"])
imageSource[feature.properties["image:streetsign"]] = feature.properties.id + " (streetsign)"
for (let i = 0; i < 10; i++) {
allImages.add(feature.properties["image:" + i])
imageSource[feature.properties["image:" + i]] = `${feature.properties.id} (image:${i})`
}
}
allImages.delete(undefined)
allImages.delete(null)
imageSource.delete(undefined)
imageSource.delete(null)
return {allImages, imageSource}
}
async downloadMetadata(datapath: string): Promise<void> {
const {allImages, imageSource} = this.loadImageUrls(datapath)
console.log("Detected", allImages.size, "images")
let i = 0
let d = 0
@ -110,10 +140,12 @@ export default class GenerateImageAnalysis extends Script {
const msg = `${i}/${
allImages.size
} downloaded: ${d},skipped: ${s}, failed: ${f}, running: ${runningSecs}sec, ETA: ${estimatedActualMinutes}:${
estimatedActualSeconds % 60
}`
ScriptUtils.erasableLog(msg)
} downloaded: ${d},skipped: ${s}, failed: ${f}, running: ${Math.floor(
runningSecs
)}sec, ETA: ${estimatedActualMinutes}:${estimatedActualSeconds % 60}`
if (d + f % 1000 === 1 || downloaded) {
ScriptUtils.erasableLog(msg)
}
if (downloaded) {
d++
} else {
@ -121,10 +153,80 @@ export default class GenerateImageAnalysis extends Script {
}
if (d + f == 75000) {
console.log("Used 75000 API calls, leaving 5000 for the rest of the day...")
break
}
} catch (e) {
// console.log(e)
console.log("Offending image hash is", image, "from https://openstreetmap.org/" + imageSource[image])
f++
}
}
}
async downloadImage(url: string, imagePath: string): Promise<boolean> {
const filenameLong = url.replace(/[\/:.\-%]/g, "_") + ".jpg"
const targetPathLong = imagePath + "/" + filenameLong
const filename = url.substring("https://i.imgur.com/".length)
const targetPath = imagePath + "/" + filename
if (fs.existsSync(targetPathLong)) {
if (fs.existsSync(targetPath)) {
fs.unlinkSync(targetPathLong)
console.log("Unlinking duplicate")
return false
}
console.log("Renaming...")
fs.renameSync(targetPathLong, targetPath)
return false
}
if (fs.existsSync(targetPath)) {
return false
}
await ScriptUtils.DownloadFileTo(url, targetPath)
return true
}
async downloadAllImages(datapath: string, imagePath: string): Promise<void> {
const {allImages} = this.loadImageUrls(datapath)
let skipped = 0
let failed = 0
let downloaded = 0
let invalid = 0
const startTime = Date.now()
const urls = Array.from(allImages).filter(url => url.startsWith("https://i.imgur.com"))
for (const url of urls) {
const runningTime = ((Date.now()) - startTime) / 1000
const handled = skipped + downloaded + failed
const itemsLeft = allImages.size - handled
const speed = handled / runningTime
const timeLeft = Math.round(itemsLeft * speed)
try {
const downloadedStatus = await Promise.all(url.split(";").map(url =>
this.downloadImage(url.trim(), imagePath),
))
for (const b of downloadedStatus) {
if (b) {
downloaded += 1
} else {
skipped += 1
}
}
if (downloadedStatus.some(i => i) || skipped % 10000 === 0) {
console.log("Handled", url, JSON.stringify({
skipped,
failed,
downloaded,
invalid,
total: allImages.size,
eta: timeLeft + "s"
}))
}
} catch (e) {
console.log(e)
f++
failed++
}
}
}
@ -138,7 +240,7 @@ export default class GenerateImageAnalysis extends Script {
if (!file.endsWith(".json")) {
continue
}
const attr = <LicenseInfo>JSON.parse(fs.readFileSync(file, { encoding: "utf8" }))
const attr = <LicenseInfo>JSON.parse(fs.readFileSync(file, {encoding: "utf8"}))
const license = attr.licenseShortName
if (license === undefined || attr.artist === undefined) {
@ -217,7 +319,7 @@ export default class GenerateImageAnalysis extends Script {
...Array.from(licenseByAuthor.get("CC-BY-SA 4.0").values()),
]
console.log("Total number of correctly licenses pictures: ", totalLicensedImages)
console.log("Total number of correctly licenses pictures: ", totalLicensedImages, "(out of ", files.length, " images)")
console.log("Total number of authors:", byAuthor.size)
console.log(
"Total number of authors which used a valid, non CC0 license at one point in time",
@ -227,10 +329,15 @@ export default class GenerateImageAnalysis extends Script {
}
async main(args: string[]): Promise<void> {
const datapath = args[0] ?? "../MapComplete-data/ImageLicenseInfo"
await this.downloadData(datapath)
console.log("Usage: [--cached] to use the cached osm data")
console.log("Args are", args)
const cached = args.indexOf("--cached") < 0
args = args.filter(a => a !== "--cached")
const datapath = args[0] ?? "../../git/MapComplete-data/ImageLicenseInfo"
await this.downloadData(datapath, cached)
//await this.downloadMetadata(datapath)
await this.downloadMetadata(datapath)
await this.downloadAllImages(datapath, "/home/pietervdvn/data/imgur-image-backup")
this.analyze(datapath)
}
}

73
scripts/generateLayerOverview.ts
View file

@ -1,9 +1,10 @@
import ScriptUtils from "./ScriptUtils"
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "fs"
import * as licenses from "../assets/generated/license_info.json"
import licenses from "../assets/generated/license_info.json"
import { LayoutConfigJson } from "../Models/ThemeConfig/Json/LayoutConfigJson"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import Constants from "../Models/Constants"
import * as fakedom from "fake-dom"
import {
DetectDuplicateFilters,
DoesImageExist,
@ -14,23 +15,25 @@ import {
} from "../Models/ThemeConfig/Conversion/Validation"
import { Translation } from "../UI/i18n/Translation"
import { TagRenderingConfigJson } from "../Models/ThemeConfig/Json/TagRenderingConfigJson"
import * as questions from "../assets/tagRenderings/questions.json"
import * as icons from "../assets/tagRenderings/icons.json"
import questions from "../assets/tagRenderings/questions.json"
import PointRenderingConfigJson from "../Models/ThemeConfig/Json/PointRenderingConfigJson"
import { PrepareLayer } from "../Models/ThemeConfig/Conversion/PrepareLayer"
import { PrepareTheme } from "../Models/ThemeConfig/Conversion/PrepareTheme"
import { DesugaringContext } from "../Models/ThemeConfig/Conversion/Conversion"
import { Utils } from "../Utils"
import { AllKnownLayouts } from "../Customizations/AllKnownLayouts"
import { Script } from "vm"
import Script from "./Script"
import { AllSharedLayers } from "../Customizations/AllSharedLayers"
// This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files.
// It spits out an overview of those to be used to load them
class LayerOverviewUtils {
class LayerOverviewUtils extends Script {
public static readonly layerPath = "./assets/generated/layers/"
public static readonly themePath = "./assets/generated/themes/"
constructor() {
super("Reviews and generates the compiled themes")
}
private static publicLayerIdsFrom(themefiles: LayoutConfigJson[]): Set<string> {
const publicThemes = [].concat(...themefiles.filter((th) => !th.hideFromOverview))
@ -155,7 +158,7 @@ class LayerOverviewUtils {
const dict = new Map<string, TagRenderingConfigJson>()
const validator = new ValidateTagRenderings(undefined, doesImageExist)
for (const key in questions["default"]) {
for (const key in questions) {
if (key === "id") {
continue
}
@ -168,21 +171,6 @@ class LayerOverviewUtils {
)
dict.set(key, config)
}
for (const key in icons["default"]) {
if (key === "id") {
continue
}
if (typeof icons[key] !== "object") {
continue
}
icons[key].id = key
const config = <TagRenderingConfigJson>icons[key]
validator.convertStrict(
config,
"generate-layer-overview:tagRenderings/icons.json:" + key
)
dict.set(key, config)
}
dict.forEach((value, key) => {
if (key === "id") {
@ -241,7 +229,10 @@ class LayerOverviewUtils {
}
}
main(args: string[]) {
async main(args: string[]) {
if (fakedom === undefined) {
throw "Fakedom not initialized"
}
const forceReload = args.some((a) => a == "--force")
const licensePaths = new Set<string>()
@ -252,16 +243,15 @@ class LayerOverviewUtils {
const sharedLayers = this.buildLayerIndex(doesImageExist, forceReload)
const recompiledThemes: string[] = []
const sharedThemes = this.buildThemeIndex(
doesImageExist,
licensePaths,
sharedLayers,
recompiledThemes,
forceReload
)
writeFileSync(
"./assets/generated/known_layers_and_themes.json",
"./assets/generated/known_themes.json",
JSON.stringify({
layers: Array.from(sharedLayers.values()),
themes: Array.from(sharedThemes.values()),
})
)
@ -306,7 +296,7 @@ class LayerOverviewUtils {
"GenerateLayerOverview:"
)
if (AllKnownLayouts.getSharedLayersConfigs().size == 0) {
if (AllSharedLayers.getSharedLayersConfigs().size == 0) {
console.error("This was a bootstrapping-run. Run generate layeroverview again!")
} else {
const green = (s) => "\x1b[92m" + s + "\x1b[0m"
@ -325,7 +315,7 @@ class LayerOverviewUtils {
const sharedTagRenderings = this.getSharedTagRenderings(doesImageExist)
const state: DesugaringContext = {
tagRenderings: sharedTagRenderings,
sharedLayers: AllKnownLayouts.getSharedLayersConfigs(),
sharedLayers: AllSharedLayers.getSharedLayersConfigs(),
}
const sharedLayers = new Map<string, LayerConfigJson>()
const prepLayer = new PrepareLayer(state)
@ -382,7 +372,7 @@ class LayerOverviewUtils {
}
private buildThemeIndex(
doesImageExist: DoesImageExist,
licensePaths: Set<string>,
sharedLayers: Map<string, LayerConfigJson>,
recompiledThemes: string[],
forceReload: boolean
@ -397,9 +387,26 @@ class LayerOverviewUtils {
const convertState: DesugaringContext = {
sharedLayers,
tagRenderings: this.getSharedTagRenderings(doesImageExist),
tagRenderings: this.getSharedTagRenderings(
new DoesImageExist(licensePaths, existsSync)
),
publicLayers,
}
const knownTagRenderings = new Set<string>()
convertState.tagRenderings.forEach((_, key) => knownTagRenderings.add(key))
sharedLayers.forEach((layer) => {
for (const tagRendering of layer.tagRenderings ?? []) {
if (tagRendering["id"]) {
knownTagRenderings.add(layer.id + "." + tagRendering["id"])
}
if (tagRendering["labels"]) {
for (const label of tagRendering["labels"]) {
knownTagRenderings.add(layer.id + "." + label)
}
}
}
})
const skippedThemes: string[] = []
for (const themeInfo of themeFiles) {
const themePath = themeInfo.path
@ -434,10 +441,10 @@ class LayerOverviewUtils {
themeFile = new PrepareTheme(convertState).convertStrict(themeFile, themePath)
new ValidateThemeAndLayers(
doesImageExist,
new DoesImageExist(licensePaths, existsSync, knownTagRenderings),
themePath,
true,
convertState.tagRenderings
knownTagRenderings
).convertStrict(themeFile, themePath)
if (themeFile.icon.endsWith(".svg")) {
@ -502,4 +509,4 @@ class LayerOverviewUtils {
}
}
new LayerOverviewUtils().main(process.argv)
new LayerOverviewUtils().run()

26
scripts/generateLayouts.ts
View file

@ -2,7 +2,7 @@ import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFile, writeFi
import Locale from "../UI/i18n/Locale"
import Translations from "../UI/i18n/Translations"
import { Translation } from "../UI/i18n/Translation"
import * as all_known_layouts from "../assets/generated/known_layers_and_themes.json"
import all_known_layouts from "../assets/generated/known_themes.json"
import { LayoutConfigJson } from "../Models/ThemeConfig/Json/LayoutConfigJson"
import LayoutConfig from "../Models/ThemeConfig/LayoutConfig"
import xml2js from "xml2js"
@ -24,7 +24,7 @@ async function createIcon(iconPath: string, size: number, alreadyWritten: string
}
const newname = `assets/generated/images/${name.replace(/\//g, "_")}${size}.png`
const targetpath = `public/${newname}`
if (alreadyWritten.indexOf(newname) >= 0) {
return newname
}
@ -41,7 +41,7 @@ async function createIcon(iconPath: string, size: number, alreadyWritten: string
// We already read to file, in order to crash here if the file is not found
let img = await sharp(iconPath)
let resized = await img.resize(size)
await resized.toFile(newname)
await resized.toFile(targetpath)
console.log("Created png version at ", newname)
} catch (e) {
console.error("Could not read icon", iconPath, " to create a PNG due to", e)
@ -60,7 +60,7 @@ async function createSocialImage(layout: LayoutConfig, template: "" | "Wide"): P
)
return undefined
}
const path = `./assets/generated/images/social_image_${layout.id}_${template}.svg`
const path = `./public/assets/generated/images/social_image_${layout.id}_${template}.svg`
if (existsSync(path)) {
return path
}
@ -121,7 +121,7 @@ async function createManifest(
// This is an svg. Lets create the needed pngs and do some checkes!
const whiteBackgroundPath =
"./assets/generated/images/theme_" + layout.id + "_white_background.svg"
"./public/assets/generated/images/theme_" + layout.id + "_white_background.svg"
{
const svg = await ScriptUtils.ReadSvg(icon)
const width: string = svg.$.width
@ -136,7 +136,7 @@ async function createManifest(
let path = layout.icon
if (layout.icon.startsWith("<")) {
// THis is already the svg
path = "./assets/generated/images/" + layout.id + "_logo.svg"
path = "./public/assets/generated/images/" + layout.id + "_logo.svg"
writeFileSync(path, layout.icon)
}
@ -235,7 +235,7 @@ async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alr
let icon = layout.icon
if (icon.startsWith("<?xml") || icon.startsWith("<svg")) {
// This already is an svg
icon = `./assets/generated/images/${layout.id}_icon.svg`
icon = `./public/assets/generated/images/${layout.id}_icon.svg`
writeFileSync(icon, layout.icon)
}
@ -273,7 +273,7 @@ async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alr
'<script type="module" src="./index.ts"></script>',
`<script type="module" src='./index_${layout.id}.ts'></script>`
)
0
try {
output = output
.replace(
@ -295,7 +295,7 @@ async function createIndexFor(theme: LayoutConfig) {
const filename = "index_" + theme.id + ".ts"
writeFileSync(
filename,
`import * as themeConfig from "./assets/generated/themes/${theme.id}.json"\n`
`import themeConfig from "./assets/generated/themes/${theme.id}.json"\n`
)
appendFileSync(filename, codeTemplate)
}
@ -311,7 +311,9 @@ async function main(): Promise<void> {
createDir("./assets/generated")
createDir("./assets/generated/layers")
createDir("./assets/generated/themes")
createDir("./assets/generated/images")
createDir("./public/assets/")
createDir("./public/assets/generated")
createDir("./public/assets/generated/images")
const blacklist = [
"",
@ -353,7 +355,7 @@ async function main(): Promise<void> {
const { manifest, whiteIcons } = await createManifest(layout, alreadyWritten)
const manif = JSON.stringify(manifest, undefined, 2)
const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest"
writeFile(manifestLocation, manif, err)
writeFile("public/" + manifestLocation, manif, err)
// Create a landing page for the given theme
const landing = await createLandingPage(layout, manifest, whiteIcons, alreadyWritten)
@ -377,7 +379,7 @@ async function main(): Promise<void> {
)
const manif = JSON.stringify(manifest, undefined, 2)
writeFileSync("index.manifest", manif)
writeFileSync("public/index.webmanifest", manif)
}
main().then(() => {

632
scripts/generateLicenseInfo.ts
View file

@ -1,326 +1,352 @@
import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
import SmallLicense from "../Models/smallLicense"
import ScriptUtils from "./ScriptUtils"
import Script from "./Script"
const prompt = require("prompt-sync")()
function validateLicenseInfo(l: SmallLicense) {
l.sources.map((s) => {
try {
return new URL(s)
} catch (e) {
throw "Could not parse URL " + s + " for a license for " + l.path + " due to " + e
}
})
}
/**
* Sweeps the entire 'assets/' (except assets/generated) directory for image files and any 'license_info.json'-file.
* Checks that the license info is included for each of them and generates a compiles license_info.json for those
*/
function generateLicenseInfos(paths: string[]): SmallLicense[] {
const licenses = []
for (const path of paths) {
try {
const parsed = JSON.parse(readFileSync(path, { encoding: "utf8" }))
if (Array.isArray(parsed)) {
const l: SmallLicense[] = parsed
for (const smallLicens of l) {
smallLicens.path =
path.substring(0, path.length - "license_info.json".length) +
smallLicens.path
}
licenses.push(...l)
} else {
const smallLicens: SmallLicense = parsed
smallLicens.path = path.substring(0, 1 + path.lastIndexOf("/")) + smallLicens.path
licenses.push(smallLicens)
}
} catch (e) {
console.error("Error: ", e, "while handling", path)
}
}
return licenses
}
function missingLicenseInfos(licenseInfos: SmallLicense[], allIcons: string[]) {
const missing = []
const knownPaths = new Set<string>()
for (const licenseInfo of licenseInfos) {
knownPaths.add(licenseInfo.path)
export class GenerateLicenseInfo extends Script {
constructor() {
super("Validates the licenses and compiles them into one single asset file")
}
for (const iconPath of allIcons) {
if (iconPath.indexOf("license_info.json") >= 0) {
continue
}
if (knownPaths.has(iconPath)) {
continue
}
missing.push(iconPath)
}
return missing
}
static defaultLicenses() {
const knownLicenses = new Map<string, SmallLicense>()
knownLicenses.set("me", {
authors: ["Pieter Vander Vennet"],
path: undefined,
license: "CC0",
sources: [],
})
knownLicenses.set("streetcomplete", {
authors: ["Tobias Zwick (westnordost)"],
path: undefined,
license: "CC0",
sources: [
"https://github.com/streetcomplete/StreetComplete/tree/master/res/graphics",
"https://f-droid.org/packages/de.westnordost.streetcomplete/",
],
})
const knownLicenses = new Map<string, SmallLicense>()
knownLicenses.set("me", {
authors: ["Pieter Vander Vennet"],
path: undefined,
license: "CC0",
sources: [],
})
knownLicenses.set("streetcomplete", {
authors: ["Tobias Zwick (westnordost)"],
path: undefined,
license: "CC0",
sources: [
"https://github.com/streetcomplete/StreetComplete/tree/master/res/graphics",
"https://f-droid.org/packages/de.westnordost.streetcomplete/",
],
})
knownLicenses.set("temaki", {
authors: ["Temaki"],
path: undefined,
license: "CC0",
sources: [
"https://github.com/ideditor/temaki",
"https://ideditor.github.io/temaki/docs/",
],
})
knownLicenses.set("temaki", {
authors: ["Temaki"],
path: undefined,
license: "CC0",
sources: ["https://github.com/ideditor/temaki", "https://ideditor.github.io/temaki/docs/"],
})
knownLicenses.set("maki", {
authors: ["Maki"],
path: undefined,
license: "CC0",
sources: ["https://labs.mapbox.com/maki-icons/"],
})
knownLicenses.set("maki", {
authors: ["Maki"],
path: undefined,
license: "CC0",
sources: ["https://labs.mapbox.com/maki-icons/"],
})
knownLicenses.set("t", {
authors: [],
path: undefined,
license: "CC0; trivial",
sources: [],
})
knownLicenses.set("na", {
authors: [],
path: undefined,
license: "CC0",
sources: [],
})
knownLicenses.set("tv", {
authors: ["Toerisme Vlaanderen"],
path: undefined,
license: "CC0",
sources: [
"https://toerismevlaanderen.be/pinjepunt",
"https://mapcomplete.osm.be/toerisme_vlaanderenn",
],
})
knownLicenses.set("tvf", {
authors: ["Jo De Baerdemaeker "],
path: undefined,
license: "All rights reserved",
sources: ["https://www.studiotype.be/fonts/flandersart"],
})
knownLicenses.set("twemoji", {
authors: ["Twemoji"],
path: undefined,
license: "CC-BY 4.0",
sources: ["https://github.com/twitter/twemoji"],
})
function promptLicenseFor(path): SmallLicense {
console.log("License abbreviations:")
knownLicenses.forEach((value, key) => {
console.log(key, " => ", value)
})
const author = prompt("What is the author for artwork " + path + "? (or: [Q]uit, [S]kip) > ")
path = path.substring(path.lastIndexOf("/") + 1)
if (knownLicenses.has(author)) {
const license = knownLicenses.get(author)
license.path = path
return license
knownLicenses.set("t", {
authors: [],
path: undefined,
license: "CC0; trivial",
sources: [],
})
knownLicenses.set("na", {
authors: [],
path: undefined,
license: "CC0",
sources: [],
})
knownLicenses.set("tv", {
authors: ["Toerisme Vlaanderen"],
path: undefined,
license: "CC0",
sources: [
"https://toerismevlaanderen.be/pinjepunt",
"https://mapcomplete.osm.be/toerisme_vlaanderenn",
],
})
knownLicenses.set("tvf", {
authors: ["Jo De Baerdemaeker "],
path: undefined,
license: "All rights reserved",
sources: ["https://www.studiotype.be/fonts/flandersart"],
})
knownLicenses.set("twemoji", {
authors: ["Twemoji"],
path: undefined,
license: "CC-BY 4.0",
sources: ["https://github.com/twitter/twemoji"],
})
return knownLicenses
}
if (author == "s") {
return null
}
if (author == "Q" || author == "q" || author == "") {
throw "Quitting now!"
}
let authors = author.split(";")
if (author.toLowerCase() == "none") {
authors = []
}
return {
authors: author.split(";"),
path: path,
license: prompt("What is the license for artwork " + path + "? > "),
sources: prompt("Where was this artwork found? > ").split(";"),
}
}
function createLicenseInfoFor(path): void {
const li = promptLicenseFor(path)
if (li == null) {
return
}
writeFileSync(path + ".license_info.json", JSON.stringify(li, null, " "))
}
function cleanLicenseInfo(allPaths: string[], allLicenseInfos: SmallLicense[]) {
// Read the license info file from the generated assets, creates a compiled license info in every directory
// Note: this removes all the old license infos
for (const licensePath of allPaths) {
unlinkSync(licensePath)
}
const perDirectory = new Map<string, SmallLicense[]>()
for (const license of allLicenseInfos) {
const p = license.path
const dir = p.substring(0, p.lastIndexOf("/"))
license.path = p.substring(dir.length + 1)
if (!perDirectory.has(dir)) {
perDirectory.set(dir, [])
}
const cloned: SmallLicense = {
// We make a clone to force the order of the keys
path: license.path,
license: license.license,
authors: license.authors,
sources: license.sources,
}
perDirectory.get(dir).push(cloned)
}
perDirectory.forEach((licenses, dir) => {
for (let i = licenses.length - 1; i >= 0; i--) {
const license = licenses[i]
const path = dir + "/" + license.path
if (!existsSync(path)) {
console.log(
"Found license for now missing file: ",
path,
" - removing this license"
)
licenses.splice(i, 1)
}
}
licenses.sort((a, b) => (a.path < b.path ? -1 : 1))
writeFileSync(dir + "/license_info.json", JSON.stringify(licenses, null, 2))
})
}
function queryMissingLicenses(missingLicenses: string[]) {
process.on("SIGINT", function () {
console.log("Aborting... Bye!")
process.exit()
})
let i = 1
for (const missingLicens of missingLicenses) {
console.log(i + " / " + missingLicenses.length)
i++
if (i < missingLicenses.length - 5) {
// continue
}
createLicenseInfoFor(missingLicens)
}
console.log("You're through!")
}
/**
* Creates the humongous license_info in the generated assets, containing all licenses with a path relative to the root
* @param licensePaths
*/
function createFullLicenseOverview(licensePaths: string[]) {
const allLicenses: SmallLicense[] = []
for (const licensePath of licensePaths) {
if (!existsSync(licensePath)) {
continue
}
const licenses = <SmallLicense[]>JSON.parse(readFileSync(licensePath, { encoding: "utf8" }))
for (const license of licenses) {
validateLicenseInfo(license)
const dir = licensePath.substring(0, licensePath.length - "license_info.json".length)
license.path = dir + license.path
allLicenses.push(license)
}
}
writeFileSync("./assets/generated/license_info.json", JSON.stringify(allLicenses, null, " "))
}
function main(args: string[]) {
console.log("Checking and compiling license info")
if (!existsSync("./assets/generated")) {
mkdirSync("./assets/generated")
}
let contents = ScriptUtils.readDirRecSync("./assets")
.filter((p) => !p.startsWith("./assets/templates/"))
.filter((entry) => entry.indexOf("./assets/generated") != 0)
let licensePaths = contents.filter((entry) => entry.indexOf("license_info.json") >= 0)
let licenseInfos = generateLicenseInfos(licensePaths)
const artwork = contents.filter(
(pth) => pth.match(/(.svg|.png|.jpg|.ttf|.otf|.woff)$/i) != null
)
const missingLicenses = missingLicenseInfos(licenseInfos, artwork)
if (args.indexOf("--prompt") >= 0 || args.indexOf("--query") >= 0) {
queryMissingLicenses(missingLicenses)
return main([])
}
const invalidLicenses = licenseInfos
.filter((l) => (l.license ?? "") === "")
.map((l) => `License for artwork ${l.path} is empty string or undefined`)
let invalid = 0
for (const licenseInfo of licenseInfos) {
const isTrivial =
licenseInfo.license
.split(";")
.map((l) => l.trim().toLowerCase())
.indexOf("trivial") >= 0
if (licenseInfo.sources.length + licenseInfo.authors.length == 0 && !isTrivial) {
invalid++
invalidLicenses.push(
"Invalid license: No sources nor authors given in the license for " +
JSON.stringify(licenseInfo)
)
continue
}
for (const source of licenseInfo.sources) {
if (source == "") {
invalidLicenses.push(
"Invalid license: empty string in " + JSON.stringify(licenseInfo)
)
}
validateLicenseInfo(l: SmallLicense) {
l.sources.map((s) => {
try {
new URL(source)
} catch {
invalidLicenses.push("Not a valid URL: " + source)
return new URL(s)
} catch (e) {
throw "Could not parse URL " + s + " for a license for " + l.path + " due to " + e
}
})
}
/**
* Sweeps the entire 'assets/' (except assets/generated) directory for image files and any 'license_info.json'-file.
* Checks that the license info is included for each of them and generates a compiles license_info.json for those
*/
generateLicenseInfos(paths: string[]): SmallLicense[] {
const licenses = []
for (const path of paths) {
try {
const parsed = JSON.parse(readFileSync(path, { encoding: "utf8" }))
if (Array.isArray(parsed)) {
const l: SmallLicense[] = parsed
for (const smallLicens of l) {
smallLicens.path =
path.substring(0, path.length - "license_info.json".length) +
smallLicens.path
}
licenses.push(...l)
} else {
const smallLicens: SmallLicense = parsed
smallLicens.path =
path.substring(0, 1 + path.lastIndexOf("/")) + smallLicens.path
licenses.push(smallLicens)
}
} catch (e) {
console.error("Error: ", e, "while handling", path)
}
}
return licenses
}
if (missingLicenses.length > 0 || invalidLicenses.length) {
const msg = `There are ${missingLicenses.length} licenses missing and ${invalidLicenses.length} invalid licenses.`
console.log(missingLicenses.concat(invalidLicenses).join("\n"))
console.error(msg)
if (args.indexOf("--no-fail") < 0) {
throw msg
missingLicenseInfos(licenseInfos: SmallLicense[], allIcons: string[]) {
const missing = []
const knownPaths = new Set<string>()
for (const licenseInfo of licenseInfos) {
knownPaths.add(licenseInfo.path)
}
for (const iconPath of allIcons) {
if (iconPath.indexOf("license_info.json") >= 0) {
continue
}
if (knownPaths.has(iconPath)) {
continue
}
missing.push(iconPath)
}
return missing
}
promptLicenseFor(path): SmallLicense {
const knownLicenses = GenerateLicenseInfo.defaultLicenses()
console.log("License abbreviations:")
knownLicenses.forEach((value, key) => {
console.log(key, " => ", value)
})
const author = prompt(
"What is the author for artwork " + path + "? (or: [Q]uit, [S]kip) > "
)
path = path.substring(path.lastIndexOf("/") + 1)
if (knownLicenses.has(author)) {
const license = knownLicenses.get(author)
license.path = path
return license
}
if (author == "s") {
return null
}
if (author == "Q" || author == "q" || author == "") {
throw "Quitting now!"
}
let authors = author.split(";")
if (author.toLowerCase() == "none") {
authors = []
}
return {
authors: author.split(";"),
path: path,
license: prompt("What is the license for artwork " + path + "? > "),
sources: prompt("Where was this artwork found? > ").split(";"),
}
}
cleanLicenseInfo(licensePaths, licenseInfos)
createFullLicenseOverview(licensePaths)
createLicenseInfoFor(path): void {
const li = this.promptLicenseFor(path)
if (li == null) {
return
}
writeFileSync(path + ".license_info.json", JSON.stringify(li, null, " "))
}
cleanLicenseInfo(allPaths: string[], allLicenseInfos: SmallLicense[]) {
// Read the license info file from the generated assets, creates a compiled license info in every directory
// Note: this removes all the old license infos
for (const licensePath of allPaths) {
unlinkSync(licensePath)
}
const perDirectory = new Map<string, SmallLicense[]>()
for (const license of allLicenseInfos) {
const p = license.path
const dir = p.substring(0, p.lastIndexOf("/"))
license.path = p.substring(dir.length + 1)
if (!perDirectory.has(dir)) {
perDirectory.set(dir, [])
}
const cloned: SmallLicense = {
// We make a clone to force the order of the keys
path: license.path,
license: license.license,
authors: license.authors,
sources: license.sources,
}
perDirectory.get(dir).push(cloned)
}
perDirectory.forEach((licenses, dir) => {
for (let i = licenses.length - 1; i >= 0; i--) {
const license = licenses[i]
const path = dir + "/" + license.path
if (!existsSync(path)) {
console.log(
"Found license for now missing file: ",
path,
" - removing this license"
)
licenses.splice(i, 1)
}
}
licenses.sort((a, b) => (a.path < b.path ? -1 : 1))
writeFileSync(dir + "/license_info.json", JSON.stringify(licenses, null, 2))
})
}
queryMissingLicenses(missingLicenses: string[]) {
process.on("SIGINT", function () {
console.log("Aborting... Bye!")
process.exit()
})
let i = 1
for (const missingLicens of missingLicenses) {
console.log(i + " / " + missingLicenses.length)
i++
if (i < missingLicenses.length - 5) {
// continue
}
this.createLicenseInfoFor(missingLicens)
}
console.log("You're through!")
}
/**
* Creates the humongous license_info in the generated assets, containing all licenses with a path relative to the root
* @param licensePaths
*/
createFullLicenseOverview(licensePaths: string[]) {
const allLicenses: SmallLicense[] = []
for (const licensePath of licensePaths) {
if (!existsSync(licensePath)) {
continue
}
const licenses = <SmallLicense[]>(
JSON.parse(readFileSync(licensePath, { encoding: "utf8" }))
)
for (const license of licenses) {
this.validateLicenseInfo(license)
const dir = licensePath.substring(
0,
licensePath.length - "license_info.json".length
)
license.path = dir + license.path
allLicenses.push(license)
}
}
writeFileSync(
"./assets/generated/license_info.json",
JSON.stringify(allLicenses, null, " ")
)
}
async main(args: string[]) {
console.log("Checking and compiling license info")
if (!existsSync("./assets/generated")) {
mkdirSync("./assets/generated")
}
let contents = ScriptUtils.readDirRecSync("./assets")
.filter((p) => !p.startsWith("./assets/templates/"))
.filter((entry) => entry.indexOf("./assets/generated") != 0)
let licensePaths = contents.filter((entry) => entry.indexOf("license_info.json") >= 0)
let licenseInfos = this.generateLicenseInfos(licensePaths)
const artwork = contents.filter(
(pth) => pth.match(/(.svg|.png|.jpg|.ttf|.otf|.woff)$/i) != null
)
const missingLicenses = this.missingLicenseInfos(licenseInfos, artwork)
if (args.indexOf("--prompt") >= 0 || args.indexOf("--query") >= 0) {
this.queryMissingLicenses(missingLicenses)
return this.main([])
}
const invalidLicenses = licenseInfos
.filter((l) => (l.license ?? "") === "")
.map((l) => `License for artwork ${l.path} is empty string or undefined`)
let invalid = 0
for (const licenseInfo of licenseInfos) {
const isTrivial =
licenseInfo.license
.split(";")
.map((l) => l.trim().toLowerCase())
.indexOf("trivial") >= 0
if (licenseInfo.sources.length + licenseInfo.authors.length == 0 && !isTrivial) {
invalid++
invalidLicenses.push(
"Invalid license: No sources nor authors given in the license for " +
JSON.stringify(licenseInfo)
)
continue
}
for (const source of licenseInfo.sources) {
if (source == "") {
invalidLicenses.push(
"Invalid license: empty string in " + JSON.stringify(licenseInfo)
)
}
try {
new URL(source)
} catch {
invalidLicenses.push("Not a valid URL: " + source)
}
}
}
if (missingLicenses.length > 0 || invalidLicenses.length) {
const msg = `There are ${missingLicenses.length} licenses missing and ${invalidLicenses.length} invalid licenses.`
console.log(missingLicenses.concat(invalidLicenses).join("\n"))
console.error(msg)
if (args.indexOf("--no-fail") < 0) {
throw msg
}
}
this.cleanLicenseInfo(licensePaths, licenseInfos)
this.createFullLicenseOverview(licensePaths)
}
}
main(process.argv.slice(2))
new GenerateLicenseInfo().run()

4
scripts/generateStats.ts
View file

@ -1,4 +1,4 @@
import * as known_layers from "../assets/generated/known_layers.json"
import known_layers from "../assets/generated/known_layers.json"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import { TagUtils } from "../Logic/Tags/TagUtils"
import { Utils } from "../Utils"
@ -10,7 +10,7 @@ import Constants from "../Models/Constants"
async function main(includeTags = true) {
ScriptUtils.fixUtils()
const layers: LayerConfigJson[] = (known_layers["default"] ?? known_layers).layers
const layers = <LayerConfigJson[]>known_layers.layers
const keysAndTags = new Map<string, Set<string>>()

2
scripts/generateTaginfoProjectFiles.ts
View file

@ -205,7 +205,7 @@ function main() {
const files = []
for (const layout of AllKnownLayouts.layoutsList) {
for (const layout of AllKnownLayouts.allKnownLayouts.values()) {
if (layout.hideFromOverview) {
continue
}

5
scripts/generateTranslations.ts
View file

@ -710,10 +710,7 @@ const l3 = generateTranslationsObjectFrom(
const usedLanguages: string[] = Utils.Dedup(l1.concat(l2).concat(l3)).filter((v) => v !== "*")
usedLanguages.sort()
fs.writeFileSync(
"./assets/generated/used_languages.json",
JSON.stringify({ languages: usedLanguages })
)
fs.writeFileSync("./assets/used_languages.json", JSON.stringify({ languages: usedLanguages }))
if (!themeOverwritesWeblate) {
// Generates the core translations

3
scripts/printVersion.ts
View file

@ -1,3 +0,0 @@
import Constants from "../Models/Constants"
console.log("git tag -a", Constants.vNumber, `-m "Deployed on ${new Date()}"`)

2
scripts/removeTranslationString.ts
View file

@ -13,7 +13,7 @@ async function main(args: string[]) {
console.log("Removing translation string ", path, "from the general translations")
const files = ScriptUtils.readDirRecSync("./langs", 1).filter((f) => f.endsWith(".json"))
for (const file of files) {
const json = JSON.parse(fs.readFileSync(file, "UTF-8"))
const json = JSON.parse(fs.readFileSync(file, { encoding: "utf-8" }))
Utils.WalkPath(path, json, (_) => undefined)
fs.writeFileSync(file, JSON.stringify(json, null, " ") + "\n")
}

4098
scripts/schools/Onderwijsinstellingen.csv
View file

File diff suppressed because it is too large Load diff

3
scripts/schools/README.md
View file

@ -1,3 +0,0 @@
# Little scripts to parse Belgian school data

364
scripts/schools/amendSchoolData.ts
View file

@ -1,364 +0,0 @@
import { parse } from "csv-parse/sync"
import { readFileSync, writeFileSync } from "fs"
import { Utils } from "../../Utils"
import { GeoJSONObject, geometry } from "@turf/turf"
function parseAndClean(filename: string): Record<any, string>[] {
const csvOptions = {
columns: true,
skip_empty_lines: true,
trim: true,
}
const records: Record<any, string>[] = parse(readFileSync(filename), csvOptions)
return records.map((r) => {
for (const key of Object.keys(r)) {
if (r[key].endsWith("niet van toepassing")) {
delete r[key]
}
}
return r
})
}
const structuren = {
"Voltijds Gewoon Secundair Onderwijs": "secondary",
"Gewoon Lager Onderwijs": "primary",
"Gewoon Kleuteronderwijs": "kindergarten",
Kleuteronderwijs: "kindergarten",
"Buitengewoon Lager Onderwijs": "primary",
"Buitengewoon Secundair Onderwijs": "secondary",
"Buitengewoon Kleuteronderwijs": "kindergarten",
"Deeltijds Beroepssecundair Onderwijs": "secondary",
}
const degreesMapping = {
"Derde graad": "upper_secondary",
"Tweede graad": "middle_secondary",
"Eerste graad": "lower_secondary",
}
const classificationOrder = [
"kindergarten",
"primary",
"secondary",
"lower_secondary",
"middle_secondary",
"upper_secondary",
]
const stelselsMapping = {
"Beide stelsels": "linear_courses;modular_courses",
"Lineair stelsel": "linear_courses",
"Modulair stelsel": "modular_courses",
}
const rmKeys = [
"schoolnummer",
"instellingstype",
"adres",
"begindatum",
"hoofdzetel",
"huisnummer",
"kbo-nummer",
"beheerder(s)",
"bestuur",
"clb",
"ingerichte hoofdstructuren",
"busnummer",
"crab-code",
"crab-huisnr",
"einddatum",
"fax",
"gemeente",
"intern_vplnummer",
"kbo_nummer",
"lx",
"ly",
"niscode",
"onderwijsniveau",
"onderwijsvorm",
"scholengemeenschap",
"postcode",
"provincie",
"provinciecode",
"soort instelling",
"status erkenning",
"straat",
"VWO-vestigingsplaatscode",
"taalstelsel",
"net",
]
const rename = {
"e-mail": "email",
naam: "name",
telefoon: "phone",
}
function fuzzIdenticals(features: { geometry: { coordinates: [number, number] } }[]) {
var seen = new Set<string>()
for (const feature of features) {
var coors = feature.geometry.coordinates
let k = coors[0] + "," + coors[1]
while (seen.has(k)) {
coors[0] += 0.00025
k = coors[0] + "," + coors[1]
}
seen.add(k)
}
}
/**
* Sorts classifications in order
*
* sortClassifications(["primary","secondary","kindergarten"] // => ["kindergarten", "primary", "secondary"]
*/
function sortClassifications(classification: string[]) {
return classification.sort(
(a, b) => classificationOrder.indexOf(a) - classificationOrder.indexOf(b)
)
}
function main() {
console.log("Parsing schools...")
const aantallen = "/home/pietervdvn/Downloads/Scholen/aantallen.csv"
const perSchool = "/home/pietervdvn/Downloads/Scholen/perschool.csv"
const schoolfields = [
"schoolnummer",
"intern_vplnummer",
"net",
"naam",
"hoofdzetel",
"adres",
"straat",
"huisnummer",
"busnummer",
"postcode",
"gemeente",
"niscode",
"provinciecode",
"provincie",
"VWO-vestigingsplaatscode",
"crab-code",
"crab-huisnr",
"lx",
"ly",
"kbo-nummer",
"telefoon",
"fax",
"e-mail",
"website",
"beheerder(s)",
"soort instelling",
"onderwijsniveau",
"instellingstype",
"begindatum",
"einddatum",
"status erkenning",
"clb",
"bestuur",
"scholengemeenschap",
"taalstelsel",
"ingerichte hoofdstructuren",
] as const
const schoolGeojson: {
features: {
properties: Record<typeof schoolfields[number], string>
geometry: {
type: "Point"
coordinates: [number, number]
}
}[]
} = JSON.parse(readFileSync("scripts/schools/scholen.geojson", "utf8"))
fuzzIdenticals(schoolGeojson.features)
const aantallenFields = [
"schooljaar",
"nr koepel",
"koepel",
"instellingscode",
"intern volgnr vpl",
"volgnr vpl",
"naam instelling",
"GON-school",
"GOK-school",
"instellingsnummer scholengemeenschap",
"scholengemeenschap",
"code schoolbestuur",
"schoolbestuur",
"type vestigingsplaats",
"fusiegemeente hoofdvestigingsplaats",
"straatnaam vestigingsplaats",
"huisnr vestigingsplaats",
"bus vestigingsplaats",
"postcode vestigingsplaats",
"deelgemeente vestigingsplaats",
"fusiegemeente vestigingsplaats",
"hoofdstructuur (code)",
"hoofdstructuur",
"administratieve groep (code)",
"administratieve groep",
"graad lager onderwijs",
"pedagogische methode",
"graad secundair onderwijs",
"leerjaar",
"A of B-stroom",
"basisopties",
"beroepenveld",
"onderwijsvorm",
"studiegebied",
"studierichting",
"stelsel",
"okan cluster",
"type buitengewoon onderwijs",
"opleidingsvorm (code)",
"opleidingsvorm",
"fase",
"opleidingen",
"geslacht",
"aantal inschrijvingen",
] as const
const aantallenParsed: Record<typeof aantallenFields[number], string>[] =
parseAndClean(aantallen)
const perschoolFields = [
"schooljaar",
"nr koepel",
"koepel",
"instellingscode",
"naam instelling",
"straatnaam",
"huisnr",
"bus",
"postcode",
"deelgemeente",
"fusiegemeente",
"aantal inschrijvingen",
] as const
const perschoolParsed: Record<typeof perschoolFields[number], string>[] =
parseAndClean(perSchool)
schoolGeojson.features = schoolGeojson.features
.filter((sch) => sch.properties.lx != "0" && sch.properties.ly != "0")
.filter((sch) => sch.properties.instellingstype !== "Universiteit")
const c = schoolGeojson.features.length
console.log("Got ", schoolGeojson.features.length, "items after filtering")
let i = 0
let lastWrite = 0
for (const feature of schoolGeojson.features) {
i++
const now = Date.now()
if (now - lastWrite > 1000) {
lastWrite = now
console.log("Processing " + i + "/" + c)
}
const props = feature.properties
const aantallen = aantallenParsed.filter((i) => i.instellingscode == props.schoolnummer)
if (aantallen.length > 0) {
const fetch = (key: typeof aantallenFields[number]) =>
Utils.NoNull(Utils.Dedup(aantallen.map((x) => x[key])))
props["onderwijsvorm"] = fetch("onderwijsvorm").join(";")
/*
const gonSchool = aantallen.some(x => x["GON-school"] === "GON-school")
const gokSchool = aantallen.some(x => x["GOK-school"] === "GON-school")
const onderwijsvorm = fetch("onderwijsvorm")
const koepel = fetch("koepel")
const stelsel = fetch("stelsel").join(";")
const scholengemeenschap = fetch("scholengemeenschap")
*/
const hoofdstructuur = fetch("hoofdstructuur")
let specialEducation = false
let classification = hoofdstructuur.map((s) => {
const v = structuren[s]
if (s.startsWith("Buitengewoon")) {
specialEducation = true
}
if (v === undefined) {
console.error("Type not found: " + s)
return ""
}
return v
})
const graden = fetch("graad secundair onderwijs")
if (classification[0] === "secondary") {
if (graden.length !== 3) {
classification = graden.map((degree) => degreesMapping[degree])
}
}
sortClassifications(classification)
props["school"] = Utils.Dedup(classification).join("; ")
// props["koepel"] = koepel.join(";")
// props["scholengemeenschap"] = scholengemeenschap.join(";")
// props["stelsel"] = stelselsMapping[stelsel]
if (specialEducation) {
props["school:for"] = "special_education"
}
if (props.taalstelsel === "Nederlandstalig") {
props["language:nl"] = "yes"
}
if (props.instellingstype === "Instelling voor deeltijds kunstonderwijs") {
props["amenity"] = "college"
props["school:subject"] = "art"
}
}
const schoolinfo = perschoolParsed.filter((i) => i.instellingscode == props.schoolnummer)
if (schoolinfo.length == 0) {
// pass
} else if (schoolinfo.length == 1) {
props["capacity"] = schoolinfo[0]["aantal inschrijvingen"]
.split(";")
.map((i) => Number(i))
.reduce((sum, i) => sum + i, 0)
} else {
throw "Multiple schoolinfo's found for " + props.schoolnummer
}
//props["source:ref"] = props.schoolnummer
props["amenity"] = "school"
if (props["school"] === "kindergarten") {
props["amenity"] = "kindergarten"
props["isced:2011:level"] = "early_education"
delete props["school"]
}
for (const renameKey in rename) {
const into = rename[renameKey]
if (props[renameKey] !== undefined) {
props[into] = props[renameKey]
delete props[renameKey]
}
}
for (const rmKey of rmKeys) {
delete props[rmKey]
}
}
//schoolGeojson.features = schoolGeojson.features.filter(f => f.properties["capacity"] !== undefined)
/*schoolGeojson.features.forEach((f, i) => {
f.properties["id"] = "school/"+i
})*/
schoolGeojson.features = schoolGeojson.features.filter(
(f) => f.properties["amenity"] === "kindergarten"
)
writeFileSync("scripts/schools/amended_schools.geojson", JSON.stringify(schoolGeojson), "utf8")
console.log("Done")
}
if (!process.argv[1].endsWith("mocha")) {
main()
}

1
scripts/schools/amended_schools.geojson
View file

File diff suppressed because one or more lines are too long

4098
scripts/schools/scholen.csv
View file

File diff suppressed because it is too large Load diff

1
scripts/schools/scholen.geojson
View file

File diff suppressed because one or more lines are too long

32
scripts/slice.ts
View file

@ -134,20 +134,40 @@ class Slice extends Script {
}
delete f.bbox
}
const maxNumberOfTiles = Math.pow(2, zoomlevel) * Math.pow(2, zoomlevel)
let handled = 0
TiledFeatureSource.createHierarchy(StaticFeatureSource.fromGeojson(allFeatures), {
minZoomLevel: zoomlevel,
maxZoomLevel: zoomlevel,
maxFeatureCount: Number.MAX_VALUE,
registerTile: (tile) => {
handled = handled + 1
const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson`
const box = BBox.fromTile(tile.z, tile.x, tile.y)
let features = tile.features.data.map((ff) => ff.feature)
if (doSlice) {
features = Utils.NoNull(
features.map((f) => {
const bbox = box.asGeoJson({})
const properties = {
...f.properties,
id:
(f.properties?.id ?? "") +
"_" +
tile.z +
"_" +
tile.x +
"_" +
tile.y,
}
if (GeoOperations.completelyWithin(bbox, f)) {
bbox.properties = properties
return bbox
}
const intersection = GeoOperations.intersect(f, box.asGeoJson({}))
if (intersection) {
intersection.properties = f.properties
intersection.properties = properties
}
return intersection
})
@ -156,6 +176,15 @@ class Slice extends Script {
features.forEach((f) => {
delete f.bbox
})
if (features.length === 0) {
ScriptUtils.erasableLog(
handled + "/" + maxNumberOfTiles,
"Not writing ",
path,
": no features"
)
return
}
fs.writeFileSync(
path,
JSON.stringify(
@ -168,6 +197,7 @@ class Slice extends Script {
)
)
ScriptUtils.erasableLog(
handled + "/" + maxNumberOfTiles,
"Written ",
path,
"which has ",

2
scripts/thieves/readIdPresets.ts
View file

@ -3,7 +3,7 @@
*/
import ScriptUtils from "../ScriptUtils"
import { existsSync, readFileSync, writeFileSync } from "fs"
import * as known_languages from "../../assets/language_native.json"
import known_languages from "../../assets/language_native.json"
import { LayerConfigJson } from "../../Models/ThemeConfig/Json/LayerConfigJson"
import { MappingConfigJson } from "../../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import SmallLicense from "../../Models/smallLicense"