forked from MapComplete/MapComplete
reset to previous commit
This commit is contained in:
karelleketers
parent
d5b614fc44
commit
196d40084d
90 changed files with 4953 additions and 1922 deletions
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Generates a collection of geojson files based on an overpass query for a given theme
|
||||
*/
|
||||
import {TileRange, Utils} from "../Utils";
|
||||
import {Utils} from "../Utils";
|
||||
|
||||
Utils.runningFromConsole = true
|
||||
import {Overpass} from "../Logic/Osm/Overpass";
|
||||
|
|
@ -17,6 +17,8 @@ import MetaTagging from "../Logic/MetaTagging";
|
|||
import LayerConfig from "../Customizations/JSON/LayerConfig";
|
||||
import {GeoOperations} from "../Logic/GeoOperations";
|
||||
import {UIEventSource} from "../Logic/UIEventSource";
|
||||
import * as fs from "fs";
|
||||
import {TileRange} from "../Models/TileRange";
|
||||
|
||||
|
||||
function createOverpassObject(theme: LayoutConfig) {
|
||||
|
|
@ -139,7 +141,7 @@ async function downloadExtraData(theme: LayoutConfig)/* : any[] */ {
|
|||
return allFeatures;
|
||||
}
|
||||
|
||||
async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig, extraFeatures: any[]) {
|
||||
function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig, extraFeatures: any[]) {
|
||||
let processed = 0;
|
||||
const layerIndex = theme.LayerIndex();
|
||||
for (let x = r.xstart; x <= r.xend; x++) {
|
||||
|
|
@ -211,8 +213,9 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig,
|
|||
}
|
||||
}
|
||||
|
||||
async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfig) {
|
||||
function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfig) {
|
||||
const z = r.zoomlevel;
|
||||
const generated = {} // layer --> x --> y[]
|
||||
for (let x = r.xstart; x <= r.xend; x++) {
|
||||
for (let y = r.ystart; y <= r.yend; y++) {
|
||||
const file = readFileSync(geoJsonName(targetdir + ".unfiltered", x, y, z), "UTF8")
|
||||
|
|
@ -227,10 +230,8 @@ async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfi
|
|||
.filter(f => f._matching_layer_id === layer.id)
|
||||
.filter(f => {
|
||||
const isShown = layer.isShown.GetRenderValue(f.properties).txt
|
||||
if (isShown === "no") {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
return isShown !== "no";
|
||||
|
||||
})
|
||||
const new_path = geoJsonName(targetdir + "_" + layer.id, x, y, z);
|
||||
console.log(new_path, " has ", geojson.features.length, " features after filtering (dropped ", oldLength - geojson.features.length, ")")
|
||||
|
|
@ -239,18 +240,66 @@ async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfi
|
|||
continue;
|
||||
}
|
||||
writeFileSync(new_path, JSON.stringify(geojson, null, " "))
|
||||
|
||||
if (generated[layer.id] === undefined) {
|
||||
generated[layer.id] = {}
|
||||
}
|
||||
if (generated[layer.id][x] === undefined) {
|
||||
generated[layer.id][x] = []
|
||||
}
|
||||
generated[layer.id][x].push(y)
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
for (const layer of theme.layers) {
|
||||
const id = layer.id
|
||||
const loaded = generated[id]
|
||||
if(loaded === undefined){
|
||||
console.log("No features loaded for layer ",id)
|
||||
continue;
|
||||
}
|
||||
writeFileSync(targetdir + "_" + id + "_overview.json", JSON.stringify(loaded))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
async function createOverview(targetdir: string, r: TileRange, z: number, layername: string) {
|
||||
const allFeatures = []
|
||||
for (let x = r.xstart; x <= r.xend; x++) {
|
||||
for (let y = r.ystart; y <= r.yend; y++) {
|
||||
const read_path = geoJsonName(targetdir + "_" + layername, x, y, z);
|
||||
if (!fs.existsSync(read_path)) {
|
||||
continue;
|
||||
}
|
||||
const features = JSON.parse(fs.readFileSync(read_path, "UTF-8")).features
|
||||
const pointsOnly = features.map(f => {
|
||||
|
||||
f.properties["_last_edit:timestamp"] = "1970-01-01"
|
||||
|
||||
if (f.geometry.type === "Point") {
|
||||
return f
|
||||
} else {
|
||||
return GeoOperations.centerpoint(f)
|
||||
}
|
||||
|
||||
})
|
||||
allFeatures.push(...pointsOnly)
|
||||
}
|
||||
}
|
||||
|
||||
const geojson = {
|
||||
"type": "FeatureCollection",
|
||||
"features": allFeatures
|
||||
}
|
||||
writeFileSync(targetdir + "_" + layername + "_points.geojson", JSON.stringify(geojson, null, " "))
|
||||
}
|
||||
|
||||
async function main(args: string[]) {
|
||||
|
||||
if (args.length == 0) {
|
||||
console.error("Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1")
|
||||
console.error("Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1 [--generate-point-overview layer-name]")
|
||||
return;
|
||||
}
|
||||
const themeName = args[0]
|
||||
|
|
@ -285,8 +334,18 @@ async function main(args: string[]) {
|
|||
} while (failed > 0)
|
||||
|
||||
const extraFeatures = await downloadExtraData(theme);
|
||||
await postProcess(targetdir, tileRange, theme, extraFeatures)
|
||||
await splitPerLayer(targetdir, tileRange, theme)
|
||||
postProcess(targetdir, tileRange, theme, extraFeatures)
|
||||
splitPerLayer(targetdir, tileRange, theme)
|
||||
|
||||
if (args[7] === "--generate-point-overview") {
|
||||
const targetLayers = args[8].split(",")
|
||||
for (const targetLayer of targetLayers) {
|
||||
if (!theme.layers.some(l => l.id === targetLayer)) {
|
||||
throw "Target layer " + targetLayer + " not found, did you mistype the name? Found layers are: " + theme.layers.map(l => l.id).join(",")
|
||||
}
|
||||
createOverview(targetdir, tileRange, zoomlevel, targetLayer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import LayerConfig from "../Customizations/JSON/LayerConfig";
|
|||
import * as licenses from "../assets/generated/license_info.json"
|
||||
import LayoutConfig from "../Customizations/JSON/LayoutConfig";
|
||||
import {LayerConfigJson} from "../Customizations/JSON/LayerConfigJson";
|
||||
import {Translation} from "../UI/i18n/Translation";
|
||||
import {LayoutConfigJson} from "../Customizations/JSON/LayoutConfigJson";
|
||||
import AllKnownLayers from "../Customizations/AllKnownLayers";
|
||||
|
||||
|
|
@ -77,63 +76,6 @@ class LayerOverviewUtils {
|
|||
return errorCount
|
||||
}
|
||||
|
||||
validateTranslationCompletenessOfObject(object: any, expectedLanguages: string[], context: string) {
|
||||
const missingTranlations = []
|
||||
const translations: { tr: Translation, context: string }[] = [];
|
||||
const queue: { object: any, context: string }[] = [{object: object, context: context}]
|
||||
|
||||
while (queue.length > 0) {
|
||||
const item = queue.pop();
|
||||
const o = item.object
|
||||
for (const key in o) {
|
||||
const v = o[key];
|
||||
if (v === undefined) {
|
||||
continue;
|
||||
}
|
||||
if (v instanceof Translation || v?.translations !== undefined) {
|
||||
translations.push({tr: v, context: item.context});
|
||||
} else if (
|
||||
["string", "function", "boolean", "number"].indexOf(typeof (v)) < 0) {
|
||||
queue.push({object: v, context: item.context + "." + key})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const missing = {}
|
||||
const present = {}
|
||||
for (const ln of expectedLanguages) {
|
||||
missing[ln] = 0;
|
||||
present[ln] = 0;
|
||||
for (const translation of translations) {
|
||||
if (translation.tr.translations["*"] !== undefined) {
|
||||
continue;
|
||||
}
|
||||
const txt = translation.tr.translations[ln];
|
||||
const isMissing = txt === undefined || txt === "" || txt.toLowerCase().indexOf("todo") >= 0;
|
||||
if (isMissing) {
|
||||
missingTranlations.push(`${translation.context},${ln},${translation.tr.txt}`)
|
||||
missing[ln]++
|
||||
} else {
|
||||
present[ln]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let message = `Translation completeness for ${context}`
|
||||
let isComplete = true;
|
||||
for (const ln of expectedLanguages) {
|
||||
const amiss = missing[ln];
|
||||
const ok = present[ln];
|
||||
const total = amiss + ok;
|
||||
message += ` ${ln}: ${ok}/${total}`
|
||||
if (ok !== total) {
|
||||
isComplete = false;
|
||||
}
|
||||
}
|
||||
return missingTranlations
|
||||
|
||||
}
|
||||
|
||||
main(args: string[]) {
|
||||
|
||||
const lt = this.loadThemesAndLayers();
|
||||
|
|
@ -160,7 +102,6 @@ class LayerOverviewUtils {
|
|||
}
|
||||
|
||||
let themeErrorCount = []
|
||||
let missingTranslations = []
|
||||
for (const themeFile of themeFiles) {
|
||||
if (typeof themeFile.language === "string") {
|
||||
themeErrorCount.push("The theme " + themeFile.id + " has a string as language. Please use a list of strings")
|
||||
|
|
@ -169,10 +110,6 @@ class LayerOverviewUtils {
|
|||
if (typeof layer === "string") {
|
||||
if (!knownLayerIds.has(layer)) {
|
||||
themeErrorCount.push(`Unknown layer id: ${layer} in theme ${themeFile.id}`)
|
||||
} else {
|
||||
const layerConfig = knownLayerIds.get(layer);
|
||||
missingTranslations.push(...this.validateTranslationCompletenessOfObject(layerConfig, themeFile.language, "Layer " + layer))
|
||||
|
||||
}
|
||||
} else if (layer.builtin !== undefined) {
|
||||
let names = layer.builtin;
|
||||
|
|
@ -197,7 +134,6 @@ class LayerOverviewUtils {
|
|||
.filter(l => typeof l != "string") // We remove all the builtin layer references as they don't work with ts-node for some weird reason
|
||||
.filter(l => l.builtin === undefined)
|
||||
|
||||
missingTranslations.push(...this.validateTranslationCompletenessOfObject(themeFile, themeFile.language, "Theme " + themeFile.id))
|
||||
|
||||
try {
|
||||
const theme = new LayoutConfig(themeFile, true, "test")
|
||||
|
|
@ -209,11 +145,6 @@ class LayerOverviewUtils {
|
|||
}
|
||||
}
|
||||
|
||||
if (missingTranslations.length > 0) {
|
||||
console.log(missingTranslations.length, "missing translations")
|
||||
writeFileSync("missing_translations.txt", missingTranslations.join("\n"))
|
||||
}
|
||||
|
||||
if (layerErrorCount.length + themeErrorCount.length == 0) {
|
||||
console.log("All good!")
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue