diff --git a/scripts/generateCache.ts b/scripts/generateCache.ts index 0134c941f..9980c1959 100644 --- a/scripts/generateCache.ts +++ b/scripts/generateCache.ts @@ -24,6 +24,7 @@ import {GeoOperations} from "../Logic/GeoOperations"; import SimpleMetaTaggers from "../Logic/SimpleMetaTagger"; import FilteringFeatureSource from "../Logic/FeatureSource/Sources/FilteringFeatureSource"; import Loc from "../Models/Loc"; + ScriptUtils.fixUtils() function createOverpassObject(theme: LayoutConfig, relationTracker: RelationsTracker, backend: string) { @@ -166,7 +167,7 @@ function loadAllTiles(targetdir: string, r: TileRange, theme: LayoutConfig, extr // Create and save the geojson file - which is the main chunk of the data const geojson = OsmToGeoJson.default(rawOsm); - console.log(" which as",geojson.features.length, "features") + console.log(" which as", geojson.features.length, "features") allFeatures.push(...geojson.features) } @@ -180,23 +181,24 @@ function loadAllTiles(targetdir: string, r: TileRange, theme: LayoutConfig, extr function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relationsTracker: RelationsTracker, targetdir: string, pointsOnlyLayers: string[]) { const skippedLayers = new Set() - const indexedFeatures : Map = new Map() + const indexedFeatures: Map = new Map() let indexisBuilt = false; - function buildIndex(){ + + function buildIndex() { for (const ff of allFeatures.features.data) { const f = ff.feature indexedFeatures.set(f.properties.id, f) } indexisBuilt = true; } - - function getFeatureById(id){ - if(!indexisBuilt){ + + function getFeatureById(id) { + if (!indexisBuilt) { buildIndex() } return indexedFeatures.get(id) } - + async function handleLayer(source: FeatureSourceForLayer) { const layer = source.layer.layerDef; const targetZoomLevel = layer.source.geojsonZoomLevel ?? 0 @@ -225,11 +227,10 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations includeDates: false, includeNonDates: true }); - - + while (SimpleMetaTaggers.country.runningTasks.size > 0) { - console.log("Still waiting for ", SimpleMetaTaggers.country.runningTasks.size," features which don't have a country yet") + console.log("Still waiting for ", SimpleMetaTaggers.country.runningTasks.size, " features which don't have a country yet") await ScriptUtils.sleep(1) } @@ -246,18 +247,18 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations if (tile.features.data.length === 0) { return } - + const filteredTile = new FilteringFeatureSource({ - locationControl: new UIEventSource(undefined), - allElements: undefined, - selectedElement: new UIEventSource(undefined) - }, + locationControl: new UIEventSource(undefined), + allElements: undefined, + selectedElement: new UIEventSource(undefined) + }, tileIndex, tile, new UIEventSource(undefined) - ) + ) - console.log("Tile "+layer.id+"."+tileIndex+" contains "+filteredTile.features.data.length+" features after filtering ("+tile.features.data.length+") features before") + console.log("Tile " + layer.id + "." + tileIndex + " contains " + filteredTile.features.data.length + " features after filtering (" + tile.features.data.length + ") features before") if (filteredTile.features.data.length === 0) { return } @@ -266,22 +267,22 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations for (const feature of filteredTile.features.data) { // Some cleanup delete feature.feature["bbox"] - - if(tile.layer.layerDef.calculatedTags !== undefined){ - - // Evaluate all the calculated tags strictly - const calculatedTagKeys = tile.layer.layerDef.calculatedTags.map(ct => ct[0]) - featureCount++ - for (const calculatedTagKey of calculatedTagKeys) { - const strict = feature.feature.properties[calculatedTagKey] - feature.feature.properties[calculatedTagKey] =strict - strictlyCalculated ++; - if(strictlyCalculated % 100 === 0){ - console.log("Strictly calculated ", strictlyCalculated, "values for tile",tileIndex,": now at ", featureCount,"/",filteredTile.features.data.length, "examle value: ", strict) + + if (tile.layer.layerDef.calculatedTags !== undefined) { + + // Evaluate all the calculated tags strictly + const calculatedTagKeys = tile.layer.layerDef.calculatedTags.map(ct => ct[0]) + featureCount++ + for (const calculatedTagKey of calculatedTagKeys) { + const strict = feature.feature.properties[calculatedTagKey] + feature.feature.properties[calculatedTagKey] = strict + strictlyCalculated++; + if (strictlyCalculated % 100 === 0) { + console.log("Strictly calculated ", strictlyCalculated, "values for tile", tileIndex, ": now at ", featureCount, "/", filteredTile.features.data.length, "examle value: ", strict) + } } } - } - + } // Lets save this tile! const [z, x, y] = Tiles.tile_from_index(tileIndex) @@ -293,7 +294,7 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations type: "FeatureCollection", features: filteredTile.features.data.map(f => f.feature) }, null, " ")) - console.log("Written tile", targetPath,"with", filteredTile.features.data.length) + console.log("Written tile", targetPath, "with", filteredTile.features.data.length) } }) @@ -315,16 +316,16 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations if (pointsOnlyLayers.indexOf(layer.id) >= 0) { const filtered = new FilteringFeatureSource({ - locationControl: new UIEventSource(undefined), + locationControl: new UIEventSource(undefined), allElements: undefined, selectedElement: new UIEventSource(undefined) }, - Tiles.tile_index(0,0,0), + Tiles.tile_index(0, 0, 0), source, new UIEventSource(undefined) ) const features = filtered.features.data.map(f => f.feature) - + const points = features.map(feature => GeoOperations.centerpoint(feature)) console.log("Writing points overview for ", layerId) const targetPath = targetdir + "_" + layerId + "_points.geojson" @@ -375,8 +376,6 @@ async function main(args: string[]) { const lat1 = Number(args[5]) const lon1 = Number(args[6]) - - const tileRange = Tiles.TileRangeBetween(zoomlevel, lat0, lon0, lat1, lon1) @@ -397,29 +396,29 @@ async function main(args: string[]) { let generatePointLayersFor = [] if (args[7] == "--generate-point-overview") { - if(args[8] === undefined){ + if (args[8] === undefined) { throw "--generate-point-overview needs a list of layers to generate the overview for (or * for all)" - }else if (args[8] === '*'){ + } else if (args[8] === '*') { generatePointLayersFor = theme.layers.map(l => l.id) - }else{ + } else { generatePointLayersFor = args[8].split(",") } console.log("Also generating a point overview for layers ", generatePointLayersFor.join(",")) } { - - const index = args.indexOf("--force-zoom-level") - if(index >= 0){ - const forcedZoomLevel = Number(args[index + 1]) - for (const layer of theme.layers) { - layer.source.geojsonSource = "https://127.0.0.1/cache_{layer}_{z}_{x}_{y}.geojson" - layer.source.isOsmCacheLayer = true - layer.source.geojsonZoomLevel = forcedZoomLevel + + const index = args.indexOf("--force-zoom-level") + if (index >= 0) { + const forcedZoomLevel = Number(args[index + 1]) + for (const layer of theme.layers) { + layer.source.geojsonSource = "https://127.0.0.1/cache_{layer}_{z}_{x}_{y}.geojson" + layer.source.isOsmCacheLayer = true + layer.source.geojsonZoomLevel = forcedZoomLevel + } } } - } - - + + const relationTracker = new RelationsTracker() let failed = 0; diff --git a/scripts/generateLayerOverview.ts b/scripts/generateLayerOverview.ts index 4b61c41b2..1ddea2e7e 100644 --- a/scripts/generateLayerOverview.ts +++ b/scripts/generateLayerOverview.ts @@ -6,7 +6,8 @@ import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson"; import Constants from "../Models/Constants"; import { DesugaringContext, - PrepareLayer, PrepareTheme, + PrepareLayer, + PrepareTheme, ValidateLayer, ValidateThemeAndLayers } from "../Models/ThemeConfig/Conversion/LegacyJsonConvert"; @@ -80,7 +81,7 @@ class LayerOverviewUtils { dict.set(key, questions[key]) } for (const key in icons["default"]) { - if(typeof icons[key] !== "object"){ + if (typeof icons[key] !== "object") { continue } icons[key].id = key; @@ -94,6 +95,32 @@ class LayerOverviewUtils { return dict; } + main(_: string[]) { + + const licensePaths = new Set() + for (const i in licenses) { + licensePaths.add(licenses[i].path) + } + + const sharedLayers = this.buildLayerIndex(licensePaths); + const sharedThemes = this.buildThemeIndex(licensePaths, sharedLayers) + + writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({ + "layers": Array.from(sharedLayers.values()), + "themes": Array.from(sharedThemes.values()) + })) + + writeFileSync("./assets/generated/known_layers.json", JSON.stringify(Array.from(sharedLayers.values()))) + + writeFileSync('./assets/themes/mapcomplete-changes/icons-mapping.txt', JSON.stringify( + Array.from(sharedThemes.values()).map(th => ({ + if: "theme=" + th.id, + then: th.icon + })) + )) + + + } private buildLayerIndex(knownImagePaths: Set): Map { // First, we expand and validate all builtin layers. These are written to assets/generated/layers @@ -126,7 +153,6 @@ class LayerOverviewUtils { return sharedLayers; } - private buildThemeIndex(knownImagePaths: Set, sharedLayers: Map): Map { console.log(" ---------- VALIDATING BUILTIN THEMES ---------") const themeFiles = ScriptUtils.getThemeFiles(); @@ -139,7 +165,7 @@ class LayerOverviewUtils { for (const themeInfo of themeFiles) { let themeFile = themeInfo.parsed const themePath = themeInfo.path - + themeFile = new PrepareTheme().convertStrict(convertState, themeFile, themePath) new ValidateThemeAndLayers(knownImagePaths, themePath, true) @@ -160,24 +186,6 @@ class LayerOverviewUtils { return fixed; } - - main(_: string[]) { - - const licensePaths = new Set() - for (const i in licenses) { - licensePaths.add(licenses[i].path) - } - - const sharedLayers = this.buildLayerIndex(licensePaths); - const sharedThemes = this.buildThemeIndex(licensePaths, sharedLayers) - - writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({ - "layers": Array.from(sharedLayers.values()), - "themes": Array.from(sharedThemes.values()) - })) - - writeFileSync("./assets/generated/known_layers.json", JSON.stringify(Array.from(sharedLayers.values()))) - } } new LayerOverviewUtils().main(process.argv) diff --git a/scripts/slice.ts b/scripts/slice.ts index c8c6331a0..8fab5c52e 100644 --- a/scripts/slice.ts +++ b/scripts/slice.ts @@ -3,6 +3,7 @@ import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledF import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource"; import * as readline from "readline"; import ScriptUtils from "./ScriptUtils"; +import {Utils} from "../Utils"; /** * This script slices a big newline-delimeted geojson file into tiled geojson @@ -103,16 +104,23 @@ async function main(args: string[]) { let allFeatures: any []; if (inputFile.endsWith(".geojson")) { + console.log("Detected geojson") allFeatures = await readFeaturesFromGeoJson(inputFile) } else { + console.log("Loading as newline-delimited features") allFeatures = await readFeaturesFromLineDelimitedJsonFile(inputFile) } + allFeatures = Utils.NoNull(allFeatures) console.log("Loaded all", allFeatures.length, "points") const keysToRemove = ["STRAATNMID", "GEMEENTE", "POSTCODE"] for (const f of allFeatures) { + if(f.properties === null){ + console.log("Got a feature without properties!", f) + continue + } for (const keyToRm of keysToRemove) { delete f.properties[keyToRm] }