midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Improve slice script, formatting

Browse source
This commit is contained in:
Pieter Vander Vennet 2022-01-16 01:59:06 +01:00
parent cce6a9e832
commit ccb548816f
3 changed files with 88 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

101
scripts/generateCache.ts
View file

@ -24,6 +24,7 @@ import {GeoOperations} from "../Logic/GeoOperations";
import SimpleMetaTaggers from "../Logic/SimpleMetaTagger"; import SimpleMetaTaggers from "../Logic/SimpleMetaTagger";
import FilteringFeatureSource from "../Logic/FeatureSource/Sources/FilteringFeatureSource"; import FilteringFeatureSource from "../Logic/FeatureSource/Sources/FilteringFeatureSource";
import Loc from "../Models/Loc"; import Loc from "../Models/Loc";
ScriptUtils.fixUtils() ScriptUtils.fixUtils()
function createOverpassObject(theme: LayoutConfig, relationTracker: RelationsTracker, backend: string) { function createOverpassObject(theme: LayoutConfig, relationTracker: RelationsTracker, backend: string) {
@ -166,7 +167,7 @@ function loadAllTiles(targetdir: string, r: TileRange, theme: LayoutConfig, extr
// Create and save the geojson file - which is the main chunk of the data // Create and save the geojson file - which is the main chunk of the data
const geojson = OsmToGeoJson.default(rawOsm); const geojson = OsmToGeoJson.default(rawOsm);
console.log(" which as",geojson.features.length, "features") console.log(" which as", geojson.features.length, "features")
allFeatures.push(...geojson.features) allFeatures.push(...geojson.features)
} }
@ -180,23 +181,24 @@ function loadAllTiles(targetdir: string, r: TileRange, theme: LayoutConfig, extr
function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relationsTracker: RelationsTracker, targetdir: string, pointsOnlyLayers: string[]) { function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relationsTracker: RelationsTracker, targetdir: string, pointsOnlyLayers: string[]) {
const skippedLayers = new Set<string>() const skippedLayers = new Set<string>()
const indexedFeatures : Map<string, any> = new Map<string, any>() const indexedFeatures: Map<string, any> = new Map<string, any>()
let indexisBuilt = false; let indexisBuilt = false;
function buildIndex(){
function buildIndex() {
for (const ff of allFeatures.features.data) { for (const ff of allFeatures.features.data) {
const f = ff.feature const f = ff.feature
indexedFeatures.set(f.properties.id, f) indexedFeatures.set(f.properties.id, f)
} }
indexisBuilt = true; indexisBuilt = true;
} }
function getFeatureById(id){ function getFeatureById(id) {
if(!indexisBuilt){ if (!indexisBuilt) {
buildIndex() buildIndex()
} }
return indexedFeatures.get(id) return indexedFeatures.get(id)
} }
async function handleLayer(source: FeatureSourceForLayer) { async function handleLayer(source: FeatureSourceForLayer) {
const layer = source.layer.layerDef; const layer = source.layer.layerDef;
const targetZoomLevel = layer.source.geojsonZoomLevel ?? 0 const targetZoomLevel = layer.source.geojsonZoomLevel ?? 0
@ -225,11 +227,10 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
includeDates: false, includeDates: false,
includeNonDates: true includeNonDates: true
}); });
while (SimpleMetaTaggers.country.runningTasks.size > 0) { while (SimpleMetaTaggers.country.runningTasks.size > 0) {
console.log("Still waiting for ", SimpleMetaTaggers.country.runningTasks.size," features which don't have a country yet") console.log("Still waiting for ", SimpleMetaTaggers.country.runningTasks.size, " features which don't have a country yet")
await ScriptUtils.sleep(1) await ScriptUtils.sleep(1)
} }
@ -246,18 +247,18 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
if (tile.features.data.length === 0) { if (tile.features.data.length === 0) {
return return
} }
const filteredTile = new FilteringFeatureSource({ const filteredTile = new FilteringFeatureSource({
locationControl: new UIEventSource<Loc>(undefined), locationControl: new UIEventSource<Loc>(undefined),
allElements: undefined, allElements: undefined,
selectedElement: new UIEventSource<any>(undefined) selectedElement: new UIEventSource<any>(undefined)
}, },
tileIndex, tileIndex,
tile, tile,
new UIEventSource<any>(undefined) new UIEventSource<any>(undefined)
) )
console.log("Tile "+layer.id+"."+tileIndex+" contains "+filteredTile.features.data.length+" features after filtering ("+tile.features.data.length+") features before") console.log("Tile " + layer.id + "." + tileIndex + " contains " + filteredTile.features.data.length + " features after filtering (" + tile.features.data.length + ") features before")
if (filteredTile.features.data.length === 0) { if (filteredTile.features.data.length === 0) {
return return
} }
@ -266,22 +267,22 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
for (const feature of filteredTile.features.data) { for (const feature of filteredTile.features.data) {
// Some cleanup // Some cleanup
delete feature.feature["bbox"] delete feature.feature["bbox"]
if(tile.layer.layerDef.calculatedTags !== undefined){ if (tile.layer.layerDef.calculatedTags !== undefined) {
// Evaluate all the calculated tags strictly // Evaluate all the calculated tags strictly
const calculatedTagKeys = tile.layer.layerDef.calculatedTags.map(ct => ct[0]) const calculatedTagKeys = tile.layer.layerDef.calculatedTags.map(ct => ct[0])
featureCount++ featureCount++
for (const calculatedTagKey of calculatedTagKeys) { for (const calculatedTagKey of calculatedTagKeys) {
const strict = feature.feature.properties[calculatedTagKey] const strict = feature.feature.properties[calculatedTagKey]
feature.feature.properties[calculatedTagKey] =strict feature.feature.properties[calculatedTagKey] = strict
strictlyCalculated ++; strictlyCalculated++;
if(strictlyCalculated % 100 === 0){ if (strictlyCalculated % 100 === 0) {
console.log("Strictly calculated ", strictlyCalculated, "values for tile",tileIndex,": now at ", featureCount,"/",filteredTile.features.data.length, "examle value: ", strict) console.log("Strictly calculated ", strictlyCalculated, "values for tile", tileIndex, ": now at ", featureCount, "/", filteredTile.features.data.length, "examle value: ", strict)
}
} }
} }
}
} }
// Lets save this tile! // Lets save this tile!
const [z, x, y] = Tiles.tile_from_index(tileIndex) const [z, x, y] = Tiles.tile_from_index(tileIndex)
@ -293,7 +294,7 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
type: "FeatureCollection", type: "FeatureCollection",
features: filteredTile.features.data.map(f => f.feature) features: filteredTile.features.data.map(f => f.feature)
}, null, " ")) }, null, " "))
console.log("Written tile", targetPath,"with", filteredTile.features.data.length) console.log("Written tile", targetPath, "with", filteredTile.features.data.length)
} }
}) })
@ -315,16 +316,16 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
if (pointsOnlyLayers.indexOf(layer.id) >= 0) { if (pointsOnlyLayers.indexOf(layer.id) >= 0) {
const filtered = new FilteringFeatureSource({ const filtered = new FilteringFeatureSource({
locationControl: new UIEventSource<Loc>(undefined), locationControl: new UIEventSource<Loc>(undefined),
allElements: undefined, allElements: undefined,
selectedElement: new UIEventSource<any>(undefined) selectedElement: new UIEventSource<any>(undefined)
}, },
Tiles.tile_index(0,0,0), Tiles.tile_index(0, 0, 0),
source, source,
new UIEventSource<any>(undefined) new UIEventSource<any>(undefined)
) )
const features = filtered.features.data.map(f => f.feature) const features = filtered.features.data.map(f => f.feature)
const points = features.map(feature => GeoOperations.centerpoint(feature)) const points = features.map(feature => GeoOperations.centerpoint(feature))
console.log("Writing points overview for ", layerId) console.log("Writing points overview for ", layerId)
const targetPath = targetdir + "_" + layerId + "_points.geojson" const targetPath = targetdir + "_" + layerId + "_points.geojson"
@ -375,8 +376,6 @@ async function main(args: string[]) {
const lat1 = Number(args[5]) const lat1 = Number(args[5])
const lon1 = Number(args[6]) const lon1 = Number(args[6])
const tileRange = Tiles.TileRangeBetween(zoomlevel, lat0, lon0, lat1, lon1) const tileRange = Tiles.TileRangeBetween(zoomlevel, lat0, lon0, lat1, lon1)
@ -397,29 +396,29 @@ async function main(args: string[]) {
let generatePointLayersFor = [] let generatePointLayersFor = []
if (args[7] == "--generate-point-overview") { if (args[7] == "--generate-point-overview") {
if(args[8] === undefined){ if (args[8] === undefined) {
throw "--generate-point-overview needs a list of layers to generate the overview for (or * for all)" throw "--generate-point-overview needs a list of layers to generate the overview for (or * for all)"
}else if (args[8] === '*'){ } else if (args[8] === '*') {
generatePointLayersFor = theme.layers.map(l => l.id) generatePointLayersFor = theme.layers.map(l => l.id)
}else{ } else {
generatePointLayersFor = args[8].split(",") generatePointLayersFor = args[8].split(",")
} }
console.log("Also generating a point overview for layers ", generatePointLayersFor.join(",")) console.log("Also generating a point overview for layers ", generatePointLayersFor.join(","))
} }
{ {
const index = args.indexOf("--force-zoom-level") const index = args.indexOf("--force-zoom-level")
if(index >= 0){ if (index >= 0) {
const forcedZoomLevel = Number(args[index + 1]) const forcedZoomLevel = Number(args[index + 1])
for (const layer of theme.layers) { for (const layer of theme.layers) {
layer.source.geojsonSource = "https://127.0.0.1/cache_{layer}_{z}_{x}_{y}.geojson" layer.source.geojsonSource = "https://127.0.0.1/cache_{layer}_{z}_{x}_{y}.geojson"
layer.source.isOsmCacheLayer = true layer.source.isOsmCacheLayer = true
layer.source.geojsonZoomLevel = forcedZoomLevel layer.source.geojsonZoomLevel = forcedZoomLevel
}
} }
} }
}
const relationTracker = new RelationsTracker() const relationTracker = new RelationsTracker()
let failed = 0; let failed = 0;

52
scripts/generateLayerOverview.ts
View file

@ -6,7 +6,8 @@ import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
import Constants from "../Models/Constants"; import Constants from "../Models/Constants";
import { import {
DesugaringContext, DesugaringContext,
PrepareLayer, PrepareTheme, PrepareLayer,
PrepareTheme,
ValidateLayer, ValidateLayer,
ValidateThemeAndLayers ValidateThemeAndLayers
} from "../Models/ThemeConfig/Conversion/LegacyJsonConvert"; } from "../Models/ThemeConfig/Conversion/LegacyJsonConvert";
@ -80,7 +81,7 @@ class LayerOverviewUtils {
dict.set(key, <TagRenderingConfigJson>questions[key]) dict.set(key, <TagRenderingConfigJson>questions[key])
} }
for (const key in icons["default"]) { for (const key in icons["default"]) {
if(typeof icons[key] !== "object"){ if (typeof icons[key] !== "object") {
continue continue
} }
icons[key].id = key; icons[key].id = key;
@ -94,6 +95,32 @@ class LayerOverviewUtils {
return dict; return dict;
} }
main(_: string[]) {
const licensePaths = new Set<string>()
for (const i in licenses) {
licensePaths.add(licenses[i].path)
}
const sharedLayers = this.buildLayerIndex(licensePaths);
const sharedThemes = this.buildThemeIndex(licensePaths, sharedLayers)
writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
"layers": Array.from(sharedLayers.values()),
"themes": Array.from(sharedThemes.values())
}))
writeFileSync("./assets/generated/known_layers.json", JSON.stringify(Array.from(sharedLayers.values())))
writeFileSync('./assets/themes/mapcomplete-changes/icons-mapping.txt', JSON.stringify(
Array.from(sharedThemes.values()).map(th => ({
if: "theme=" + th.id,
then: th.icon
}))
))
}
private buildLayerIndex(knownImagePaths: Set<string>): Map<string, LayerConfigJson> { private buildLayerIndex(knownImagePaths: Set<string>): Map<string, LayerConfigJson> {
// First, we expand and validate all builtin layers. These are written to assets/generated/layers // First, we expand and validate all builtin layers. These are written to assets/generated/layers
@ -126,7 +153,6 @@ class LayerOverviewUtils {
return sharedLayers; return sharedLayers;
} }
private buildThemeIndex(knownImagePaths: Set<string>, sharedLayers: Map<string, LayerConfigJson>): Map<string, LayoutConfigJson> { private buildThemeIndex(knownImagePaths: Set<string>, sharedLayers: Map<string, LayerConfigJson>): Map<string, LayoutConfigJson> {
console.log(" ---------- VALIDATING BUILTIN THEMES ---------") console.log(" ---------- VALIDATING BUILTIN THEMES ---------")
const themeFiles = ScriptUtils.getThemeFiles(); const themeFiles = ScriptUtils.getThemeFiles();
@ -139,7 +165,7 @@ class LayerOverviewUtils {
for (const themeInfo of themeFiles) { for (const themeInfo of themeFiles) {
let themeFile = themeInfo.parsed let themeFile = themeInfo.parsed
const themePath = themeInfo.path const themePath = themeInfo.path
themeFile = new PrepareTheme().convertStrict(convertState, themeFile, themePath) themeFile = new PrepareTheme().convertStrict(convertState, themeFile, themePath)
new ValidateThemeAndLayers(knownImagePaths, themePath, true) new ValidateThemeAndLayers(knownImagePaths, themePath, true)
@ -160,24 +186,6 @@ class LayerOverviewUtils {
return fixed; return fixed;
} }
main(_: string[]) {
const licensePaths = new Set<string>()
for (const i in licenses) {
licensePaths.add(licenses[i].path)
}
const sharedLayers = this.buildLayerIndex(licensePaths);
const sharedThemes = this.buildThemeIndex(licensePaths, sharedLayers)
writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
"layers": Array.from(sharedLayers.values()),
"themes": Array.from(sharedThemes.values())
}))
writeFileSync("./assets/generated/known_layers.json", JSON.stringify(Array.from(sharedLayers.values())))
}
} }
new LayerOverviewUtils().main(process.argv) new LayerOverviewUtils().main(process.argv)

8
scripts/slice.ts
View file

@ -3,6 +3,7 @@ import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledF
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource"; import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource";
import * as readline from "readline"; import * as readline from "readline";
import ScriptUtils from "./ScriptUtils"; import ScriptUtils from "./ScriptUtils";
import {Utils} from "../Utils";
/** /**
* This script slices a big newline-delimeted geojson file into tiled geojson * This script slices a big newline-delimeted geojson file into tiled geojson
@ -103,16 +104,23 @@ async function main(args: string[]) {
let allFeatures: any []; let allFeatures: any [];
if (inputFile.endsWith(".geojson")) { if (inputFile.endsWith(".geojson")) {
console.log("Detected geojson")
allFeatures = await readFeaturesFromGeoJson(inputFile) allFeatures = await readFeaturesFromGeoJson(inputFile)
} else { } else {
console.log("Loading as newline-delimited features")
allFeatures = await readFeaturesFromLineDelimitedJsonFile(inputFile) allFeatures = await readFeaturesFromLineDelimitedJsonFile(inputFile)
} }
allFeatures = Utils.NoNull(allFeatures)
console.log("Loaded all", allFeatures.length, "points") console.log("Loaded all", allFeatures.length, "points")
const keysToRemove = ["STRAATNMID", "GEMEENTE", "POSTCODE"] const keysToRemove = ["STRAATNMID", "GEMEENTE", "POSTCODE"]
for (const f of allFeatures) { for (const f of allFeatures) {
if(f.properties === null){
console.log("Got a feature without properties!", f)
continue
}
for (const keyToRm of keysToRemove) { for (const keyToRm of keysToRemove) {
delete f.properties[keyToRm] delete f.properties[keyToRm]
} }