midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Chore: reformat all files with prettier

Browse source
This commit is contained in:
Pieter Vander Vennet 2023-06-14 20:39:36 +02:00
parent 5757ae5dea
commit d008dcb54d
214 changed files with 8926 additions and 8196 deletions
Download patch file Download diff file Expand all files Collapse all files

429
scripts/generateCache.ts
View file

@ -1,37 +1,34 @@
/**
* Generates a collection of geojson files based on an overpass query for a given theme
*/
import {Utils} from "../Utils"
import {Overpass} from "../Logic/Osm/Overpass"
import {existsSync, readFileSync, writeFileSync} from "fs"
import {TagsFilter} from "../Logic/Tags/TagsFilter"
import {Or} from "../Logic/Tags/Or"
import {AllKnownLayouts} from "../Customizations/AllKnownLayouts"
import { Utils } from "../Utils"
import { Overpass } from "../Logic/Osm/Overpass"
import { existsSync, readFileSync, writeFileSync } from "fs"
import { TagsFilter } from "../Logic/Tags/TagsFilter"
import { Or } from "../Logic/Tags/Or"
import { AllKnownLayouts } from "../Customizations/AllKnownLayouts"
import * as OsmToGeoJson from "osmtogeojson"
import MetaTagging from "../Logic/MetaTagging"
import {UIEventSource} from "../Logic/UIEventSource"
import {TileRange, Tiles} from "../Models/TileRange"
import { UIEventSource } from "../Logic/UIEventSource"
import { TileRange, Tiles } from "../Models/TileRange"
import LayoutConfig from "../Models/ThemeConfig/LayoutConfig"
import ScriptUtils from "./ScriptUtils"
import PerLayerFeatureSourceSplitter from "../Logic/FeatureSource/PerLayerFeatureSourceSplitter"
import FilteredLayer from "../Models/FilteredLayer"
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource"
import Constants from "../Models/Constants"
import {GeoOperations} from "../Logic/GeoOperations"
import SimpleMetaTaggers, {ReferencingWaysMetaTagger} from "../Logic/SimpleMetaTagger"
import { GeoOperations } from "../Logic/GeoOperations"
import SimpleMetaTaggers, { ReferencingWaysMetaTagger } from "../Logic/SimpleMetaTagger"
import FilteringFeatureSource from "../Logic/FeatureSource/Sources/FilteringFeatureSource"
import {Feature} from "geojson"
import {BBox} from "../Logic/BBox"
import {FeatureSource} from "../Logic/FeatureSource/FeatureSource";
import OsmObjectDownloader from "../Logic/Osm/OsmObjectDownloader";
import FeaturePropertiesStore from "../Logic/FeatureSource/Actors/FeaturePropertiesStore";
import { Feature } from "geojson"
import { BBox } from "../Logic/BBox"
import { FeatureSource } from "../Logic/FeatureSource/FeatureSource"
import OsmObjectDownloader from "../Logic/Osm/OsmObjectDownloader"
import FeaturePropertiesStore from "../Logic/FeatureSource/Actors/FeaturePropertiesStore"
ScriptUtils.fixUtils()
function createOverpassObject(
theme: LayoutConfig,
backend: string
) {
function createOverpassObject(theme: LayoutConfig, backend: string) {
let filters: TagsFilter[] = []
let extraScripts: string[] = []
for (const layer of theme.layers) {
@ -59,12 +56,7 @@ function createOverpassObject(
if (filters.length + extraScripts.length === 0) {
throw "Nothing to download! The theme doesn't declare anything to download"
}
return new Overpass(
new Or(filters),
extraScripts,
backend,
new UIEventSource<number>(60),
)
return new Overpass(new Or(filters), extraScripts, backend, new UIEventSource<number>(60))
}
function rawJsonName(targetDir: string, x: number, y: number, z: number): string {
@ -79,7 +71,7 @@ function geoJsonName(targetDir: string, x: number, y: number, z: number): string
async function downloadRaw(
targetdir: string,
r: TileRange,
theme: LayoutConfig,
theme: LayoutConfig
): Promise<{ failed: number; skipped: number }> {
let downloaded = 0
let failed = 0
@ -127,14 +119,14 @@ async function downloadRaw(
)
const url = overpass.buildQuery(
"[bbox:" +
bounds.south +
"," +
bounds.west +
"," +
bounds.north +
"," +
bounds.east +
"]"
bounds.south +
"," +
bounds.west +
"," +
bounds.north +
"," +
bounds.east +
"]"
)
try {
@ -164,7 +156,7 @@ async function downloadRaw(
}
}
return {failed: failed, skipped: skipped}
return { failed: failed, skipped: skipped }
}
/*
@ -174,7 +166,7 @@ async function downloadRaw(
async function downloadExtraData(theme: LayoutConfig) /* : any[] */ {
const allFeatures: any[] = []
for (const layer of theme.layers) {
if(!layer.source?.geojsonSource){
if (!layer.source?.geojsonSource) {
continue
}
const source = layer.source.geojsonSource
@ -182,7 +174,7 @@ async function downloadExtraData(theme: LayoutConfig) /* : any[] */ {
// Cached layers are not considered here
continue
}
if(source.startsWith("https://api.openstreetmap.org/api/0.6/notes.json")){
if (source.startsWith("https://api.openstreetmap.org/api/0.6/notes.json")) {
// We ignore map notes
continue
}
@ -211,7 +203,7 @@ function loadAllTiles(
}
// We read the raw OSM-file and convert it to a geojson
const rawOsm = JSON.parse(readFileSync(filename, {encoding: "utf8"}))
const rawOsm = JSON.parse(readFileSync(filename, { encoding: "utf8" }))
// Create and save the geojson file - which is the main chunk of the data
const geojson = OsmToGeoJson.default(rawOsm)
@ -254,80 +246,76 @@ async function sliceToTiles(
return indexedFeatures.get(id)
}
const flayers: FilteredLayer[] = theme.layers.map((l) => new FilteredLayer(l))
const perLayer = new PerLayerFeatureSourceSplitter(
flayers,
allFeatures,
)
const perLayer = new PerLayerFeatureSourceSplitter(flayers, allFeatures)
for (const [layerId, source] of perLayer.perLayer) {
const layer = flayers.find(flayer => flayer.layerDef.id === layerId).layerDef
const targetZoomLevel = layer.source.geojsonZoomLevel ?? targetzoomLevel
const layer = flayers.find((flayer) => flayer.layerDef.id === layerId).layerDef
const targetZoomLevel = layer.source.geojsonZoomLevel ?? targetzoomLevel
if (layer.source.geojsonSource && layer.source.isOsmCacheLayer !== true) {
console.log("Skipping layer ", layerId, ": not a caching layer")
skippedLayers.add(layer.id)
continue
}
const flayer: FilteredLayer = new FilteredLayer(layer)
console.log(
"Handling layer ",
layerId,
"which has",
source.features.data.length,
"features"
)
if (source.features.data.length === 0) {
continue
}
const featureProperties: FeaturePropertiesStore = new FeaturePropertiesStore(source)
if (layer.source.geojsonSource && layer.source.isOsmCacheLayer !== true) {
console.log("Skipping layer ", layerId, ": not a caching layer")
skippedLayers.add(layer.id)
continue
}
const flayer: FilteredLayer = new FilteredLayer(layer)
console.log(
"Handling layer ",
layerId,
"which has",
source.features.data.length,
"features"
)
if (source.features.data.length === 0) {
continue
}
const featureProperties: FeaturePropertiesStore = new FeaturePropertiesStore(source)
MetaTagging.addMetatags(
source.features.data,
{
getFeaturesWithin: (_) => {
return <any>[allFeatures.features.data]
},
getFeatureById: getFeatureById,
MetaTagging.addMetatags(
source.features.data,
{
getFeaturesWithin: (_) => {
return <any>[allFeatures.features.data]
},
layer,
theme,
osmObjectDownloader,
featureProperties,
{
includeDates: false,
includeNonDates: true,
evaluateStrict: true,
}
)
while (SimpleMetaTaggers.country.runningTasks.size > 0) {
console.log(
"Still waiting for ",
SimpleMetaTaggers.country.runningTasks.size,
" features which don't have a country yet"
)
await ScriptUtils.sleep(250)
getFeatureById: getFeatureById,
},
layer,
theme,
osmObjectDownloader,
featureProperties,
{
includeDates: false,
includeNonDates: true,
evaluateStrict: true,
}
)
const createdTiles = []
// At this point, we have all the features of the entire area.
// However, we want to export them per tile of a fixed size, so we use a dynamicTileSOurce to split it up
const features = source.features.data
const perBbox = GeoOperations.spreadIntoBboxes(features, targetZoomLevel)
while (SimpleMetaTaggers.country.runningTasks.size > 0) {
console.log(
"Still waiting for ",
SimpleMetaTaggers.country.runningTasks.size,
" features which don't have a country yet"
)
await ScriptUtils.sleep(250)
}
for (let [tileIndex, features] of perBbox) {
const bbox = BBox.fromTileIndex(tileIndex).asGeoJson({})
console.log("Got tile:", tileIndex, layer.id)
if (features.length === 0) {
continue
}
const filteredTile = new FilteringFeatureSource(
flayer,
new StaticFeatureSource(features)
)
console.log(
"Tile " +
const createdTiles = []
// At this point, we have all the features of the entire area.
// However, we want to export them per tile of a fixed size, so we use a dynamicTileSOurce to split it up
const features = source.features.data
const perBbox = GeoOperations.spreadIntoBboxes(features, targetZoomLevel)
for (let [tileIndex, features] of perBbox) {
const bbox = BBox.fromTileIndex(tileIndex).asGeoJson({})
console.log("Got tile:", tileIndex, layer.id)
if (features.length === 0) {
continue
}
const filteredTile = new FilteringFeatureSource(
flayer,
new StaticFeatureSource(features)
)
console.log(
"Tile " +
layer.id +
"." +
tileIndex +
@ -336,128 +324,121 @@ async function sliceToTiles(
" features after filtering (" +
features.length +
") features before"
)
if (filteredTile.features.data.length === 0) {
continue
}
)
if (filteredTile.features.data.length === 0) {
continue
}
let strictlyCalculated = 0
let featureCount = 0
let strictlyCalculated = 0
let featureCount = 0
for (const feature of features) {
// Some cleanup
for (const feature of features) {
// Some cleanup
if (layer.calculatedTags !== undefined) {
// Evaluate all the calculated tags strictly
const calculatedTagKeys = layer.calculatedTags.map(
(ct) => ct[0]
)
featureCount++
const props = feature.properties
for (const calculatedTagKey of calculatedTagKeys) {
const strict = props[calculatedTagKey]
if (layer.calculatedTags !== undefined) {
// Evaluate all the calculated tags strictly
const calculatedTagKeys = layer.calculatedTags.map((ct) => ct[0])
featureCount++
const props = feature.properties
for (const calculatedTagKey of calculatedTagKeys) {
const strict = props[calculatedTagKey]
if (props.hasOwnProperty(calculatedTagKey)) {
delete props[calculatedTagKey]
}
if (props.hasOwnProperty(calculatedTagKey)) {
delete props[calculatedTagKey]
}
props[calculatedTagKey] = strict
strictlyCalculated++
if (strictlyCalculated % 100 === 0) {
console.log(
"Strictly calculated ",
strictlyCalculated,
"values for tile",
tileIndex,
": now at ",
featureCount,
"/",
filteredTile.features.data.length,
"examle value: ",
strict
)
}
props[calculatedTagKey] = strict
strictlyCalculated++
if (strictlyCalculated % 100 === 0) {
console.log(
"Strictly calculated ",
strictlyCalculated,
"values for tile",
tileIndex,
": now at ",
featureCount,
"/",
filteredTile.features.data.length,
"examle value: ",
strict
)
}
}
delete feature["bbox"]
}
if (clip) {
console.log("Clipping features")
features = [].concat(
...features.map((f: Feature) => GeoOperations.clipWith(<any>f, bbox))
)
}
// Lets save this tile!
const [z, x, y] = Tiles.tile_from_index(tileIndex)
// console.log("Writing tile ", z, x, y, layerId)
const targetPath = geoJsonName(targetdir + "_" + layerId, x, y, z)
createdTiles.push(tileIndex)
// This is the geojson file containing all features for this tile
writeFileSync(
targetPath,
JSON.stringify(
{
type: "FeatureCollection",
features,
},
null,
" "
)
)
console.log("Written tile", targetPath, "with", filteredTile.features.data.length)
delete feature["bbox"]
}
// All the tiles are written at this point
// Only thing left to do is to create the index
const path = targetdir + "_" + layerId + "_" + targetZoomLevel + "_overview.json"
const perX = {}
createdTiles
.map((i) => Tiles.tile_from_index(i))
.forEach(([z, x, y]) => {
const key = "" + x
if (perX[key] === undefined) {
perX[key] = []
}
perX[key].push(y)
})
console.log("Written overview: ", path, "with ", createdTiles.length, "tiles")
writeFileSync(path, JSON.stringify(perX))
// And, if needed, to create a points-only layer
if (pointsOnlyLayers.indexOf(layer.id) >= 0) {
const filtered = new FilteringFeatureSource(
flayer,
source
)
const features = filtered.features.data
const points = features.map((feature) => GeoOperations.centerpoint(feature))
console.log("Writing points overview for ", layerId)
const targetPath = targetdir + "_" + layerId + "_points.geojson"
// This is the geojson file containing all features for this tile
writeFileSync(
targetPath,
JSON.stringify(
{
type: "FeatureCollection",
features: points,
},
null,
" "
)
if (clip) {
console.log("Clipping features")
features = [].concat(
...features.map((f: Feature) => GeoOperations.clipWith(<any>f, bbox))
)
}
// Lets save this tile!
const [z, x, y] = Tiles.tile_from_index(tileIndex)
// console.log("Writing tile ", z, x, y, layerId)
const targetPath = geoJsonName(targetdir + "_" + layerId, x, y, z)
createdTiles.push(tileIndex)
// This is the geojson file containing all features for this tile
writeFileSync(
targetPath,
JSON.stringify(
{
type: "FeatureCollection",
features,
},
null,
" "
)
)
console.log("Written tile", targetPath, "with", filteredTile.features.data.length)
}
// All the tiles are written at this point
// Only thing left to do is to create the index
const path = targetdir + "_" + layerId + "_" + targetZoomLevel + "_overview.json"
const perX = {}
createdTiles
.map((i) => Tiles.tile_from_index(i))
.forEach(([z, x, y]) => {
const key = "" + x
if (perX[key] === undefined) {
perX[key] = []
}
perX[key].push(y)
})
console.log("Written overview: ", path, "with ", createdTiles.length, "tiles")
writeFileSync(path, JSON.stringify(perX))
// And, if needed, to create a points-only layer
if (pointsOnlyLayers.indexOf(layer.id) >= 0) {
const filtered = new FilteringFeatureSource(flayer, source)
const features = filtered.features.data
const points = features.map((feature) => GeoOperations.centerpoint(feature))
console.log("Writing points overview for ", layerId)
const targetPath = targetdir + "_" + layerId + "_points.geojson"
// This is the geojson file containing all features for this tile
writeFileSync(
targetPath,
JSON.stringify(
{
type: "FeatureCollection",
features: points,
},
null,
" "
)
)
}
}
const skipped = Array.from(skippedLayers)
if (skipped.length > 0) {
console.warn(
"Did not save any cache files for layers " +
skipped.join(", ") +
" as these didn't set the flag `isOsmCache` to true"
skipped.join(", ") +
" as these didn't set the flag `isOsmCache` to true"
)
}
}
@ -468,14 +449,18 @@ export async function main(args: string[]) {
if (args.length < 6) {
console.error(
"Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1 [--generate-point-overview layer-name,layer-name,...] [--force-zoom-level z] [--clip]" +
"--force-zoom-level causes non-cached-layers to be donwnloaded\n" +
"--clip will erase parts of the feature falling outside of the bounding box"
"--force-zoom-level causes non-cached-layers to be donwnloaded\n" +
"--clip will erase parts of the feature falling outside of the bounding box"
)
return
}
const themeName = args[0]
const zoomlevel = Number(args[1])
console.log("Target zoomlevel for the tiles is",zoomlevel,"; this can be overridden by the individual layers")
console.log(
"Target zoomlevel for the tiles is",
zoomlevel,
"; this can be overridden by the individual layers"
)
const targetdir = args[2] + "/" + themeName
if (!existsSync(args[2])) {
@ -522,8 +507,11 @@ export async function main(args: string[]) {
return
}
theme.layers = theme.layers.filter(l => Constants.priviliged_layers.indexOf(<any> l.id) < 0 && !l.id.startsWith("note_import_"))
console.log("Layers to download:", theme.layers.map(l => l.id).join(", "))
theme.layers = theme.layers.filter(
(l) =>
Constants.priviliged_layers.indexOf(<any>l.id) < 0 && !l.id.startsWith("note_import_")
)
console.log("Layers to download:", theme.layers.map((l) => l.id).join(", "))
let generatePointLayersFor = []
if (args[7] == "--generate-point-overview") {
@ -553,14 +541,13 @@ export async function main(args: string[]) {
let failed = 0
do {
try{
const cachingResult = await downloadRaw(targetdir, tileRange, theme)
failed = cachingResult.failed
if (failed > 0) {
await ScriptUtils.sleep(30000)
}
}catch(e){
try {
const cachingResult = await downloadRaw(targetdir, tileRange, theme)
failed = cachingResult.failed
if (failed > 0) {
await ScriptUtils.sleep(30000)
}
} catch (e) {
console.error(e)
return
}