Refactoring: fix generateCache-script

Logic/GeoOperations.ts

@@ -408,12 +408,38 @@ export class GeoOperations {
     /**
      * Calculates line intersection between two features.
      */
-    public static LineIntersections(feature, otherFeature): [number, number][] {
+    public static LineIntersections(feature: Feature<LineString | MultiLineString | Polygon | MultiPolygon>, otherFeature: Feature<LineString | MultiLineString | Polygon | MultiPolygon>): [number, number][] {
         return turf
             .lineIntersect(feature, otherFeature)
             .features.map((p) => <[number, number]>p.geometry.coordinates)
     }
 
+    /**
+     * Given a list of features, will construct a map of slippy map tile-indices.
+     * Features of which the BBOX overlaps with the corresponding slippy map tile are added to the corresponding array
+     * @param features
+     * @param zoomlevel
+     */
+    public static spreadIntoBboxes(features: Feature[], zoomlevel: number) : Map<number, Feature[]> {
+
+        const perBbox = new Map<number, Feature[]>()
+
+        for (const feature of features) {
+            const bbox = BBox.get(feature)
+            const tilerange = bbox.expandToTileBounds(zoomlevel).containingTileRange(zoomlevel)
+            Tiles.MapRange(tilerange, (x, y) => {
+                const tileNumber = Tiles.tile_index(zoomlevel, x, y)
+                let newFeatureList = perBbox.get(tileNumber)
+                if(newFeatureList === undefined){
+                    newFeatureList = []
+                    perBbox.set(tileNumber, newFeatureList)
+                }
+                newFeatureList.push(feature)
+            })
+        }
+
+        return perBbox
+    }
     public static toGpx(
         locations:
             | Feature<LineString>

scripts/generateCache.ts

@@ -1,30 +1,30 @@
 /**
  * Generates a collection of geojson files based on an overpass query for a given theme
  */
-import { Utils } from "../Utils"
+import {Utils} from "../Utils"
-import { Overpass } from "../Logic/Osm/Overpass"
+import {Overpass} from "../Logic/Osm/Overpass"
-import { existsSync, readFileSync, writeFileSync } from "fs"
+import {existsSync, readFileSync, writeFileSync} from "fs"
-import { TagsFilter } from "../Logic/Tags/TagsFilter"
+import {TagsFilter} from "../Logic/Tags/TagsFilter"
-import { Or } from "../Logic/Tags/Or"
+import {Or} from "../Logic/Tags/Or"
-import { AllKnownLayouts } from "../Customizations/AllKnownLayouts"
+import {AllKnownLayouts} from "../Customizations/AllKnownLayouts"
 import * as OsmToGeoJson from "osmtogeojson"
 import MetaTagging from "../Logic/MetaTagging"
-import { ImmutableStore, UIEventSource } from "../Logic/UIEventSource"
+import {ImmutableStore, UIEventSource} from "../Logic/UIEventSource"
-import { TileRange, Tiles } from "../Models/TileRange"
+import {TileRange, Tiles} from "../Models/TileRange"
 import LayoutConfig from "../Models/ThemeConfig/LayoutConfig"
 import ScriptUtils from "./ScriptUtils"
 import PerLayerFeatureSourceSplitter from "../Logic/FeatureSource/PerLayerFeatureSourceSplitter"
 import FilteredLayer from "../Models/FilteredLayer"
-import FeatureSource, { FeatureSourceForLayer } from "../Logic/FeatureSource/FeatureSource"
 import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource"
-import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource"
 import Constants from "../Models/Constants"
-import { GeoOperations } from "../Logic/GeoOperations"
+import {GeoOperations} from "../Logic/GeoOperations"
-import SimpleMetaTaggers, { ReferencingWaysMetaTagger } from "../Logic/SimpleMetaTagger"
+import SimpleMetaTaggers, {ReferencingWaysMetaTagger} from "../Logic/SimpleMetaTagger"
 import FilteringFeatureSource from "../Logic/FeatureSource/Sources/FilteringFeatureSource"
-import Loc from "../Models/Loc"
+import {Feature} from "geojson"
-import { Feature } from "geojson"
+import {BBox} from "../Logic/BBox"
-import { BBox } from "../Logic/BBox"
+import {FeatureSource, FeatureSourceForLayer} from "../Logic/FeatureSource/FeatureSource";
+import OsmObjectDownloader from "../Logic/Osm/OsmObjectDownloader";
+import FeaturePropertiesStore from "../Logic/FeatureSource/Actors/FeaturePropertiesStore";
 
 ScriptUtils.fixUtils()
 
@@ -41,7 +41,10 @@ function createOverpassObject(
         if (layer.doNotDownload) {
             continue
         }
-        if (layer.source.geojsonSource !== undefined) {
+        if (!layer.source) {
+            continue
+        }
+        if (layer.source.geojsonSource) {
             // This layer defines a geoJson-source
             // SHould it be cached?
             if (layer.source.isOsmCacheLayer !== true) {
@@ -156,12 +159,12 @@ async function downloadRaw(
                     "Could not download - probably hit the rate limit; waiting a bit. (" + err + ")"
                 )
                 failed++
-                await ScriptUtils.sleep(1000)
+                await ScriptUtils.sleep(1)
             }
         }
     }
 
-    return { failed: failed, skipped: skipped }
+    return {failed: failed, skipped: skipped}
 }
 
 /*
@@ -171,14 +174,18 @@ async function downloadRaw(
 async function downloadExtraData(theme: LayoutConfig) /* : any[] */ {
     const allFeatures: any[] = []
     for (const layer of theme.layers) {
-        const source = layer.source.geojsonSource
+        if(!layer.source?.geojsonSource){
-        if (source === undefined) {
             continue
         }
+        const source = layer.source.geojsonSource
         if (layer.source.isOsmCacheLayer !== undefined && layer.source.isOsmCacheLayer !== false) {
             // Cached layers are not considered here
             continue
         }
+        if(source.startsWith("https://api.openstreetmap.org/api/0.6/notes.json")){
+            // We ignore map notes
+            continue
+        }
         console.log("Downloading extra data: ", source)
         await Utils.downloadJson(source).then((json) => allFeatures.push(...json.features))
     }
@@ -204,7 +211,7 @@ function loadAllTiles(
             }
 
             // We read the raw OSM-file and convert it to a geojson
-            const rawOsm = JSON.parse(readFileSync(filename, { encoding: "utf8" }))
+            const rawOsm = JSON.parse(readFileSync(filename, {encoding: "utf8"}))
 
             // Create and save the geojson file - which is the main chunk of the data
             const geojson = OsmToGeoJson.default(rawOsm)
@@ -219,17 +226,19 @@ function loadAllTiles(
 /**
  * Load all the tiles into memory from disk
  */
-function sliceToTiles(
+async function sliceToTiles(
     allFeatures: FeatureSource,
     theme: LayoutConfig,
     targetdir: string,
     pointsOnlyLayers: string[],
-    clip: boolean
+    clip: boolean,
+    targetzoomLevel: number = 9
 ) {
     const skippedLayers = new Set<string>()
 
     const indexedFeatures: Map<string, any> = new Map<string, any>()
     let indexisBuilt = false
+    const osmObjectDownloader = new OsmObjectDownloader()
 
     function buildIndex() {
         for (const f of allFeatures.features.data) {
@@ -245,16 +254,22 @@ function sliceToTiles(
         return indexedFeatures.get(id)
     }
 
-    async function handleLayer(source: FeatureSourceForLayer) {
-        const layer = source.layer.layerDef
-        const targetZoomLevel = layer.source.geojsonZoomLevel ?? 0
 
-        const layerId = layer.id
+    const flayers: FilteredLayer[] = theme.layers.map((l) => new FilteredLayer(l))
-        if (layer.source.isOsmCacheLayer !== true) {
+    const perLayer = new PerLayerFeatureSourceSplitter(
+        flayers,
+        allFeatures,
+    )
+    for (const [layerId, source] of perLayer.perLayer) {
+            const layer = flayers.find(flayer => flayer.layerDef.id === layerId).layerDef
+            const targetZoomLevel = layer.source.geojsonZoomLevel ?? targetzoomLevel
+
+            if (layer.source.geojsonSource && layer.source.isOsmCacheLayer !== true) {
                 console.log("Skipping layer ", layerId, ": not a caching layer")
                 skippedLayers.add(layer.id)
-            return
+                continue
             }
+            const flayer: FilteredLayer = new FilteredLayer(layer)
             console.log(
                 "Handling layer ",
                 layerId,
@@ -263,18 +278,22 @@ function sliceToTiles(
                 "features"
             )
             if (source.features.data.length === 0) {
-            return
+                continue
             }
+            const featureProperties: FeaturePropertiesStore = new FeaturePropertiesStore(source)
+
             MetaTagging.addMetatags(
                 source.features.data,
                 {
                     getFeaturesWithin: (_) => {
-                    return <any> [allFeatures.features.data]
+                        return <any>[allFeatures.features.data]
                     },
                     getFeatureById: getFeatureById,
                 },
                 layer,
-            {},
+                theme,
+                osmObjectDownloader,
+                featureProperties,
                 {
                     includeDates: false,
                     includeNonDates: true,
@@ -288,36 +307,25 @@ function sliceToTiles(
                     SimpleMetaTaggers.country.runningTasks.size,
                     " features which don't have a country yet"
                 )
-            await ScriptUtils.sleep(1)
+                await ScriptUtils.sleep(250)
             }
 
             const createdTiles = []
             // At this point, we have all the features of the entire area.
             // However, we want to export them per tile of a fixed size, so we use a dynamicTileSOurce to split it up
-        TiledFeatureSource.createHierarchy(source, {
+            const features = source.features.data
-            minZoomLevel: targetZoomLevel,
+            const perBbox = GeoOperations.spreadIntoBboxes(features, targetZoomLevel)
-            maxZoomLevel: targetZoomLevel,
+
-            maxFeatureCount: undefined,
+            for (let [tileIndex, features] of perBbox) {
-            registerTile: (tile) => {
-                const tileIndex = tile.tileIndex
                 const bbox = BBox.fromTileIndex(tileIndex).asGeoJson({})
-                console.log("Got tile:", tileIndex, tile.layer.layerDef.id)
+                console.log("Got tile:", tileIndex, layer.id)
-                if (tile.features.data.length === 0) {
+                if (features.length === 0) {
-                    return
+                    continue
                 }
-
                 const filteredTile = new FilteringFeatureSource(
-                    {
+                    flayer,
-                        locationControl: new ImmutableStore<Loc>(undefined),
+                    new StaticFeatureSource(features)
-                        allElements: undefined,
-                        selectedElement: new ImmutableStore<any>(undefined),
-                        globalFilters: new ImmutableStore([]),
-                    },
-                    tileIndex,
-                    tile,
-                    new UIEventSource<any>(undefined)
                 )
-
                 console.log(
                     "Tile " +
                     layer.id +
@@ -326,21 +334,22 @@ function sliceToTiles(
                     " contains " +
                     filteredTile.features.data.length +
                     " features after filtering (" +
-                        tile.features.data.length +
+                    features.length +
                     ") features before"
                 )
                 if (filteredTile.features.data.length === 0) {
-                    return
+                    continue
                 }
+
                 let strictlyCalculated = 0
                 let featureCount = 0
-                let features: Feature[] = filteredTile.features.data
+
                 for (const feature of features) {
                     // Some cleanup
 
-                    if (tile.layer.layerDef.calculatedTags !== undefined) {
+                    if (layer.calculatedTags !== undefined) {
                         // Evaluate all the calculated tags strictly
-                        const calculatedTagKeys = tile.layer.layerDef.calculatedTags.map(
+                        const calculatedTagKeys = layer.calculatedTags.map(
                             (ct) => ct[0]
                         )
                         featureCount++
@@ -379,7 +388,6 @@ function sliceToTiles(
                         ...features.map((f: Feature) => GeoOperations.clipWith(<any>f, bbox))
                     )
                 }
-
                 // Lets save this tile!
                 const [z, x, y] = Tiles.tile_from_index(tileIndex)
                 // console.log("Writing tile ", z, x, y, layerId)
@@ -398,8 +406,9 @@ function sliceToTiles(
                     )
                 )
                 console.log("Written tile", targetPath, "with", filteredTile.features.data.length)
-            },
+
-        })
+            }
+
 
             // All the tiles are written at this point
             // Only thing left to do is to create the index
@@ -420,15 +429,8 @@ function sliceToTiles(
             // And, if needed, to create a points-only layer
             if (pointsOnlyLayers.indexOf(layer.id) >= 0) {
                 const filtered = new FilteringFeatureSource(
-                {
+                    flayer,
-                    locationControl: new ImmutableStore<Loc>(undefined),
+                    source
-                    allElements: undefined,
-                    selectedElement: new ImmutableStore<any>(undefined),
-                    globalFilters: new ImmutableStore([]),
-                },
-                Tiles.tile_index(0, 0, 0),
-                source,
-                new UIEventSource<any>(undefined)
                 )
                 const features = filtered.features.data
 
@@ -450,18 +452,6 @@ function sliceToTiles(
             }
         }
 
-    new PerLayerFeatureSourceSplitter(
-        new UIEventSource<FilteredLayer[]>(
-            theme.layers.map((l) => ({
-                layerDef: l,
-                isDisplayed: new UIEventSource<boolean>(true),
-                appliedFilters: new UIEventSource(undefined),
-            }))
-        ),
-        handleLayer,
-        allFeatures
-    )
-
     const skipped = Array.from(skippedLayers)
     if (skipped.length > 0) {
         console.warn(
@@ -473,7 +463,7 @@ function sliceToTiles(
 }
 
 export async function main(args: string[]) {
-    console.log("Cache builder started with args ", args.join(", "))
+    console.log("Cache builder started with args ", args.join(" "))
     ReferencingWaysMetaTagger.enabled = false
     if (args.length < 6) {
         console.error(
@@ -485,6 +475,7 @@ export async function main(args: string[]) {
     }
     const themeName = args[0]
     const zoomlevel = Number(args[1])
+    console.log("Target zoomlevel for the tiles is",zoomlevel,"; this can be overridden by the individual layers")
 
     const targetdir = args[2] + "/" + themeName
     if (!existsSync(args[2])) {
@@ -531,6 +522,9 @@ export async function main(args: string[]) {
         return
     }
 
+    theme.layers = theme.layers.filter(l => Constants.priviliged_layers.indexOf(<any> l.id) < 0 && !l.id.startsWith("note_import_"))
+    console.log("Layers to download:", theme.layers.map(l => l.id).join(", "))
+
     let generatePointLayersFor = []
     if (args[7] == "--generate-point-overview") {
         if (args[8] === undefined) {
@@ -559,16 +553,22 @@ export async function main(args: string[]) {
 
     let failed = 0
     do {
+        try{
+
         const cachingResult = await downloadRaw(targetdir, tileRange, theme)
         failed = cachingResult.failed
         if (failed > 0) {
             await ScriptUtils.sleep(30000)
         }
+        }catch(e){
+            console.error(e)
+            return
+        }
     } while (failed > 0)
 
     const extraFeatures = await downloadExtraData(theme)
     const allFeaturesSource = loadAllTiles(targetdir, tileRange, theme, extraFeatures)
-    sliceToTiles(allFeaturesSource, theme, targetdir, generatePointLayersFor, clip)
+    await sliceToTiles(allFeaturesSource, theme, targetdir, generatePointLayersFor, clip, zoomlevel)
 }
 
 let args = [...process.argv]