Add script to download all summary tiles

package.json

@@ -119,7 +119,8 @@
     "server:ldjson": "vite-node scripts/serverLdScrape.ts",
     "sever:studio": "vite-node scripts/studioServer -- /root/git/MapComplete/assets",
     "server:errorreport": "vite-node scripts/serverErrorReport.ts -- /root/error_reports/",
-    "generate:buildDbScript": "vite-node scripts/osm2pgsql/generateBuildDbScript.ts"
+    "generate:buildDbScript": "vite-node scripts/osm2pgsql/generateBuildDbScript.ts",
+    "generate:summaryCache": "vite-node scripts/generateSummaryTileCache.ts"
   },
   "keywords": [
     "OpenStreetMap",

scripts/generateSummaryTileCache.ts

@@ -0,0 +1,88 @@
+import Script from "./Script"
+import Constants from "../src/Models/Constants"
+import { Utils } from "../src/Utils"
+import ScriptUtils from "./ScriptUtils"
+import { SummaryTileSource } from "../src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource"
+import { Tiles } from "../src/Models/TileRange"
+import { Feature, Point } from "geojson"
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
+
+class GenerateSummaryTileCache extends Script {
+    private readonly url: string
+    private readonly cacheDir = "./data/summary_cache/"
+
+    constructor() {
+        super("Generates all summary tiles up to z=12")
+        this.url = Constants.SummaryServer
+        if (!existsSync(this.cacheDir)) {
+            mkdirSync(this.cacheDir)
+        }
+    }
+
+    async fetchTile(z: number, x: number, y: number, layersSummed: string): Promise<Feature<Point>> {
+        const index = Tiles.tile_index(z, x, y)
+        let feature: Feature<Point> | any = (await SummaryTileSource.downloadTile(index, this.url, layersSummed).AsPromise())[0]
+        if (!feature) {
+            feature = { properties: { total: 0 } }
+        }
+        delete feature.properties.layers
+        delete feature.properties.id
+        delete feature.properties.total_metric
+        delete feature.properties.summary // is simply "yes" for rendering
+        delete feature.properties.lon
+        delete feature.properties.lat
+        return feature
+    }
+
+    async fetchTileRecursive(z: number, x: number, y: number, layersSummed: string): Promise<Feature<Point>> {
+        const index = Tiles.tile_index(z, x, y)
+        const path = this.cacheDir + "tile_" + z + "_" + x + "_" + y + ".json"
+        if (existsSync(path)) {
+            return JSON.parse(readFileSync(path, "utf8"))
+        }
+        let feature: Feature<Point>
+        if (z >= 14) {
+            feature = await this.fetchTile(z, x, y, layersSummed)
+        } else {
+            const parts = await Promise.all([
+                this.fetchTileRecursive(z + 1, x * 2, y * 2, layersSummed),
+                this.fetchTileRecursive(z + 1, x * 2 + 1, y * 2, layersSummed),
+                this.fetchTileRecursive(z + 1, x * 2, y * 2 + 1, layersSummed),
+                this.fetchTileRecursive(z + 1, x * 2 + 1, y * 2 + 1, layersSummed)])
+            const sum = this.sumTotals(parts.map(f => f.properties))
+            feature = <Feature<Point>>{
+                type: "Feature",
+                properties: sum,
+                geometry: {
+                    type: "Point",
+                    coordinates: Tiles.centerPointOf(z, x, y),
+                },
+            }
+        }
+
+        writeFileSync(path, JSON.stringify(feature))
+        return feature
+    }
+
+    sumTotals(properties: Record<string, number>[]): Record<string, number> {
+        const sum: Record<string, number> = {}
+        for (const property of properties) {
+            for (const k in property) {
+                sum[k] = (sum[k] ?? 0) + property[k]
+            }
+        }
+        return sum
+    }
+
+
+    async main(args: string[]): Promise<void> {
+
+        const layers = await Utils.downloadJson<{ layers: string[], meta: object }>(this.url + "/status.json")
+        const layersSummed = layers.layers.map(l => encodeURIComponent(l)).join("+")
+        const r = await this.fetchTileRecursive(12, 2084, 1367, layersSummed)
+        console.log(r)
+
+    }
+}
+
+new GenerateSummaryTileCache().run()

src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource.ts

@@ -84,11 +84,31 @@ export class SummaryTileSource extends DynamicTileSource {
             zoomRounded,
             0, // minzoom
             (tileIndex) => {
+                const features = SummaryTileSource.downloadTile(tileIndex, cacheserver, layersSummed)
+                const [z] = Tiles.tile_from_index(tileIndex)
+                return new StaticFeatureSource(
+                    features.map(
+                        (f) => {
+                            if (z - zDiff !== zoomRounded.data) {
+                                return SummaryTileSource.empty
+                            }
+                            return f
+                        },
+                        [zoomRounded]
+                    )
+                )
+            },
+            mapProperties,
+            { ...options, zDiff }
+        )
+    }
+
+    public static downloadTile(tileIndex: number, cacheserver: string, layersSummed: string): Store<Feature<Point>[]>{
         const [z, x, y] = Tiles.tile_from_index(tileIndex)
         let coordinates = Tiles.centerPointOf(z, x, y)
         const url = `${cacheserver}/${layersSummed}/${z}/${x}/${y}.json`
         const count = UIEventSource.FromPromiseWithErr(Utils.downloadJson(url))
-                const features: Store<Feature<Point>[]> = count.mapD((count) => {
+        return count.mapD((count) => {
             if (count["error"] !== undefined) {
                 console.error(
                     "Could not download count for tile",
@@ -138,20 +158,5 @@ export class SummaryTileSource extends DynamicTileSource {
                 },
             ]
         })
-                return new StaticFeatureSource(
-                    features.map(
-                        (f) => {
-                            if (z - zDiff !== zoomRounded.data) {
-                                return SummaryTileSource.empty
-                            }
-                            return f
-                        },
-                        [zoomRounded]
-                    )
-                )
-            },
-            mapProperties,
-            { ...options, zDiff }
-        )
     }
 }