Add script to download all summary tiles

package.json

@@ -119,7 +119,8 @@
     "server:ldjson": "vite-node scripts/serverLdScrape.ts",
     "sever:studio": "vite-node scripts/studioServer -- /root/git/MapComplete/assets",
     "server:errorreport": "vite-node scripts/serverErrorReport.ts -- /root/error_reports/",
-    "generate:buildDbScript": "vite-node scripts/osm2pgsql/generateBuildDbScript.ts"
+    "generate:buildDbScript": "vite-node scripts/osm2pgsql/generateBuildDbScript.ts",
+    "generate:summaryCache": "vite-node scripts/generateSummaryTileCache.ts"
   },
   "keywords": [
     "OpenStreetMap",

scripts/generateSummaryTileCache.ts

@@ -0,0 +1,88 @@
+import Script from "./Script"
+import Constants from "../src/Models/Constants"
+import { Utils } from "../src/Utils"
+import ScriptUtils from "./ScriptUtils"
+import { SummaryTileSource } from "../src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource"
+import { Tiles } from "../src/Models/TileRange"
+import { Feature, Point } from "geojson"
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
+
+class GenerateSummaryTileCache extends Script {
+    private readonly url: string
+    private readonly cacheDir = "./data/summary_cache/"
+
+    constructor() {
+        super("Generates all summary tiles up to z=12")
+        this.url = Constants.SummaryServer
+        if (!existsSync(this.cacheDir)) {
+            mkdirSync(this.cacheDir)
+        }
+    }
+
+    async fetchTile(z: number, x: number, y: number, layersSummed: string): Promise<Feature<Point>> {
+        const index = Tiles.tile_index(z, x, y)
+        let feature: Feature<Point> | any = (await SummaryTileSource.downloadTile(index, this.url, layersSummed).AsPromise())[0]
+        if (!feature) {
+            feature = { properties: { total: 0 } }
+        }
+        delete feature.properties.layers
+        delete feature.properties.id
+        delete feature.properties.total_metric
+        delete feature.properties.summary // is simply "yes" for rendering
+        delete feature.properties.lon
+        delete feature.properties.lat
+        return feature
+    }
+
+    async fetchTileRecursive(z: number, x: number, y: number, layersSummed: string): Promise<Feature<Point>> {
+        const index = Tiles.tile_index(z, x, y)
+        const path = this.cacheDir + "tile_" + z + "_" + x + "_" + y + ".json"
+        if (existsSync(path)) {
+            return JSON.parse(readFileSync(path, "utf8"))
+        }
+        let feature: Feature<Point>
+        if (z >= 14) {
+            feature = await this.fetchTile(z, x, y, layersSummed)
+        } else {
+            const parts = await Promise.all([
+                this.fetchTileRecursive(z + 1, x * 2, y * 2, layersSummed),
+                this.fetchTileRecursive(z + 1, x * 2 + 1, y * 2, layersSummed),
+                this.fetchTileRecursive(z + 1, x * 2, y * 2 + 1, layersSummed),
+                this.fetchTileRecursive(z + 1, x * 2 + 1, y * 2 + 1, layersSummed)])
+            const sum = this.sumTotals(parts.map(f => f.properties))
+            feature = <Feature<Point>>{
+                type: "Feature",
+                properties: sum,
+                geometry: {
+                    type: "Point",
+                    coordinates: Tiles.centerPointOf(z, x, y),
+                },
+            }
+        }
+
+        writeFileSync(path, JSON.stringify(feature))
+        return feature
+    }
+
+    sumTotals(properties: Record<string, number>[]): Record<string, number> {
+        const sum: Record<string, number> = {}
+        for (const property of properties) {
+            for (const k in property) {
+                sum[k] = (sum[k] ?? 0) + property[k]
+            }
+        }
+        return sum
+    }
+
+
+    async main(args: string[]): Promise<void> {
+
+        const layers = await Utils.downloadJson<{ layers: string[], meta: object }>(this.url + "/status.json")
+        const layersSummed = layers.layers.map(l => encodeURIComponent(l)).join("+")
+        const r = await this.fetchTileRecursive(12, 2084, 1367, layersSummed)
+        console.log(r)
+
+    }
+}
+
+new GenerateSummaryTileCache().run()

src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource.ts

@@ -84,60 +84,8 @@ export class SummaryTileSource extends DynamicTileSource {
             zoomRounded,
             0, // minzoom
             (tileIndex) => {
-                const [z, x, y] = Tiles.tile_from_index(tileIndex)
+                const features = SummaryTileSource.downloadTile(tileIndex, cacheserver, layersSummed)
-                let coordinates = Tiles.centerPointOf(z, x, y)
+                const [z] = Tiles.tile_from_index(tileIndex)
-                const url = `${cacheserver}/${layersSummed}/${z}/${x}/${y}.json`
-                const count = UIEventSource.FromPromiseWithErr(Utils.downloadJson(url))
-                const features: Store<Feature<Point>[]> = count.mapD((count) => {
-                    if (count["error"] !== undefined) {
-                        console.error(
-                            "Could not download count for tile",
-                            z,
-                            x,
-                            y,
-                            "due to",
-                            count["error"]
-                        )
-                        return SummaryTileSource.empty
-                    }
-                    const counts = count["success"]
-                    const total = Number(counts?.["total"] ?? 0)
-                    if (total === 0) {
-                        return SummaryTileSource.empty
-                    }
-                    const lat = counts["lat"]
-                    const lon = counts["lon"]
-                    const tileBbox = new BBox(Tiles.tile_bounds_lon_lat(z, x, y))
-                    if (!tileBbox.contains([lon, lat])) {
-                        console.error(
-                            "Average coordinate is outside of bbox!?",
-                            lon,
-                            lat,
-                            tileBbox,
-                            counts,
-                            url
-                        )
-                    } else {
-                        coordinates = [lon, lat]
-                    }
-                    return [
-                        {
-                            type: "Feature",
-                            properties: {
-                                id: "summary_" + tileIndex,
-                                summary: "yes",
-                                ...counts,
-                                total,
-                                total_metric: Utils.numberWithMetricPrefix(total),
-                                layers: layersSummed,
-                            },
-                            geometry: {
-                                type: "Point",
-                                coordinates,
-                            },
-                        },
-                    ]
-                })
                 return new StaticFeatureSource(
                     features.map(
                         (f) => {
@@ -154,4 +102,61 @@ export class SummaryTileSource extends DynamicTileSource {
             { ...options, zDiff }
         )
     }
+
+    public static downloadTile(tileIndex: number, cacheserver: string, layersSummed: string): Store<Feature<Point>[]>{
+        const [z, x, y] = Tiles.tile_from_index(tileIndex)
+        let coordinates = Tiles.centerPointOf(z, x, y)
+        const url = `${cacheserver}/${layersSummed}/${z}/${x}/${y}.json`
+        const count = UIEventSource.FromPromiseWithErr(Utils.downloadJson(url))
+        return count.mapD((count) => {
+            if (count["error"] !== undefined) {
+                console.error(
+                    "Could not download count for tile",
+                    z,
+                    x,
+                    y,
+                    "due to",
+                    count["error"]
+                )
+                return SummaryTileSource.empty
+            }
+            const counts = count["success"]
+            const total = Number(counts?.["total"] ?? 0)
+            if (total === 0) {
+                return SummaryTileSource.empty
+            }
+            const lat = counts["lat"]
+            const lon = counts["lon"]
+            const tileBbox = new BBox(Tiles.tile_bounds_lon_lat(z, x, y))
+            if (!tileBbox.contains([lon, lat])) {
+                console.error(
+                    "Average coordinate is outside of bbox!?",
+                    lon,
+                    lat,
+                    tileBbox,
+                    counts,
+                    url
+                )
+            } else {
+                coordinates = [lon, lat]
+            }
+            return [
+                {
+                    type: "Feature",
+                    properties: {
+                        id: "summary_" + tileIndex,
+                        summary: "yes",
+                        ...counts,
+                        total,
+                        total_metric: Utils.numberWithMetricPrefix(total),
+                        layers: layersSummed,
+                    },
+                    geometry: {
+                        type: "Point",
+                        coordinates,
+                    },
+                },
+            ]
+        })
+    }
 }