diff --git a/package.json b/package.json index 32a9f05ed..c289219f1 100644 --- a/package.json +++ b/package.json @@ -119,7 +119,8 @@ "server:ldjson": "vite-node scripts/serverLdScrape.ts", "sever:studio": "vite-node scripts/studioServer -- /root/git/MapComplete/assets", "server:errorreport": "vite-node scripts/serverErrorReport.ts -- /root/error_reports/", - "generate:buildDbScript": "vite-node scripts/osm2pgsql/generateBuildDbScript.ts" + "generate:buildDbScript": "vite-node scripts/osm2pgsql/generateBuildDbScript.ts", + "generate:summaryCache": "vite-node scripts/generateSummaryTileCache.ts" }, "keywords": [ "OpenStreetMap", diff --git a/scripts/generateSummaryTileCache.ts b/scripts/generateSummaryTileCache.ts new file mode 100644 index 000000000..bab208986 --- /dev/null +++ b/scripts/generateSummaryTileCache.ts @@ -0,0 +1,88 @@ +import Script from "./Script" +import Constants from "../src/Models/Constants" +import { Utils } from "../src/Utils" +import ScriptUtils from "./ScriptUtils" +import { SummaryTileSource } from "../src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource" +import { Tiles } from "../src/Models/TileRange" +import { Feature, Point } from "geojson" +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs" + +class GenerateSummaryTileCache extends Script { + private readonly url: string + private readonly cacheDir = "./data/summary_cache/" + + constructor() { + super("Generates all summary tiles up to z=12") + this.url = Constants.SummaryServer + if (!existsSync(this.cacheDir)) { + mkdirSync(this.cacheDir) + } + } + + async fetchTile(z: number, x: number, y: number, layersSummed: string): Promise> { + const index = Tiles.tile_index(z, x, y) + let feature: Feature | any = (await SummaryTileSource.downloadTile(index, this.url, layersSummed).AsPromise())[0] + if (!feature) { + feature = { properties: { total: 0 } } + } + delete feature.properties.layers + delete feature.properties.id + delete feature.properties.total_metric + delete feature.properties.summary // is simply "yes" for rendering + delete feature.properties.lon + delete feature.properties.lat + return feature + } + + async fetchTileRecursive(z: number, x: number, y: number, layersSummed: string): Promise> { + const index = Tiles.tile_index(z, x, y) + const path = this.cacheDir + "tile_" + z + "_" + x + "_" + y + ".json" + if (existsSync(path)) { + return JSON.parse(readFileSync(path, "utf8")) + } + let feature: Feature + if (z >= 14) { + feature = await this.fetchTile(z, x, y, layersSummed) + } else { + const parts = await Promise.all([ + this.fetchTileRecursive(z + 1, x * 2, y * 2, layersSummed), + this.fetchTileRecursive(z + 1, x * 2 + 1, y * 2, layersSummed), + this.fetchTileRecursive(z + 1, x * 2, y * 2 + 1, layersSummed), + this.fetchTileRecursive(z + 1, x * 2 + 1, y * 2 + 1, layersSummed)]) + const sum = this.sumTotals(parts.map(f => f.properties)) + feature = >{ + type: "Feature", + properties: sum, + geometry: { + type: "Point", + coordinates: Tiles.centerPointOf(z, x, y), + }, + } + } + + writeFileSync(path, JSON.stringify(feature)) + return feature + } + + sumTotals(properties: Record[]): Record { + const sum: Record = {} + for (const property of properties) { + for (const k in property) { + sum[k] = (sum[k] ?? 0) + property[k] + } + } + return sum + } + + + async main(args: string[]): Promise { + + const layers = await Utils.downloadJson<{ layers: string[], meta: object }>(this.url + "/status.json") + const layersSummed = layers.layers.map(l => encodeURIComponent(l)).join("+") + const r = await this.fetchTileRecursive(12, 2084, 1367, layersSummed) + console.log(r) + + } +} + +new GenerateSummaryTileCache().run() diff --git a/src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource.ts index 0fd5e3ea0..7e1643e47 100644 --- a/src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource.ts +++ b/src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource.ts @@ -84,60 +84,8 @@ export class SummaryTileSource extends DynamicTileSource { zoomRounded, 0, // minzoom (tileIndex) => { - const [z, x, y] = Tiles.tile_from_index(tileIndex) - let coordinates = Tiles.centerPointOf(z, x, y) - const url = `${cacheserver}/${layersSummed}/${z}/${x}/${y}.json` - const count = UIEventSource.FromPromiseWithErr(Utils.downloadJson(url)) - const features: Store[]> = count.mapD((count) => { - if (count["error"] !== undefined) { - console.error( - "Could not download count for tile", - z, - x, - y, - "due to", - count["error"] - ) - return SummaryTileSource.empty - } - const counts = count["success"] - const total = Number(counts?.["total"] ?? 0) - if (total === 0) { - return SummaryTileSource.empty - } - const lat = counts["lat"] - const lon = counts["lon"] - const tileBbox = new BBox(Tiles.tile_bounds_lon_lat(z, x, y)) - if (!tileBbox.contains([lon, lat])) { - console.error( - "Average coordinate is outside of bbox!?", - lon, - lat, - tileBbox, - counts, - url - ) - } else { - coordinates = [lon, lat] - } - return [ - { - type: "Feature", - properties: { - id: "summary_" + tileIndex, - summary: "yes", - ...counts, - total, - total_metric: Utils.numberWithMetricPrefix(total), - layers: layersSummed, - }, - geometry: { - type: "Point", - coordinates, - }, - }, - ] - }) + const features = SummaryTileSource.downloadTile(tileIndex, cacheserver, layersSummed) + const [z] = Tiles.tile_from_index(tileIndex) return new StaticFeatureSource( features.map( (f) => { @@ -154,4 +102,61 @@ export class SummaryTileSource extends DynamicTileSource { { ...options, zDiff } ) } + + public static downloadTile(tileIndex: number, cacheserver: string, layersSummed: string): Store[]>{ + const [z, x, y] = Tiles.tile_from_index(tileIndex) + let coordinates = Tiles.centerPointOf(z, x, y) + const url = `${cacheserver}/${layersSummed}/${z}/${x}/${y}.json` + const count = UIEventSource.FromPromiseWithErr(Utils.downloadJson(url)) + return count.mapD((count) => { + if (count["error"] !== undefined) { + console.error( + "Could not download count for tile", + z, + x, + y, + "due to", + count["error"] + ) + return SummaryTileSource.empty + } + const counts = count["success"] + const total = Number(counts?.["total"] ?? 0) + if (total === 0) { + return SummaryTileSource.empty + } + const lat = counts["lat"] + const lon = counts["lon"] + const tileBbox = new BBox(Tiles.tile_bounds_lon_lat(z, x, y)) + if (!tileBbox.contains([lon, lat])) { + console.error( + "Average coordinate is outside of bbox!?", + lon, + lat, + tileBbox, + counts, + url + ) + } else { + coordinates = [lon, lat] + } + return [ + { + type: "Feature", + properties: { + id: "summary_" + tileIndex, + summary: "yes", + ...counts, + total, + total_metric: Utils.numberWithMetricPrefix(total), + layers: layersSummed, + }, + geometry: { + type: "Point", + coordinates, + }, + }, + ] + }) + } }