From 38737c32a2451c4c5bbfab7cbc07a5477054ad32 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Thu, 3 Apr 2025 02:13:32 +0200 Subject: [PATCH] Scripts: drop broken centerPoint-generation in data script --- .forgejo/workflows/daily_data_maintenance.yml | 1 + scripts/GenerateSeries.ts | 93 +------------------ 2 files changed, 5 insertions(+), 89 deletions(-) diff --git a/.forgejo/workflows/daily_data_maintenance.yml b/.forgejo/workflows/daily_data_maintenance.yml index 19b2b72f1a..e39f5872ac 100644 --- a/.forgejo/workflows/daily_data_maintenance.yml +++ b/.forgejo/workflows/daily_data_maintenance.yml @@ -1,4 +1,5 @@ on: + workflow_dispatch: schedule: - cron: "0 2 * * *" diff --git a/scripts/GenerateSeries.ts b/scripts/GenerateSeries.ts index af6bc06bb8..e0058160d3 100644 --- a/scripts/GenerateSeries.ts +++ b/scripts/GenerateSeries.ts @@ -2,10 +2,7 @@ import { existsSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from import ScriptUtils from "./ScriptUtils" import { Utils } from "../src/Utils" import Script from "./Script" -import { GeoOperations } from "../src/Logic/GeoOperations" import { Feature, Polygon } from "geojson" -import { Tiles } from "../src/Models/TileRange" -import { BBox } from "../src/Logic/BBox" class StatsDownloader { private readonly urlTemplate = @@ -205,13 +202,6 @@ class GenerateSeries extends Script { const targetDir = args[0] ?? "../../git/MapComplete-data" await this.downloadStatistics(targetDir + "/changeset-metadata") - this.generateCenterPoints( - targetDir + "/changeset-metadata", - targetDir + "/mapcomplete-changes/", - { - zoomlevel: 8, - } - ) } private async downloadStatistics(targetDir: string) { @@ -229,94 +219,19 @@ class GenerateSeries extends Script { day = Number(process.argv[4]) } - do { + let downloaded = false + while (!downloaded) { try { await new StatsDownloader(targetDir).DownloadStats(year, month, day) - break + downloaded = true } catch (e) { console.log(e) } - } while (true) - + } const allFiles = readdirSync(targetDir).filter((p) => p.endsWith(".json")) writeFileSync(targetDir + "/file-overview.json", JSON.stringify(allFiles)) } - private generateCenterPoints( - sourceDir: string, - targetDir: string, - options: { - zoomlevel: number - } - ) { - const allPaths = readdirSync(sourceDir).filter( - (p) => p.startsWith("stats.") && p.endsWith(".json") - ) - let allFeatures: ChangeSetData[] = allPaths.flatMap( - (path) => JSON.parse(readFileSync(sourceDir + "/" + path, "utf-8")).features - ) - allFeatures = allFeatures.filter( - (f) => - f?.properties !== undefined && - (f.properties.editor === null || - f.properties.editor.toLowerCase().startsWith("mapcomplete")) - ) - - allFeatures = allFeatures.filter( - (f) => f.geometry !== null && f.properties.metadata?.theme !== "EMPTY CS" - ) - allFeatures = allFeatures.filter( - (f) => - f?.properties !== undefined && - (f.properties.editor === null || - f.properties.editor.toLowerCase().startsWith("mapcomplete")) - ) - - allFeatures = allFeatures.filter( - (f) => f.properties.metadata?.theme !== "EMPTY CS" && f.geometry.coordinates.length > 0 - ) - const centerpointsAll = allFeatures.map((f) => { - const centerpoint = GeoOperations.centerpoint(f) - const c = centerpoint.geometry.coordinates - // OsmCha doesn't adhere to the Geojson standard and uses `lat` `lon` as coordinates instead of `lon`, `lat` - centerpoint.geometry.coordinates = [c[1], c[0]] - return centerpoint - }) - const centerpoints = centerpointsAll.filter((p) => { - const bbox = BBox.get(p) - if (bbox.minLat === -90 && bbox.maxLat === -90) { - // Due to some bug somewhere, those invalid bboxes might appear if the latitude is < 90 - // This crashes the 'spreadIntoBBoxes - // As workaround, we simply ignore them for now - return false - } - return true - }) - console.log("Found", centerpoints.length, " changesets in total") - - const perBbox = GeoOperations.spreadIntoBboxes(centerpoints, options.zoomlevel) - - for (const [tileNumber, features] of perBbox) { - const [z, x, y] = Tiles.tile_from_index(tileNumber) - const path = `${targetDir}/tile_${z}_${x}_${y}.geojson` - features.forEach((f) => { - delete f.bbox - }) - writeFileSync( - path, - JSON.stringify( - { - type: "FeatureCollection", - features: features, - }, - null, - " " - ) - ) - - ScriptUtils.erasableLog("Written ", path, "which has ", features.length, "features") - } - } } new GenerateSeries().run()