Improve script to generate series

Docs/Tools/cache_centerpoints.sh

@@ -1,13 +0,0 @@
-#! /usr/bin/env      bash
-
-ts-node GenerateSeries.ts
-# Move to the root of the repo
-cd ../..
-cd ../MapComplete-data
-git pull
-cd -
-ts-node scripts/slice.ts ./Docs/Tools/centerpoints.geojson 8 ../MapComplete-data/mapcomplete-changes/
-cd -
-git add mapcomplete-changes/*
-git commit -am "New changeset data"
-git push

UI/StatisticsGUI.ts

@@ -184,7 +184,7 @@ class StatisticsForOverviewFile extends Combine {
 
 export default class StatisticsGUI extends VariableUiElement {
     private static readonly homeUrl =
-        "https://raw.githubusercontent.com/pietervdvn/MapComplete/develop/Docs/Tools/stats/"
+        "https://raw.githubusercontent.com/pietervdvn/MapComplete-data/main/changeset-metadata/"
     private static readonly stats_files = "file-overview.json"
 
     constructor() {

package.json

@@ -31,12 +31,11 @@
     "generate:layeroverview": "ts-node scripts/generateLayerOverview.ts",
     "generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail",
     "query:licenses": "ts-node scripts/generateLicenseInfo.ts --query",
-    "generate:report": "cd Docs/Tools && ./compileStats.sh && git commit . -m 'New statistics ands graphs' && git push",
     "generate:contributor-list": "ts-node scripts/generateContributors.ts",
     "generate:schemas": "ts2json-schema -p Models/ThemeConfig/Json/ -o Docs/Schemas/ -t tsconfig.json -R . -m \".*ConfigJson\" && ts-node scripts/fixSchemas.ts ",
     "generate:service-worker": "tsc service-worker.ts && git_hash=$(git rev-parse HEAD) && sed -i'.bkp' \"s/GITHUB-COMMIT/$git_hash/\" service-worker.js && rm service-worker.js.bkp",
     "optimize-images": "cd assets/generated/ &&  find -name '*.png' -exec optipng '{}' \\; && echo 'PNGs are optimized'",
-    "generate:graphs": "ts-node Docs/Tools/GenerateSeries.ts",
+    "generate:stats": "ts-node scripts/GenerateSeries.ts",
     "reset:layeroverview": "echo {\\\"layers\\\":[], \\\"themes\\\":[]} > ./assets/generated/known_layers_and_themes.json && echo {\\\"layers\\\": []} > ./assets/generated/known_layers.json && rm -f ./assets/generated/layers/*.json && rm -f ./assets/generated/themes/*.json && npm run generate:layeroverview && ts-node scripts/generateLayerOverview.ts --force",
     "generate": "mkdir -p ./assets/generated; npm run generate:licenses; npm run generate:images; npm run generate:charging-stations; npm run generate:translations; npm run reset:layeroverview; npm run generate:service-worker",
     "generate:charging-stations": "cd ./assets/layers/charging_station && ts-node csvToJson.ts && cd -",

scripts/GenerateSeries.ts

@@ -1,8 +1,11 @@
-import { existsSync, mkdirSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
+import fs, { existsSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
-import ScriptUtils from "../../scripts/ScriptUtils"
+import ScriptUtils from "./ScriptUtils"
-import { Utils } from "../../Utils"
+import { Utils } from "../Utils"
-
+import Script from "./Script"
-ScriptUtils.fixUtils()
+import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource"
+import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource"
+import { GeoOperations } from "../Logic/GeoOperations"
+import { Feature, Polygon } from "geojson"
 
 class StatsDownloader {
     private readonly urlTemplate =
@@ -158,7 +161,7 @@ class StatsDownloader {
     }
 }
 
-interface ChangeSetData {
+interface ChangeSetData extends Feature<Polygon> {
     id: number
     type: "Feature"
     geometry: {
@@ -196,56 +199,130 @@ interface ChangeSetData {
     }
 }
 
-async function main(): Promise<void> {
+class GenerateSeries extends Script {
-    if (!existsSync("graphs")) {
+    constructor() {
-        mkdirSync("graphs")
+        super("Downloads metadata about changesets made by MapComplete from OsmCha")
     }
 
-    const targetDir = "Docs/Tools/stats"
+    async main(args: string[]): Promise<void> {
-    let year = 2020
+        const targetDir = args[0] ?? "../MapComplete-data"
-    let month = 5
-    let day = 1
-    if (!isNaN(Number(process.argv[2]))) {
-        year = Number(process.argv[2])
-    }
-    if (!isNaN(Number(process.argv[3]))) {
-        month = Number(process.argv[3])
-    }
 
-    if (!isNaN(Number(process.argv[4]))) {
+        await this.downloadStatistics(targetDir + "/changeset-metadata")
-        day = Number(process.argv[4])
+        await this.generateCenterPoints(
-    }
+            targetDir + "/changeset-metadata",
-
+            targetDir + "/mapcomplete-changes/",
-    do {
+            {
-        try {
+                zoomlevel: 8,
-            await new StatsDownloader(targetDir).DownloadStats(year, month, day)
+            }
-            break
-        } catch (e) {
-            console.log(e)
-        }
-    } while (true)
-    const allPaths = readdirSync(targetDir).filter(
-        (p) => p.startsWith("stats.") && p.endsWith(".json")
-    )
-    let allFeatures: ChangeSetData[] = [].concat(
-        ...allPaths.map(
-            (path) => JSON.parse(readFileSync("Docs/Tools/stats/" + path, "utf-8")).features
         )
-    )
-    allFeatures = allFeatures.filter(
-        (f) =>
-            f?.properties !== undefined &&
-            (f.properties.editor === null ||
-                f.properties.editor.toLowerCase().startsWith("mapcomplete"))
-    )
-
-    allFeatures = allFeatures.filter((f) => f.properties.metadata?.theme !== "EMPTY CS")
-
-    if (process.argv.indexOf("--no-graphs") >= 0) {
-        return
     }
-    const allFiles = readdirSync("Docs/Tools/stats").filter((p) => p.endsWith(".json"))
+
-    writeFileSync("Docs/Tools/stats/file-overview.json", JSON.stringify(allFiles))
+    private async downloadStatistics(targetDir: string) {
+        let year = 2020
+        let month = 5
+        let day = 1
+        if (!isNaN(Number(process.argv[2]))) {
+            year = Number(process.argv[2])
+        }
+        if (!isNaN(Number(process.argv[3]))) {
+            month = Number(process.argv[3])
+        }
+
+        if (!isNaN(Number(process.argv[4]))) {
+            day = Number(process.argv[4])
+        }
+
+        do {
+            try {
+                await new StatsDownloader(targetDir).DownloadStats(year, month, day)
+                break
+            } catch (e) {
+                console.log(e)
+            }
+        } while (true)
+
+        const allFiles = readdirSync(targetDir).filter((p) => p.endsWith(".json"))
+        writeFileSync(targetDir + "file-overview.json", JSON.stringify(allFiles))
+    }
+
+    private generateCenterPoints(
+        sourceDir: string,
+        targetDir: string,
+        options: {
+            zoomlevel: number
+        }
+    ) {
+        const allPaths = readdirSync(sourceDir).filter(
+            (p) => p.startsWith("stats.") && p.endsWith(".json")
+        )
+        let allFeatures: ChangeSetData[] = [].concat(
+            ...allPaths.map(
+                (path) => JSON.parse(readFileSync(sourceDir + "/" + path, "utf-8")).features
+            )
+        )
+        allFeatures = allFeatures.filter(
+            (f) =>
+                f?.properties !== undefined &&
+                (f.properties.editor === null ||
+                    f.properties.editor.toLowerCase().startsWith("mapcomplete"))
+        )
+
+        allFeatures = allFeatures.filter(
+            (f) => f.geometry !== null && f.properties.metadata?.theme !== "EMPTY CS"
+        )
+        allFeatures = allFeatures.filter(
+            (f) =>
+                f?.properties !== undefined &&
+                (f.properties.editor === null ||
+                    f.properties.editor.toLowerCase().startsWith("mapcomplete"))
+        )
+
+        allFeatures = allFeatures.filter((f) => f.properties.metadata?.theme !== "EMPTY CS")
+        const centerpoints = allFeatures.map((f) => GeoOperations.centerpoint(f))
+        console.log("Found", centerpoints.length, " changesets in total")
+        const path = `${targetDir}/all_centerpoints.geojson`
+        /*fs.writeFileSync(
+            path,
+            JSON.stringify(
+                {
+                    type: "FeatureCollection",
+                    features: centerpoints,
+                },
+                null,
+                "  "
+            )
+        )//*/
+        TiledFeatureSource.createHierarchy(StaticFeatureSource.fromGeojson(centerpoints), {
+            minZoomLevel: options.zoomlevel,
+            maxZoomLevel: options.zoomlevel,
+            maxFeatureCount: Number.MAX_VALUE,
+            registerTile: (tile) => {
+                const path = `${targetDir}/tile_${tile.z}_${tile.x}_${tile.y}.geojson`
+                const features = tile.features.data.map((ff) => ff.feature)
+                features.forEach((f) => {
+                    delete f.bbox
+                })
+                fs.writeFileSync(
+                    path,
+                    JSON.stringify(
+                        {
+                            type: "FeatureCollection",
+                            features: features,
+                        },
+                        null,
+                        "  "
+                    )
+                )
+                ScriptUtils.erasableLog(
+                    "Written ",
+                    path,
+                    "which has ",
+                    tile.features.data.length,
+                    "features"
+                )
+            },
+        })
+    }
 }
 
-main().then((_) => console.log("All done!"))
+new GenerateSeries().run()

scripts/ScriptUtils.ts

@@ -11,6 +11,13 @@ export default class ScriptUtils {
         Utils.externalDownloadFunction = ScriptUtils.Download
     }
 
+    /**
+     * Returns all files in a directory, recursively reads subdirectories.
+     * The returned paths include the path given and subdirectories.
+     *
+     * @param path
+     * @param maxDepth
+     */
     public static readDirRecSync(path, maxDepth = 999): string[] {
         const result = []
         if (maxDepth <= 0) {
@@ -46,13 +53,13 @@ export default class ScriptUtils {
         process.stdout.write("\r " + text.join(" ") + "                \r")
     }
 
-    public static sleep(ms) {
+    public static sleep(ms: number, text?: string) {
         if (ms <= 0) {
             process.stdout.write("\r                                       \r")
             return
         }
         return new Promise((resolve) => {
-            process.stdout.write("\r Sleeping for " + ms / 1000 + "s \r")
+            process.stdout.write("\r" + (text ?? "") + " Sleeping for " + ms / 1000 + "s \r")
             setTimeout(resolve, 1000)
         }).then(() => ScriptUtils.sleep(ms - 1000))
     }