forked from MapComplete/MapComplete
Scripts: drop broken centerPoint-generation in data script
This commit is contained in:
parent
4b3e1fad4f
commit
38737c32a2
2 changed files with 5 additions and 89 deletions
|
@ -1,4 +1,5 @@
|
||||||
on:
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
schedule:
|
schedule:
|
||||||
- cron: "0 2 * * *"
|
- cron: "0 2 * * *"
|
||||||
|
|
||||||
|
|
|
@ -2,10 +2,7 @@ import { existsSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from
|
||||||
import ScriptUtils from "./ScriptUtils"
|
import ScriptUtils from "./ScriptUtils"
|
||||||
import { Utils } from "../src/Utils"
|
import { Utils } from "../src/Utils"
|
||||||
import Script from "./Script"
|
import Script from "./Script"
|
||||||
import { GeoOperations } from "../src/Logic/GeoOperations"
|
|
||||||
import { Feature, Polygon } from "geojson"
|
import { Feature, Polygon } from "geojson"
|
||||||
import { Tiles } from "../src/Models/TileRange"
|
|
||||||
import { BBox } from "../src/Logic/BBox"
|
|
||||||
|
|
||||||
class StatsDownloader {
|
class StatsDownloader {
|
||||||
private readonly urlTemplate =
|
private readonly urlTemplate =
|
||||||
|
@ -205,13 +202,6 @@ class GenerateSeries extends Script {
|
||||||
const targetDir = args[0] ?? "../../git/MapComplete-data"
|
const targetDir = args[0] ?? "../../git/MapComplete-data"
|
||||||
|
|
||||||
await this.downloadStatistics(targetDir + "/changeset-metadata")
|
await this.downloadStatistics(targetDir + "/changeset-metadata")
|
||||||
this.generateCenterPoints(
|
|
||||||
targetDir + "/changeset-metadata",
|
|
||||||
targetDir + "/mapcomplete-changes/",
|
|
||||||
{
|
|
||||||
zoomlevel: 8,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async downloadStatistics(targetDir: string) {
|
private async downloadStatistics(targetDir: string) {
|
||||||
|
@ -229,94 +219,19 @@ class GenerateSeries extends Script {
|
||||||
day = Number(process.argv[4])
|
day = Number(process.argv[4])
|
||||||
}
|
}
|
||||||
|
|
||||||
do {
|
let downloaded = false
|
||||||
|
while (!downloaded) {
|
||||||
try {
|
try {
|
||||||
await new StatsDownloader(targetDir).DownloadStats(year, month, day)
|
await new StatsDownloader(targetDir).DownloadStats(year, month, day)
|
||||||
break
|
downloaded = true
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.log(e)
|
console.log(e)
|
||||||
}
|
}
|
||||||
} while (true)
|
}
|
||||||
|
|
||||||
const allFiles = readdirSync(targetDir).filter((p) => p.endsWith(".json"))
|
const allFiles = readdirSync(targetDir).filter((p) => p.endsWith(".json"))
|
||||||
writeFileSync(targetDir + "/file-overview.json", JSON.stringify(allFiles))
|
writeFileSync(targetDir + "/file-overview.json", JSON.stringify(allFiles))
|
||||||
}
|
}
|
||||||
|
|
||||||
private generateCenterPoints(
|
|
||||||
sourceDir: string,
|
|
||||||
targetDir: string,
|
|
||||||
options: {
|
|
||||||
zoomlevel: number
|
|
||||||
}
|
|
||||||
) {
|
|
||||||
const allPaths = readdirSync(sourceDir).filter(
|
|
||||||
(p) => p.startsWith("stats.") && p.endsWith(".json")
|
|
||||||
)
|
|
||||||
let allFeatures: ChangeSetData[] = allPaths.flatMap(
|
|
||||||
(path) => JSON.parse(readFileSync(sourceDir + "/" + path, "utf-8")).features
|
|
||||||
)
|
|
||||||
allFeatures = allFeatures.filter(
|
|
||||||
(f) =>
|
|
||||||
f?.properties !== undefined &&
|
|
||||||
(f.properties.editor === null ||
|
|
||||||
f.properties.editor.toLowerCase().startsWith("mapcomplete"))
|
|
||||||
)
|
|
||||||
|
|
||||||
allFeatures = allFeatures.filter(
|
|
||||||
(f) => f.geometry !== null && f.properties.metadata?.theme !== "EMPTY CS"
|
|
||||||
)
|
|
||||||
allFeatures = allFeatures.filter(
|
|
||||||
(f) =>
|
|
||||||
f?.properties !== undefined &&
|
|
||||||
(f.properties.editor === null ||
|
|
||||||
f.properties.editor.toLowerCase().startsWith("mapcomplete"))
|
|
||||||
)
|
|
||||||
|
|
||||||
allFeatures = allFeatures.filter(
|
|
||||||
(f) => f.properties.metadata?.theme !== "EMPTY CS" && f.geometry.coordinates.length > 0
|
|
||||||
)
|
|
||||||
const centerpointsAll = allFeatures.map((f) => {
|
|
||||||
const centerpoint = GeoOperations.centerpoint(f)
|
|
||||||
const c = centerpoint.geometry.coordinates
|
|
||||||
// OsmCha doesn't adhere to the Geojson standard and uses `lat` `lon` as coordinates instead of `lon`, `lat`
|
|
||||||
centerpoint.geometry.coordinates = [c[1], c[0]]
|
|
||||||
return centerpoint
|
|
||||||
})
|
|
||||||
const centerpoints = centerpointsAll.filter((p) => {
|
|
||||||
const bbox = BBox.get(p)
|
|
||||||
if (bbox.minLat === -90 && bbox.maxLat === -90) {
|
|
||||||
// Due to some bug somewhere, those invalid bboxes might appear if the latitude is < 90
|
|
||||||
// This crashes the 'spreadIntoBBoxes
|
|
||||||
// As workaround, we simply ignore them for now
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
})
|
|
||||||
console.log("Found", centerpoints.length, " changesets in total")
|
|
||||||
|
|
||||||
const perBbox = GeoOperations.spreadIntoBboxes(centerpoints, options.zoomlevel)
|
|
||||||
|
|
||||||
for (const [tileNumber, features] of perBbox) {
|
|
||||||
const [z, x, y] = Tiles.tile_from_index(tileNumber)
|
|
||||||
const path = `${targetDir}/tile_${z}_${x}_${y}.geojson`
|
|
||||||
features.forEach((f) => {
|
|
||||||
delete f.bbox
|
|
||||||
})
|
|
||||||
writeFileSync(
|
|
||||||
path,
|
|
||||||
JSON.stringify(
|
|
||||||
{
|
|
||||||
type: "FeatureCollection",
|
|
||||||
features: features,
|
|
||||||
},
|
|
||||||
null,
|
|
||||||
" "
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
ScriptUtils.erasableLog("Written ", path, "which has ", features.length, "features")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
new GenerateSeries().run()
|
new GenerateSeries().run()
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue