forked from MapComplete/MapComplete
264 lines
10 KiB
TypeScript
264 lines
10 KiB
TypeScript
import Script from "../Script"
|
|
import fs from "fs"
|
|
import LinkedDataLoader from "../../src/Logic/Web/LinkedDataLoader"
|
|
import { Utils } from "../../src/Utils"
|
|
import { Feature, FeatureCollection, Point } from "geojson"
|
|
import { BBox } from "../../src/Logic/BBox"
|
|
import { Overpass } from "../../src/Logic/Osm/Overpass"
|
|
import { RegexTag } from "../../src/Logic/Tags/RegexTag"
|
|
import { ImmutableStore } from "../../src/Logic/UIEventSource"
|
|
import Constants from "../../src/Models/Constants"
|
|
import { MaprouletteStatus } from "../../src/Logic/Maproulette"
|
|
|
|
class VeloParkToGeojson extends Script {
|
|
constructor() {
|
|
super(
|
|
"Downloads the latest Velopark data and converts it to a geojson, which will be saved at the current directory"
|
|
)
|
|
}
|
|
|
|
private static exportGeojsonTo(filename: string, features: Feature[], extension = ".geojson") {
|
|
const file = filename + "_" + /*new Date().toISOString() + */ extension
|
|
fs.writeFileSync(
|
|
file,
|
|
JSON.stringify(
|
|
extension === ".geojson"
|
|
? {
|
|
type: "FeatureCollection",
|
|
features,
|
|
}
|
|
: features,
|
|
null,
|
|
" "
|
|
)
|
|
)
|
|
console.log("Written", file, "(" + features.length, " features)")
|
|
}
|
|
|
|
private static async downloadDataFor(url: string): Promise<Feature[]> {
|
|
const cachePath =
|
|
"/home/pietervdvn/data/velopark_cache_refined/" + url.replace(/[/:.]/g, "_")
|
|
if (fs.existsSync(cachePath)) {
|
|
return JSON.parse(fs.readFileSync(cachePath, "utf8"))
|
|
}
|
|
console.log("Fetching data for", url)
|
|
|
|
const linkedData = await LinkedDataLoader.fetchVeloparkEntry(url)
|
|
const allVelopark: Feature[] = []
|
|
if (linkedData.length > 1) {
|
|
console.log("Detected multiple sections in:", url)
|
|
}
|
|
for (const sectionId in linkedData) {
|
|
const sectionInfo = linkedData[sectionId]
|
|
if (Object.keys(sectionInfo).length === 0) {
|
|
console.warn("No result for", url)
|
|
}
|
|
if (!sectionInfo.geometry?.["coordinates"]) {
|
|
throw "Invalid properties!"
|
|
}
|
|
allVelopark.push(sectionInfo)
|
|
}
|
|
fs.writeFileSync(cachePath, JSON.stringify(allVelopark), "utf8")
|
|
return allVelopark
|
|
}
|
|
|
|
private static async downloadData() {
|
|
console.log("Downloading velopark data")
|
|
// Download data for NIS-code 1000. 1000 means: all of belgium
|
|
const url = "https://www.velopark.be/api/parkings/1000"
|
|
const allVeloparkRaw = await Utils.downloadJson<{ url: string }[]>(url)
|
|
// Example multi-entry: https://data.velopark.be/data/Stad-Izegem_IZE_015
|
|
let failed = 0
|
|
console.log("Got", allVeloparkRaw.length, "items")
|
|
const allVelopark: Feature[] = []
|
|
const batchSize = 50
|
|
for (let i = 0; i < allVeloparkRaw.length; i += batchSize) {
|
|
await Promise.all(
|
|
Utils.TimesT(batchSize, (j) => j).map(async (j) => {
|
|
const f = allVeloparkRaw[i + j]
|
|
if (!f) {
|
|
return
|
|
}
|
|
try {
|
|
const sections: Feature[] = await VeloParkToGeojson.downloadDataFor(f.url)
|
|
allVelopark.push(...sections)
|
|
} catch (e) {
|
|
console.error("Loading ", f.url, " failed due to", e)
|
|
failed++
|
|
}
|
|
})
|
|
)
|
|
console.log("Batch complete:", i)
|
|
}
|
|
console.log(
|
|
"Fetching data done, got ",
|
|
allVelopark.length + "/" + allVeloparkRaw.length,
|
|
"failed:",
|
|
failed
|
|
)
|
|
VeloParkToGeojson.exportGeojsonTo("velopark_all", allVelopark)
|
|
|
|
return allVelopark
|
|
}
|
|
|
|
private static loadFromFile(maxCacheAgeSeconds = 24 * 60 * 60): Feature[] | null {
|
|
const path = "velopark_all.geojson"
|
|
if (!fs.existsSync(path)) {
|
|
return null
|
|
}
|
|
// Millis since epoch
|
|
const mtime: number = fs.statSync(path).mtime.getTime()
|
|
const stalenessSeconds = (new Date().getTime() - mtime) / 1000
|
|
if (stalenessSeconds > maxCacheAgeSeconds) {
|
|
return null
|
|
}
|
|
|
|
return JSON.parse(fs.readFileSync(path, "utf-8")).features
|
|
}
|
|
|
|
private static exportExtraAmenities(allVelopark: Feature[]) {
|
|
const amenities: Record<string, Feature[]> = {}
|
|
|
|
for (const bikeparking of allVelopark) {
|
|
const props = bikeparking.properties
|
|
if (!props["fixme_nearby_amenity"]) {
|
|
continue
|
|
}
|
|
if (props["fixme_nearby_amenity"]?.endsWith("CameraSurveillance")) {
|
|
delete props["fixme_nearby_amenity"]
|
|
continue
|
|
}
|
|
const amenity = props["fixme_nearby_amenity"].split("#")[1]
|
|
if (!amenities[amenity]) {
|
|
amenities[amenity] = []
|
|
}
|
|
amenities[amenity].push(bikeparking)
|
|
}
|
|
|
|
for (const k in amenities) {
|
|
this.exportGeojsonTo("velopark_amenity_" + k + ".geojson", amenities[k])
|
|
}
|
|
}
|
|
|
|
private static async fetchMapRouletteClosedItems() {
|
|
const challenges = ["https://maproulette.org/api/v2/challenge/view/43282"]
|
|
const solvedRefs: Set<string> = new Set<string>()
|
|
for (const url of challenges) {
|
|
const data = await Utils.downloadJson<
|
|
FeatureCollection<
|
|
Point,
|
|
{
|
|
mr_taskId: string
|
|
"ref:velopark": string
|
|
mr_taskStatus: MaprouletteStatus
|
|
mr_responses: string | undefined
|
|
}
|
|
>
|
|
>(url)
|
|
for (const challenge of data.features) {
|
|
const status = challenge.properties.mr_taskStatus
|
|
const isClosed =
|
|
status === "Fixed" ||
|
|
status === "False_positive" ||
|
|
status === "Already fixed" ||
|
|
status === "Too_Hard" ||
|
|
status === "Deleted"
|
|
if (isClosed) {
|
|
const ref = challenge.properties["ref:velopark"]
|
|
solvedRefs.add(ref)
|
|
}
|
|
}
|
|
}
|
|
console.log("Detected", solvedRefs, "as closed on mapRoulette")
|
|
return solvedRefs
|
|
}
|
|
|
|
/**
|
|
* Creates an extra version where all bicycle parkings which are already linked are removed.
|
|
* Fetches the latest OSM-data from overpass
|
|
* @param allVelopark
|
|
* @private
|
|
*/
|
|
private static async createDiff(allVelopark: Feature[]) {
|
|
console.log("Creating diff...")
|
|
const bboxBelgium = new BBox([
|
|
[2.51357303225, 49.5294835476],
|
|
[6.15665815596, 51.4750237087],
|
|
])
|
|
|
|
const alreadyLinkedQuery = new Overpass(
|
|
new RegexTag("ref:velopark", /.+/),
|
|
[],
|
|
Constants.defaultOverpassUrls[0],
|
|
new ImmutableStore(60 * 5),
|
|
false
|
|
)
|
|
const alreadyLinkedFeatures = (await alreadyLinkedQuery.queryGeoJson(bboxBelgium))[0]
|
|
const seenIds = new Set<string>(
|
|
alreadyLinkedFeatures.features.map((f) => f.properties?.["ref:velopark"])
|
|
)
|
|
this.exportGeojsonTo("osm_with_velopark_link", <Feature[]>alreadyLinkedFeatures.features)
|
|
console.log("OpenStreetMap contains", seenIds.size, "bicycle parkings with a velopark ref")
|
|
|
|
const features: Feature[] = allVelopark.filter(
|
|
(f) => !seenIds.has(f.properties["ref:velopark"])
|
|
)
|
|
VeloParkToGeojson.exportGeojsonTo("velopark_nonsynced", features)
|
|
|
|
const synced = await this.fetchMapRouletteClosedItems()
|
|
const featuresMoreFiltered = features.filter(
|
|
(f) => !synced.has(f.properties["ref:velopark"])
|
|
)
|
|
VeloParkToGeojson.exportGeojsonTo("velopark_nonsynced_nonclosed", featuresMoreFiltered)
|
|
|
|
const featuresMoreFilteredFailed = features.filter((f) =>
|
|
synced.has(f.properties["ref:velopark"])
|
|
)
|
|
VeloParkToGeojson.exportGeojsonTo(
|
|
"velopark_nonsynced_human_import_failed",
|
|
featuresMoreFilteredFailed
|
|
)
|
|
|
|
const allProperties = new Set<string>()
|
|
for (const feature of featuresMoreFiltered) {
|
|
Object.keys(feature.properties).forEach((k) => allProperties.add(k))
|
|
}
|
|
allProperties.delete("ref:velopark")
|
|
for (const feature of featuresMoreFiltered) {
|
|
allProperties.forEach((k) => {
|
|
if (k === "ref:velopark") {
|
|
return
|
|
}
|
|
delete feature.properties[k]
|
|
})
|
|
}
|
|
|
|
this.exportGeojsonTo("velopark_nonsynced_nonclosed_id_only", featuresMoreFiltered)
|
|
}
|
|
|
|
public static async findMultiSection(): Promise<string[]> {
|
|
const url = "https://www.velopark.be/api/parkings/1000"
|
|
const raw = await Utils.downloadJson<{ "@graph": {}[]; url: string }[]>(url)
|
|
const multiEntries: string[] = []
|
|
for (const entry of raw) {
|
|
if (entry["@graph"].length > 1) {
|
|
multiEntries.push(entry.url)
|
|
}
|
|
}
|
|
return multiEntries
|
|
}
|
|
|
|
async main(): Promise<void> {
|
|
// const multiEntries = new Set(await VeloParkToGeojson.findMultiSection())
|
|
const allVelopark =
|
|
VeloParkToGeojson.loadFromFile() ?? (await VeloParkToGeojson.downloadData())
|
|
console.log("Got", allVelopark.length, " items")
|
|
VeloParkToGeojson.exportExtraAmenities(allVelopark)
|
|
await VeloParkToGeojson.createDiff(allVelopark)
|
|
console.log(
|
|
"Use \nvite-node scripts/velopark/compare.ts -- velopark_all_.geojson osm_with_velopark_link_.geojson\n to compare the results and generate a diff file"
|
|
)
|
|
}
|
|
}
|
|
|
|
new VeloParkToGeojson().run()
|