MapComplete/scripts/velopark/veloParkToGeojson.ts

import Script from "../Script"
import fs from "fs"
import LinkedDataLoader from "../../src/Logic/Web/LinkedDataLoader"
import { Utils } from "../../src/Utils"
import { Feature } from "geojson"
import { BBox } from "../../src/Logic/BBox"
import { Overpass } from "../../src/Logic/Osm/Overpass"
import { RegexTag } from "../../src/Logic/Tags/RegexTag"
import { ImmutableStore } from "../../src/Logic/UIEventSource"
import Constants from "../../src/Models/Constants"

class VeloParkToGeojson extends Script {
    constructor() {
        super(
            "Downloads the latest Velopark data and converts it to a geojson, which will be saved at the current directory"
        )
    }

    private static exportGeojsonTo(filename: string, features: Feature[], extension = ".geojson") {
        const file = filename + "_" + /*new Date().toISOString() + */extension
        fs.writeFileSync(file,
            JSON.stringify(
                extension === ".geojson" ? {
                    type: "FeatureCollection",
                    features
                } : features,
                null,
                "    "
            )
        )
        console.log("Written", file, "("+features.length," features)")
    }

    public static sumProperties(data: object, addTo: Record<string, Set<string>>) {
        delete data["@context"]
        for (const k in data) {
            if (k === "@graph") {
                for (const obj of data["@graph"]) {
                    this.sumProperties(obj, addTo)
                }
                continue
            }
            if (addTo[k] === undefined) {
                addTo[k] = new Set<string>()
            }
            addTo[k].add(data[k])
        }

    }

    private static async downloadData() {
        console.log("Downloading velopark data")
        // Download data for NIS-code 1000. 1000 means: all of belgium
        const url = "https://www.velopark.be/api/parkings/1000"
        const allVeloparkRaw: { url: string }[] = await Utils.downloadJson(url)

        let failed = 0
        console.log("Got", allVeloparkRaw.length, "items")
        const allVelopark: Feature[] = []
        const allProperties = {}
        for (let i = 0; i < allVeloparkRaw.length; i++) {
            const f = allVeloparkRaw[i]
            console.log("Handling", i + "/" + allVeloparkRaw.length)
            try {
                const cachePath = "/home/pietervdvn/data/velopark_cache/" + f.url.replace(/[/:.]/g, "_")
                if (!fs.existsSync(cachePath)) {
                    const data = await Utils.downloadJson(f.url)
                    fs.writeFileSync(cachePath, JSON.stringify(data), "utf-8")
                    console.log("Saved a backup to", cachePath)
                }

                this.sumProperties(JSON.parse(fs.readFileSync(cachePath, "utf-8")), allProperties)

                const linkedData = await LinkedDataLoader.fetchVeloparkEntry(f.url)
                for (const sectionId in linkedData) {
                    const sectionInfo = linkedData[sectionId]
                    if (Object.keys(sectionInfo).length === 0) {
                        console.warn("No result for", f.url)
                    }
                    sectionInfo["ref:velopark"] = [sectionId ?? f.url]
                    allVelopark.push(sectionInfo)
                }
            } catch (e) {
                console.error("Loading ", f.url, " failed due to", e)
                failed++

            }
        }
        console.log("Fetching data done, got ", allVelopark.length + "/" + allVeloparkRaw.length, "failed:", failed)
        VeloParkToGeojson.exportGeojsonTo("velopark_all.geojson", allVelopark)
        for (const k in allProperties) {
            allProperties[k] = Array.from(allProperties[k])
        }

        fs.writeFileSync("all_properties_mashup.json", JSON.stringify(allProperties, null, "  "))

        return allVelopark
    }

    private static loadFromFile(): Feature[] {
        return JSON.parse(fs.readFileSync("velopark_all.geojson", "utf-8")).features
    }

    private static exportExtraAmenities(allVelopark: Feature[]) {
        const amenities: Record<string, Feature[]> = {}

        for (const bikeparking of allVelopark) {
            const props = bikeparking.properties
            if (!props["fixme_nearby_amenity"]) {
                continue
            }
            if (props["fixme_nearby_amenity"]?.endsWith("CameraSurveillance")) {
                delete props["fixme_nearby_amenity"]
                continue
            }
            const amenity = props["fixme_nearby_amenity"].split("#")[1]
            if (!amenities[amenity]) {
                amenities[amenity] = []
            }
            amenities[amenity].push(bikeparking)
        }

        for (const k in amenities) {
            this.exportGeojsonTo("velopark_amenity_" + k + ".geojson", amenities[k])
        }
    }

    private static async createDiff(allVelopark: Feature[]) {
        const bboxBelgium = new BBox([
            [2.51357303225, 49.5294835476],
            [6.15665815596, 51.4750237087]
        ])

        const alreadyLinkedQuery = new Overpass(
            new RegexTag("ref:velopark", /.+/),
            [],
            Constants.defaultOverpassUrls[0],
            new ImmutableStore(60 * 5),
            false
        )
        const alreadyLinkedFeatures = (await alreadyLinkedQuery.queryGeoJson(bboxBelgium))[0]
        const seenIds = new Set<string>(
            alreadyLinkedFeatures.features.map((f) => f.properties?.["ref:velopark"])
        )
        this.exportGeojsonTo("osm_with_velopark_link", <Feature[]> alreadyLinkedFeatures.features)
        console.log("OpenStreetMap contains", seenIds.size, "bicycle parkings with a velopark ref")

        const features: Feature[] = allVelopark.filter((f) => !seenIds.has(f.properties["ref:velopark"]))
        VeloParkToGeojson.exportGeojsonTo("velopark_nonsynced", features)

        const allProperties = new Set<string>()
        for (const feature of features) {
            Object.keys(feature).forEach((k) => allProperties.add(k))
        }
        allProperties.delete("ref:velopark")
        for (const feature of features) {
            allProperties.forEach((k) => {
                delete feature[k]
            })
        }

        this.exportGeojsonTo("velopark_nonsynced_id_only", features)


    }

    async main(args: string[]): Promise<void> {
        const allVelopark = VeloParkToGeojson.loadFromFile() //         VeloParkToGeojson.downloadData()
        console.log("Got", allVelopark.length, " items")
        //   VeloParkToGeojson.exportExtraAmenities(allVelopark)
        await VeloParkToGeojson.createDiff(allVelopark)
        console.log("Use vite-node script/velopark/compare to compare the results and generate a diff file")
    }
}

new VeloParkToGeojson().run()