MapComplete/scripts/postal_code_tools/openaddressestogeojson.ts

import * as fs from "fs"
import { existsSync, writeFileSync } from "fs"
import * as readline from "readline"
import ScriptUtils from "../ScriptUtils"

/**
 * Converts an open-address CSV file into a big geojson file
 */

async function main(args: string[]) {
    const inputFile = args[0]
    const outputFile = args[1]
    const fileStream = fs.createReadStream(inputFile)
    const perPostalCode = args[2] == "--per-postal-code"
    const rl = readline.createInterface({
        input: fileStream,
        crlfDelay: Infinity,
    })
    // Note: we use the crlfDelay option to recognize all instances of CR LF
    // ('\r\n') in input.txt as a single line break.

    const fields = [
        "EPSG:31370_x",
        "EPSG:31370_y",
        "EPSG:4326_lat",
        "EPSG:4326_lon",
        "address_id",
        "box_number",
        "house_number",
        "municipality_id",
        "municipality_name_de",
        "municipality_name_fr",
        "municipality_name_nl",
        "postcode",
        "postname_fr",
        "postname_nl",
        "street_id",
        "streetname_de",
        "streetname_fr",
        "streetname_nl",
        "region_code",
        "status",
    ]

    let i = 0
    let failed = 0

    let createdFiles: string[] = []
    if (!perPostalCode) {
        fs.writeFileSync(outputFile, "")
    }
    // @ts-ignore
    for await (const line of rl) {
        i++
        if (i % 10000 == 0) {
            ScriptUtils.erasableLog(
                "Converted ",
                i,
                "features (of which ",
                failed,
                "features don't have a coordinate)"
            )
        }
        const data = line.split(",")
        const parsed: any = {}
        for (let i = 0; i < fields.length; i++) {
            const field = fields[i]
            parsed[field] = data[i]
        }
        const lat = Number(parsed["EPSG:4326_lat"])
        const lon = Number(parsed["EPSG:4326_lon"])

        if (parsed["EPSG:31370_x"] === "0.0") {
            failed++
            continue
        }

        delete parsed["EPSG:4326_lat"]
        delete parsed["EPSG:4326_lon"]
        delete parsed["EPSG:31370_x"]
        delete parsed["EPSG:31370_y"]
        delete parsed["EPSG:4326_lat"]
        delete parsed["EPSG:4326_lat"]

        let targetFile = outputFile
        if (perPostalCode) {
            if (parsed["postcode"] === "") {
                continue
            }
            if (isNaN(Number(parsed["postcode"]))) {
                continue
            }
            targetFile = outputFile + "-" + parsed["postcode"] + ".geojson"
            let isFirst = false
            if (!existsSync(targetFile)) {
                writeFileSync(targetFile, '{ "type":"FeatureCollection", "features":[')
                createdFiles.push(targetFile)
                isFirst = true
            }

            if (!isFirst) {
                fs.appendFileSync(targetFile, ",\n")
            }

            fs.appendFileSync(
                targetFile,
                JSON.stringify({
                    type: "Feature",
                    properties: parsed,
                    geometry: {
                        type: "Point",
                        coordinates: [lon, lat],
                    },
                })
            )
        } else {
            fs.appendFileSync(
                outputFile,
                JSON.stringify({
                    type: "Feature",
                    properties: parsed,
                    geometry: {
                        type: "Point",
                        coordinates: [lon, lat],
                    },
                }) + "\n"
            )
        }
    }

    console.log("Closing files...")
    createdFiles.sort()
    for (const createdFile of createdFiles) {
        ScriptUtils.erasableLog("Closing ", createdFile, "and creating convex hull")
        fs.appendFileSync(createdFile, "]}")
    }

    console.log(
        "Done! Converted ",
        i,
        "features (of which ",
        failed,
        "features don't have a coordinate)"
    )
}

let args = [...process.argv]
args.splice(0, 2)

if (args.length == 0) {
    console.log("USAGE: input-csv-file output.newline-delimited-geojson.json [--per-postal-code]")
} else {
    main(args).catch((e) => console.error(e))
}