MapComplete/scripts/slice.ts

import * as fs from "fs"
import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource"
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource"
import * as readline from "readline"
import ScriptUtils from "./ScriptUtils"
import { Utils } from "../Utils"

/**
 * This script slices a big newline-delimeted geojson file into tiled geojson
 * It was used to convert the CRAB-data into geojson tiles
 */

async function readFeaturesFromLineDelimitedJsonFile(inputFile: string): Promise<any[]> {
    const fileStream = fs.createReadStream(inputFile)

    const rl = readline.createInterface({
        input: fileStream,
        crlfDelay: Infinity,
    })
    // Note: we use the crlfDelay option to recognize all instances of CR LF
    // ('\r\n') in input.txt as a single line break.

    const allFeatures: any[] = []
    // @ts-ignore
    for await (const line of rl) {
        try {
            allFeatures.push(JSON.parse(line))
        } catch (e) {
            console.error("Could not parse", line)
            break
        }
        if (allFeatures.length % 10000 === 0) {
            ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
        }
    }
    return allFeatures
}

async function readGeojsonLineByLine(inputFile: string): Promise<any[]> {
    const fileStream = fs.createReadStream(inputFile)

    const rl = readline.createInterface({
        input: fileStream,
        crlfDelay: Infinity,
    })
    // Note: we use the crlfDelay option to recognize all instances of CR LF
    // ('\r\n') in input.txt as a single line break.

    const allFeatures: any[] = []
    let featuresSeen = false
    // @ts-ignore
    for await (let line: string of rl) {
        if (!featuresSeen && line.startsWith('"features":')) {
            featuresSeen = true
            continue
        }
        if (!featuresSeen) {
            continue
        }
        if (line.endsWith(",")) {
            line = line.substring(0, line.length - 1)
        }

        try {
            allFeatures.push(JSON.parse(line))
        } catch (e) {
            console.error("Could not parse", line)
            break
        }
        if (allFeatures.length % 10000 === 0) {
            ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
        }
    }
    return allFeatures
}

async function readFeaturesFromGeoJson(inputFile: string): Promise<any[]> {
    try {
        return JSON.parse(fs.readFileSync(inputFile, "UTF-8")).features
    } catch (e) {
        // We retry, but with a line-by-line approach
        return await readGeojsonLineByLine(inputFile)
    }
}

async function main(args: string[]) {
    console.log("GeoJSON slicer")
    if (args.length < 3) {
        console.log("USAGE: <input-file.geojson> <target-zoom-level> <output-directory>")
        return
    }

    const inputFile = args[0]
    const zoomlevel = Number(args[1])
    const outputDirectory = args[2]

    if (!fs.existsSync(outputDirectory)) {
        fs.mkdirSync(outputDirectory)
        console.log("Directory created")
    }
    console.log("Using directory ", outputDirectory)

    let allFeatures: any[]
    if (inputFile.endsWith(".geojson")) {
        console.log("Detected geojson")
        allFeatures = await readFeaturesFromGeoJson(inputFile)
    } else {
        console.log("Loading as newline-delimited features")
        allFeatures = await readFeaturesFromLineDelimitedJsonFile(inputFile)
    }
    allFeatures = Utils.NoNull(allFeatures)

    console.log("Loaded all", allFeatures.length, "points")

    const keysToRemove = ["STRAATNMID", "GEMEENTE", "POSTCODE"]
    for (const f of allFeatures) {
        if (f.properties === null) {
            console.log("Got a feature without properties!", f)
            continue
        }
        for (const keyToRm of keysToRemove) {
            delete f.properties[keyToRm]
        }
        delete f.bbox
    }
    TiledFeatureSource.createHierarchy(StaticFeatureSource.fromGeojson(allFeatures), {
        minZoomLevel: zoomlevel,
        maxZoomLevel: zoomlevel,
        maxFeatureCount: Number.MAX_VALUE,
        registerTile: (tile) => {
            const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson`
            const features = tile.features.data.map((ff) => ff.feature)
            features.forEach((f) => {
                delete f.bbox
            })
            fs.writeFileSync(
                path,
                JSON.stringify(
                    {
                        type: "FeatureCollection",
                        features: features,
                    },
                    null,
                    "  "
                )
            )
            ScriptUtils.erasableLog(
                "Written ",
                path,
                "which has ",
                tile.features.data.length,
                "features"
            )
        },
    })
}

let args = [...process.argv]
args.splice(0, 2)
main(args).then((_) => {
    console.log("All done!")
})