From 8cf3b881725a1002c7e5d3c231b34aacfd63386e Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Thu, 26 Jan 2023 01:41:21 +0100 Subject: [PATCH] Slice script now clips to tile box if flag is set --- Logic/GeoOperations.ts | 10 ++ scripts/slice.ts | 286 ++++++++++++++++++++++------------------- 2 files changed, 163 insertions(+), 133 deletions(-) diff --git a/Logic/GeoOperations.ts b/Logic/GeoOperations.ts index d64550620c..86a717007f 100644 --- a/Logic/GeoOperations.ts +++ b/Logic/GeoOperations.ts @@ -783,4 +783,14 @@ export class GeoOperations { ): boolean { return booleanWithin(feature, possiblyEncloingFeature) } + + /** + * Create a union between two features + */ + static union = turf.union + + /** + * Create an intersection between two features + */ + static intersect = turf.intersect } diff --git a/scripts/slice.ts b/scripts/slice.ts index 214f5dad0c..80c502c289 100644 --- a/scripts/slice.ts +++ b/scripts/slice.ts @@ -4,159 +4,179 @@ import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSou import * as readline from "readline" import ScriptUtils from "./ScriptUtils" import { Utils } from "../Utils" +import Script from "./Script" +import { BBox } from "../Logic/BBox" +import { GeoOperations } from "../Logic/GeoOperations" /** * This script slices a big newline-delimeted geojson file into tiled geojson * It was used to convert the CRAB-data into geojson tiles */ -async function readFeaturesFromLineDelimitedJsonFile(inputFile: string): Promise { - const fileStream = fs.createReadStream(inputFile) +class Slice extends Script { + constructor() { + super("Break data into tiles") + } - const rl = readline.createInterface({ - input: fileStream, - crlfDelay: Infinity, - }) - // Note: we use the crlfDelay option to recognize all instances of CR LF - // ('\r\n') in input.txt as a single line break. + async readFeaturesFromLineDelimitedJsonFile(inputFile: string): Promise { + const fileStream = fs.createReadStream(inputFile) - const allFeatures: any[] = [] - // @ts-ignore - for await (const line of rl) { + const rl = readline.createInterface({ + input: fileStream, + crlfDelay: Infinity, + }) + // Note: we use the crlfDelay option to recognize all instances of CR LF + // ('\r\n') in input.txt as a single line break. + + const allFeatures: any[] = [] + // @ts-ignore + for await (const line of rl) { + try { + allFeatures.push(JSON.parse(line)) + } catch (e) { + console.error("Could not parse", line) + break + } + if (allFeatures.length % 10000 === 0) { + ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now") + } + } + return allFeatures + } + + async readGeojsonLineByLine(inputFile: string): Promise { + const fileStream = fs.createReadStream(inputFile) + + const rl = readline.createInterface({ + input: fileStream, + crlfDelay: Infinity, + }) + // Note: we use the crlfDelay option to recognize all instances of CR LF + // ('\r\n') in input.txt as a single line break. + + const allFeatures: any[] = [] + let featuresSeen = false + // @ts-ignore + for await (let line: string of rl) { + if (!featuresSeen && line.startsWith('"features":')) { + featuresSeen = true + continue + } + if (!featuresSeen) { + continue + } + if (line.endsWith(",")) { + line = line.substring(0, line.length - 1) + } + + try { + allFeatures.push(JSON.parse(line)) + } catch (e) { + console.error("Could not parse", line) + break + } + if (allFeatures.length % 10000 === 0) { + ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now") + } + } + return allFeatures + } + + async readFeaturesFromGeoJson(inputFile: string): Promise { try { - allFeatures.push(JSON.parse(line)) + return JSON.parse(fs.readFileSync(inputFile, { encoding: "utf-8" })).features } catch (e) { - console.error("Could not parse", line) - break - } - if (allFeatures.length % 10000 === 0) { - ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now") + // We retry, but with a line-by-line approach + return await this.readGeojsonLineByLine(inputFile) } } - return allFeatures -} -async function readGeojsonLineByLine(inputFile: string): Promise { - const fileStream = fs.createReadStream(inputFile) - - const rl = readline.createInterface({ - input: fileStream, - crlfDelay: Infinity, - }) - // Note: we use the crlfDelay option to recognize all instances of CR LF - // ('\r\n') in input.txt as a single line break. - - const allFeatures: any[] = [] - let featuresSeen = false - // @ts-ignore - for await (let line: string of rl) { - if (!featuresSeen && line.startsWith('"features":')) { - featuresSeen = true - continue - } - if (!featuresSeen) { - continue - } - if (line.endsWith(",")) { - line = line.substring(0, line.length - 1) + async main(args: string[]) { + console.log("GeoJSON slicer") + if (args.length < 3) { + console.log( + "USAGE: [--clip]" + ) + return } - try { - allFeatures.push(JSON.parse(line)) - } catch (e) { - console.error("Could not parse", line) - break + const inputFile = args[0] + const zoomlevel = Number(args[1]) + const outputDirectory = args[2] + const doSlice = args[3] === "--clip" + + if (!fs.existsSync(outputDirectory)) { + fs.mkdirSync(outputDirectory) + console.log("Directory created") } - if (allFeatures.length % 10000 === 0) { - ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now") + console.log("Using directory ", outputDirectory) + + let allFeatures: any[] + if (inputFile.endsWith(".geojson")) { + console.log("Detected geojson") + allFeatures = await this.readFeaturesFromGeoJson(inputFile) + } else { + console.log("Loading as newline-delimited features") + allFeatures = await this.readFeaturesFromLineDelimitedJsonFile(inputFile) } - } - return allFeatures -} + allFeatures = Utils.NoNull(allFeatures) -async function readFeaturesFromGeoJson(inputFile: string): Promise { - try { - return JSON.parse(fs.readFileSync(inputFile, { encoding: "utf-8" })).features - } catch (e) { - // We retry, but with a line-by-line approach - return await readGeojsonLineByLine(inputFile) - } -} + console.log("Loaded all", allFeatures.length, "points") -async function main(args: string[]) { - console.log("GeoJSON slicer") - if (args.length < 3) { - console.log("USAGE: ") - return - } - - const inputFile = args[0] - const zoomlevel = Number(args[1]) - const outputDirectory = args[2] - - if (!fs.existsSync(outputDirectory)) { - fs.mkdirSync(outputDirectory) - console.log("Directory created") - } - console.log("Using directory ", outputDirectory) - - let allFeatures: any[] - if (inputFile.endsWith(".geojson")) { - console.log("Detected geojson") - allFeatures = await readFeaturesFromGeoJson(inputFile) - } else { - console.log("Loading as newline-delimited features") - allFeatures = await readFeaturesFromLineDelimitedJsonFile(inputFile) - } - allFeatures = Utils.NoNull(allFeatures) - - console.log("Loaded all", allFeatures.length, "points") - - const keysToRemove = ["STRAATNMID", "GEMEENTE", "POSTCODE"] - for (const f of allFeatures) { - if (f.properties === null) { - console.log("Got a feature without properties!", f) - continue + const keysToRemove = ["STRAATNMID", "GEMEENTE", "POSTCODE"] + for (const f of allFeatures) { + if (f.properties === null) { + console.log("Got a feature without properties!", f) + continue + } + for (const keyToRm of keysToRemove) { + delete f.properties[keyToRm] + } + delete f.bbox } - for (const keyToRm of keysToRemove) { - delete f.properties[keyToRm] - } - delete f.bbox - } - TiledFeatureSource.createHierarchy(StaticFeatureSource.fromGeojson(allFeatures), { - minZoomLevel: zoomlevel, - maxZoomLevel: zoomlevel, - maxFeatureCount: Number.MAX_VALUE, - registerTile: (tile) => { - const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson` - const features = tile.features.data.map((ff) => ff.feature) - features.forEach((f) => { - delete f.bbox - }) - fs.writeFileSync( - path, - JSON.stringify( - { - type: "FeatureCollection", - features: features, - }, - null, - " " + TiledFeatureSource.createHierarchy(StaticFeatureSource.fromGeojson(allFeatures), { + minZoomLevel: zoomlevel, + maxZoomLevel: zoomlevel, + maxFeatureCount: Number.MAX_VALUE, + registerTile: (tile) => { + const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson` + const box = BBox.fromTile(tile.z, tile.x, tile.y) + let features = tile.features.data.map((ff) => ff.feature) + if (doSlice) { + features = Utils.NoNull( + features.map((f) => { + const intersection = GeoOperations.intersect(f, box.asGeoJson({})) + if (intersection) { + intersection.properties = f.properties + } + return intersection + }) + ) + } + features.forEach((f) => { + delete f.bbox + }) + fs.writeFileSync( + path, + JSON.stringify( + { + type: "FeatureCollection", + features: features, + }, + null, + " " + ) ) - ) - ScriptUtils.erasableLog( - "Written ", - path, - "which has ", - tile.features.data.length, - "features" - ) - }, - }) + ScriptUtils.erasableLog( + "Written ", + path, + "which has ", + tile.features.data.length, + "features" + ) + }, + }) + } } -let args = [...process.argv] -args.splice(0, 2) -main(args).then((_) => { - console.log("All done!") -}) +new Slice().run()