forked from MapComplete/MapComplete
Slice script now clips to tile box if flag is set
This commit is contained in:
parent
b88bb5b6d0
commit
8cf3b88172
2 changed files with 163 additions and 133 deletions
286
scripts/slice.ts
286
scripts/slice.ts
|
|
@ -4,159 +4,179 @@ import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSou
|
|||
import * as readline from "readline"
|
||||
import ScriptUtils from "./ScriptUtils"
|
||||
import { Utils } from "../Utils"
|
||||
import Script from "./Script"
|
||||
import { BBox } from "../Logic/BBox"
|
||||
import { GeoOperations } from "../Logic/GeoOperations"
|
||||
|
||||
/**
|
||||
* This script slices a big newline-delimeted geojson file into tiled geojson
|
||||
* It was used to convert the CRAB-data into geojson tiles
|
||||
*/
|
||||
|
||||
async function readFeaturesFromLineDelimitedJsonFile(inputFile: string): Promise<any[]> {
|
||||
const fileStream = fs.createReadStream(inputFile)
|
||||
class Slice extends Script {
|
||||
constructor() {
|
||||
super("Break data into tiles")
|
||||
}
|
||||
|
||||
const rl = readline.createInterface({
|
||||
input: fileStream,
|
||||
crlfDelay: Infinity,
|
||||
})
|
||||
// Note: we use the crlfDelay option to recognize all instances of CR LF
|
||||
// ('\r\n') in input.txt as a single line break.
|
||||
async readFeaturesFromLineDelimitedJsonFile(inputFile: string): Promise<any[]> {
|
||||
const fileStream = fs.createReadStream(inputFile)
|
||||
|
||||
const allFeatures: any[] = []
|
||||
// @ts-ignore
|
||||
for await (const line of rl) {
|
||||
const rl = readline.createInterface({
|
||||
input: fileStream,
|
||||
crlfDelay: Infinity,
|
||||
})
|
||||
// Note: we use the crlfDelay option to recognize all instances of CR LF
|
||||
// ('\r\n') in input.txt as a single line break.
|
||||
|
||||
const allFeatures: any[] = []
|
||||
// @ts-ignore
|
||||
for await (const line of rl) {
|
||||
try {
|
||||
allFeatures.push(JSON.parse(line))
|
||||
} catch (e) {
|
||||
console.error("Could not parse", line)
|
||||
break
|
||||
}
|
||||
if (allFeatures.length % 10000 === 0) {
|
||||
ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
|
||||
}
|
||||
}
|
||||
return allFeatures
|
||||
}
|
||||
|
||||
async readGeojsonLineByLine(inputFile: string): Promise<any[]> {
|
||||
const fileStream = fs.createReadStream(inputFile)
|
||||
|
||||
const rl = readline.createInterface({
|
||||
input: fileStream,
|
||||
crlfDelay: Infinity,
|
||||
})
|
||||
// Note: we use the crlfDelay option to recognize all instances of CR LF
|
||||
// ('\r\n') in input.txt as a single line break.
|
||||
|
||||
const allFeatures: any[] = []
|
||||
let featuresSeen = false
|
||||
// @ts-ignore
|
||||
for await (let line: string of rl) {
|
||||
if (!featuresSeen && line.startsWith('"features":')) {
|
||||
featuresSeen = true
|
||||
continue
|
||||
}
|
||||
if (!featuresSeen) {
|
||||
continue
|
||||
}
|
||||
if (line.endsWith(",")) {
|
||||
line = line.substring(0, line.length - 1)
|
||||
}
|
||||
|
||||
try {
|
||||
allFeatures.push(JSON.parse(line))
|
||||
} catch (e) {
|
||||
console.error("Could not parse", line)
|
||||
break
|
||||
}
|
||||
if (allFeatures.length % 10000 === 0) {
|
||||
ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
|
||||
}
|
||||
}
|
||||
return allFeatures
|
||||
}
|
||||
|
||||
async readFeaturesFromGeoJson(inputFile: string): Promise<any[]> {
|
||||
try {
|
||||
allFeatures.push(JSON.parse(line))
|
||||
return JSON.parse(fs.readFileSync(inputFile, { encoding: "utf-8" })).features
|
||||
} catch (e) {
|
||||
console.error("Could not parse", line)
|
||||
break
|
||||
}
|
||||
if (allFeatures.length % 10000 === 0) {
|
||||
ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
|
||||
// We retry, but with a line-by-line approach
|
||||
return await this.readGeojsonLineByLine(inputFile)
|
||||
}
|
||||
}
|
||||
return allFeatures
|
||||
}
|
||||
|
||||
async function readGeojsonLineByLine(inputFile: string): Promise<any[]> {
|
||||
const fileStream = fs.createReadStream(inputFile)
|
||||
|
||||
const rl = readline.createInterface({
|
||||
input: fileStream,
|
||||
crlfDelay: Infinity,
|
||||
})
|
||||
// Note: we use the crlfDelay option to recognize all instances of CR LF
|
||||
// ('\r\n') in input.txt as a single line break.
|
||||
|
||||
const allFeatures: any[] = []
|
||||
let featuresSeen = false
|
||||
// @ts-ignore
|
||||
for await (let line: string of rl) {
|
||||
if (!featuresSeen && line.startsWith('"features":')) {
|
||||
featuresSeen = true
|
||||
continue
|
||||
}
|
||||
if (!featuresSeen) {
|
||||
continue
|
||||
}
|
||||
if (line.endsWith(",")) {
|
||||
line = line.substring(0, line.length - 1)
|
||||
async main(args: string[]) {
|
||||
console.log("GeoJSON slicer")
|
||||
if (args.length < 3) {
|
||||
console.log(
|
||||
"USAGE: <input-file.geojson> <target-zoom-level> <output-directory> [--clip]"
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
allFeatures.push(JSON.parse(line))
|
||||
} catch (e) {
|
||||
console.error("Could not parse", line)
|
||||
break
|
||||
const inputFile = args[0]
|
||||
const zoomlevel = Number(args[1])
|
||||
const outputDirectory = args[2]
|
||||
const doSlice = args[3] === "--clip"
|
||||
|
||||
if (!fs.existsSync(outputDirectory)) {
|
||||
fs.mkdirSync(outputDirectory)
|
||||
console.log("Directory created")
|
||||
}
|
||||
if (allFeatures.length % 10000 === 0) {
|
||||
ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
|
||||
console.log("Using directory ", outputDirectory)
|
||||
|
||||
let allFeatures: any[]
|
||||
if (inputFile.endsWith(".geojson")) {
|
||||
console.log("Detected geojson")
|
||||
allFeatures = await this.readFeaturesFromGeoJson(inputFile)
|
||||
} else {
|
||||
console.log("Loading as newline-delimited features")
|
||||
allFeatures = await this.readFeaturesFromLineDelimitedJsonFile(inputFile)
|
||||
}
|
||||
}
|
||||
return allFeatures
|
||||
}
|
||||
allFeatures = Utils.NoNull(allFeatures)
|
||||
|
||||
async function readFeaturesFromGeoJson(inputFile: string): Promise<any[]> {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(inputFile, { encoding: "utf-8" })).features
|
||||
} catch (e) {
|
||||
// We retry, but with a line-by-line approach
|
||||
return await readGeojsonLineByLine(inputFile)
|
||||
}
|
||||
}
|
||||
console.log("Loaded all", allFeatures.length, "points")
|
||||
|
||||
async function main(args: string[]) {
|
||||
console.log("GeoJSON slicer")
|
||||
if (args.length < 3) {
|
||||
console.log("USAGE: <input-file.geojson> <target-zoom-level> <output-directory>")
|
||||
return
|
||||
}
|
||||
|
||||
const inputFile = args[0]
|
||||
const zoomlevel = Number(args[1])
|
||||
const outputDirectory = args[2]
|
||||
|
||||
if (!fs.existsSync(outputDirectory)) {
|
||||
fs.mkdirSync(outputDirectory)
|
||||
console.log("Directory created")
|
||||
}
|
||||
console.log("Using directory ", outputDirectory)
|
||||
|
||||
let allFeatures: any[]
|
||||
if (inputFile.endsWith(".geojson")) {
|
||||
console.log("Detected geojson")
|
||||
allFeatures = await readFeaturesFromGeoJson(inputFile)
|
||||
} else {
|
||||
console.log("Loading as newline-delimited features")
|
||||
allFeatures = await readFeaturesFromLineDelimitedJsonFile(inputFile)
|
||||
}
|
||||
allFeatures = Utils.NoNull(allFeatures)
|
||||
|
||||
console.log("Loaded all", allFeatures.length, "points")
|
||||
|
||||
const keysToRemove = ["STRAATNMID", "GEMEENTE", "POSTCODE"]
|
||||
for (const f of allFeatures) {
|
||||
if (f.properties === null) {
|
||||
console.log("Got a feature without properties!", f)
|
||||
continue
|
||||
const keysToRemove = ["STRAATNMID", "GEMEENTE", "POSTCODE"]
|
||||
for (const f of allFeatures) {
|
||||
if (f.properties === null) {
|
||||
console.log("Got a feature without properties!", f)
|
||||
continue
|
||||
}
|
||||
for (const keyToRm of keysToRemove) {
|
||||
delete f.properties[keyToRm]
|
||||
}
|
||||
delete f.bbox
|
||||
}
|
||||
for (const keyToRm of keysToRemove) {
|
||||
delete f.properties[keyToRm]
|
||||
}
|
||||
delete f.bbox
|
||||
}
|
||||
TiledFeatureSource.createHierarchy(StaticFeatureSource.fromGeojson(allFeatures), {
|
||||
minZoomLevel: zoomlevel,
|
||||
maxZoomLevel: zoomlevel,
|
||||
maxFeatureCount: Number.MAX_VALUE,
|
||||
registerTile: (tile) => {
|
||||
const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson`
|
||||
const features = tile.features.data.map((ff) => ff.feature)
|
||||
features.forEach((f) => {
|
||||
delete f.bbox
|
||||
})
|
||||
fs.writeFileSync(
|
||||
path,
|
||||
JSON.stringify(
|
||||
{
|
||||
type: "FeatureCollection",
|
||||
features: features,
|
||||
},
|
||||
null,
|
||||
" "
|
||||
TiledFeatureSource.createHierarchy(StaticFeatureSource.fromGeojson(allFeatures), {
|
||||
minZoomLevel: zoomlevel,
|
||||
maxZoomLevel: zoomlevel,
|
||||
maxFeatureCount: Number.MAX_VALUE,
|
||||
registerTile: (tile) => {
|
||||
const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson`
|
||||
const box = BBox.fromTile(tile.z, tile.x, tile.y)
|
||||
let features = tile.features.data.map((ff) => ff.feature)
|
||||
if (doSlice) {
|
||||
features = Utils.NoNull(
|
||||
features.map((f) => {
|
||||
const intersection = GeoOperations.intersect(f, box.asGeoJson({}))
|
||||
if (intersection) {
|
||||
intersection.properties = f.properties
|
||||
}
|
||||
return intersection
|
||||
})
|
||||
)
|
||||
}
|
||||
features.forEach((f) => {
|
||||
delete f.bbox
|
||||
})
|
||||
fs.writeFileSync(
|
||||
path,
|
||||
JSON.stringify(
|
||||
{
|
||||
type: "FeatureCollection",
|
||||
features: features,
|
||||
},
|
||||
null,
|
||||
" "
|
||||
)
|
||||
)
|
||||
)
|
||||
ScriptUtils.erasableLog(
|
||||
"Written ",
|
||||
path,
|
||||
"which has ",
|
||||
tile.features.data.length,
|
||||
"features"
|
||||
)
|
||||
},
|
||||
})
|
||||
ScriptUtils.erasableLog(
|
||||
"Written ",
|
||||
path,
|
||||
"which has ",
|
||||
tile.features.data.length,
|
||||
"features"
|
||||
)
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
let args = [...process.argv]
|
||||
args.splice(0, 2)
|
||||
main(args).then((_) => {
|
||||
console.log("All done!")
|
||||
})
|
||||
new Slice().run()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue