forked from MapComplete/MapComplete
Slice script now clips to tile box if flag is set
This commit is contained in:
parent
b88bb5b6d0
commit
8cf3b88172
2 changed files with 163 additions and 133 deletions
|
|
@ -783,4 +783,14 @@ export class GeoOperations {
|
||||||
): boolean {
|
): boolean {
|
||||||
return booleanWithin(feature, possiblyEncloingFeature)
|
return booleanWithin(feature, possiblyEncloingFeature)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a union between two features
|
||||||
|
*/
|
||||||
|
static union = turf.union
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create an intersection between two features
|
||||||
|
*/
|
||||||
|
static intersect = turf.intersect
|
||||||
}
|
}
|
||||||
|
|
|
||||||
286
scripts/slice.ts
286
scripts/slice.ts
|
|
@ -4,159 +4,179 @@ import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSou
|
||||||
import * as readline from "readline"
|
import * as readline from "readline"
|
||||||
import ScriptUtils from "./ScriptUtils"
|
import ScriptUtils from "./ScriptUtils"
|
||||||
import { Utils } from "../Utils"
|
import { Utils } from "../Utils"
|
||||||
|
import Script from "./Script"
|
||||||
|
import { BBox } from "../Logic/BBox"
|
||||||
|
import { GeoOperations } from "../Logic/GeoOperations"
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This script slices a big newline-delimeted geojson file into tiled geojson
|
* This script slices a big newline-delimeted geojson file into tiled geojson
|
||||||
* It was used to convert the CRAB-data into geojson tiles
|
* It was used to convert the CRAB-data into geojson tiles
|
||||||
*/
|
*/
|
||||||
|
|
||||||
async function readFeaturesFromLineDelimitedJsonFile(inputFile: string): Promise<any[]> {
|
class Slice extends Script {
|
||||||
const fileStream = fs.createReadStream(inputFile)
|
constructor() {
|
||||||
|
super("Break data into tiles")
|
||||||
|
}
|
||||||
|
|
||||||
const rl = readline.createInterface({
|
async readFeaturesFromLineDelimitedJsonFile(inputFile: string): Promise<any[]> {
|
||||||
input: fileStream,
|
const fileStream = fs.createReadStream(inputFile)
|
||||||
crlfDelay: Infinity,
|
|
||||||
})
|
|
||||||
// Note: we use the crlfDelay option to recognize all instances of CR LF
|
|
||||||
// ('\r\n') in input.txt as a single line break.
|
|
||||||
|
|
||||||
const allFeatures: any[] = []
|
const rl = readline.createInterface({
|
||||||
// @ts-ignore
|
input: fileStream,
|
||||||
for await (const line of rl) {
|
crlfDelay: Infinity,
|
||||||
|
})
|
||||||
|
// Note: we use the crlfDelay option to recognize all instances of CR LF
|
||||||
|
// ('\r\n') in input.txt as a single line break.
|
||||||
|
|
||||||
|
const allFeatures: any[] = []
|
||||||
|
// @ts-ignore
|
||||||
|
for await (const line of rl) {
|
||||||
|
try {
|
||||||
|
allFeatures.push(JSON.parse(line))
|
||||||
|
} catch (e) {
|
||||||
|
console.error("Could not parse", line)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if (allFeatures.length % 10000 === 0) {
|
||||||
|
ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return allFeatures
|
||||||
|
}
|
||||||
|
|
||||||
|
async readGeojsonLineByLine(inputFile: string): Promise<any[]> {
|
||||||
|
const fileStream = fs.createReadStream(inputFile)
|
||||||
|
|
||||||
|
const rl = readline.createInterface({
|
||||||
|
input: fileStream,
|
||||||
|
crlfDelay: Infinity,
|
||||||
|
})
|
||||||
|
// Note: we use the crlfDelay option to recognize all instances of CR LF
|
||||||
|
// ('\r\n') in input.txt as a single line break.
|
||||||
|
|
||||||
|
const allFeatures: any[] = []
|
||||||
|
let featuresSeen = false
|
||||||
|
// @ts-ignore
|
||||||
|
for await (let line: string of rl) {
|
||||||
|
if (!featuresSeen && line.startsWith('"features":')) {
|
||||||
|
featuresSeen = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (!featuresSeen) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (line.endsWith(",")) {
|
||||||
|
line = line.substring(0, line.length - 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
allFeatures.push(JSON.parse(line))
|
||||||
|
} catch (e) {
|
||||||
|
console.error("Could not parse", line)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if (allFeatures.length % 10000 === 0) {
|
||||||
|
ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return allFeatures
|
||||||
|
}
|
||||||
|
|
||||||
|
async readFeaturesFromGeoJson(inputFile: string): Promise<any[]> {
|
||||||
try {
|
try {
|
||||||
allFeatures.push(JSON.parse(line))
|
return JSON.parse(fs.readFileSync(inputFile, { encoding: "utf-8" })).features
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error("Could not parse", line)
|
// We retry, but with a line-by-line approach
|
||||||
break
|
return await this.readGeojsonLineByLine(inputFile)
|
||||||
}
|
|
||||||
if (allFeatures.length % 10000 === 0) {
|
|
||||||
ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return allFeatures
|
|
||||||
}
|
|
||||||
|
|
||||||
async function readGeojsonLineByLine(inputFile: string): Promise<any[]> {
|
async main(args: string[]) {
|
||||||
const fileStream = fs.createReadStream(inputFile)
|
console.log("GeoJSON slicer")
|
||||||
|
if (args.length < 3) {
|
||||||
const rl = readline.createInterface({
|
console.log(
|
||||||
input: fileStream,
|
"USAGE: <input-file.geojson> <target-zoom-level> <output-directory> [--clip]"
|
||||||
crlfDelay: Infinity,
|
)
|
||||||
})
|
return
|
||||||
// Note: we use the crlfDelay option to recognize all instances of CR LF
|
|
||||||
// ('\r\n') in input.txt as a single line break.
|
|
||||||
|
|
||||||
const allFeatures: any[] = []
|
|
||||||
let featuresSeen = false
|
|
||||||
// @ts-ignore
|
|
||||||
for await (let line: string of rl) {
|
|
||||||
if (!featuresSeen && line.startsWith('"features":')) {
|
|
||||||
featuresSeen = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if (!featuresSeen) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if (line.endsWith(",")) {
|
|
||||||
line = line.substring(0, line.length - 1)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
const inputFile = args[0]
|
||||||
allFeatures.push(JSON.parse(line))
|
const zoomlevel = Number(args[1])
|
||||||
} catch (e) {
|
const outputDirectory = args[2]
|
||||||
console.error("Could not parse", line)
|
const doSlice = args[3] === "--clip"
|
||||||
break
|
|
||||||
|
if (!fs.existsSync(outputDirectory)) {
|
||||||
|
fs.mkdirSync(outputDirectory)
|
||||||
|
console.log("Directory created")
|
||||||
}
|
}
|
||||||
if (allFeatures.length % 10000 === 0) {
|
console.log("Using directory ", outputDirectory)
|
||||||
ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
|
|
||||||
|
let allFeatures: any[]
|
||||||
|
if (inputFile.endsWith(".geojson")) {
|
||||||
|
console.log("Detected geojson")
|
||||||
|
allFeatures = await this.readFeaturesFromGeoJson(inputFile)
|
||||||
|
} else {
|
||||||
|
console.log("Loading as newline-delimited features")
|
||||||
|
allFeatures = await this.readFeaturesFromLineDelimitedJsonFile(inputFile)
|
||||||
}
|
}
|
||||||
}
|
allFeatures = Utils.NoNull(allFeatures)
|
||||||
return allFeatures
|
|
||||||
}
|
|
||||||
|
|
||||||
async function readFeaturesFromGeoJson(inputFile: string): Promise<any[]> {
|
console.log("Loaded all", allFeatures.length, "points")
|
||||||
try {
|
|
||||||
return JSON.parse(fs.readFileSync(inputFile, { encoding: "utf-8" })).features
|
|
||||||
} catch (e) {
|
|
||||||
// We retry, but with a line-by-line approach
|
|
||||||
return await readGeojsonLineByLine(inputFile)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function main(args: string[]) {
|
const keysToRemove = ["STRAATNMID", "GEMEENTE", "POSTCODE"]
|
||||||
console.log("GeoJSON slicer")
|
for (const f of allFeatures) {
|
||||||
if (args.length < 3) {
|
if (f.properties === null) {
|
||||||
console.log("USAGE: <input-file.geojson> <target-zoom-level> <output-directory>")
|
console.log("Got a feature without properties!", f)
|
||||||
return
|
continue
|
||||||
}
|
}
|
||||||
|
for (const keyToRm of keysToRemove) {
|
||||||
const inputFile = args[0]
|
delete f.properties[keyToRm]
|
||||||
const zoomlevel = Number(args[1])
|
}
|
||||||
const outputDirectory = args[2]
|
delete f.bbox
|
||||||
|
|
||||||
if (!fs.existsSync(outputDirectory)) {
|
|
||||||
fs.mkdirSync(outputDirectory)
|
|
||||||
console.log("Directory created")
|
|
||||||
}
|
|
||||||
console.log("Using directory ", outputDirectory)
|
|
||||||
|
|
||||||
let allFeatures: any[]
|
|
||||||
if (inputFile.endsWith(".geojson")) {
|
|
||||||
console.log("Detected geojson")
|
|
||||||
allFeatures = await readFeaturesFromGeoJson(inputFile)
|
|
||||||
} else {
|
|
||||||
console.log("Loading as newline-delimited features")
|
|
||||||
allFeatures = await readFeaturesFromLineDelimitedJsonFile(inputFile)
|
|
||||||
}
|
|
||||||
allFeatures = Utils.NoNull(allFeatures)
|
|
||||||
|
|
||||||
console.log("Loaded all", allFeatures.length, "points")
|
|
||||||
|
|
||||||
const keysToRemove = ["STRAATNMID", "GEMEENTE", "POSTCODE"]
|
|
||||||
for (const f of allFeatures) {
|
|
||||||
if (f.properties === null) {
|
|
||||||
console.log("Got a feature without properties!", f)
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
for (const keyToRm of keysToRemove) {
|
TiledFeatureSource.createHierarchy(StaticFeatureSource.fromGeojson(allFeatures), {
|
||||||
delete f.properties[keyToRm]
|
minZoomLevel: zoomlevel,
|
||||||
}
|
maxZoomLevel: zoomlevel,
|
||||||
delete f.bbox
|
maxFeatureCount: Number.MAX_VALUE,
|
||||||
}
|
registerTile: (tile) => {
|
||||||
TiledFeatureSource.createHierarchy(StaticFeatureSource.fromGeojson(allFeatures), {
|
const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson`
|
||||||
minZoomLevel: zoomlevel,
|
const box = BBox.fromTile(tile.z, tile.x, tile.y)
|
||||||
maxZoomLevel: zoomlevel,
|
let features = tile.features.data.map((ff) => ff.feature)
|
||||||
maxFeatureCount: Number.MAX_VALUE,
|
if (doSlice) {
|
||||||
registerTile: (tile) => {
|
features = Utils.NoNull(
|
||||||
const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson`
|
features.map((f) => {
|
||||||
const features = tile.features.data.map((ff) => ff.feature)
|
const intersection = GeoOperations.intersect(f, box.asGeoJson({}))
|
||||||
features.forEach((f) => {
|
if (intersection) {
|
||||||
delete f.bbox
|
intersection.properties = f.properties
|
||||||
})
|
}
|
||||||
fs.writeFileSync(
|
return intersection
|
||||||
path,
|
})
|
||||||
JSON.stringify(
|
)
|
||||||
{
|
}
|
||||||
type: "FeatureCollection",
|
features.forEach((f) => {
|
||||||
features: features,
|
delete f.bbox
|
||||||
},
|
})
|
||||||
null,
|
fs.writeFileSync(
|
||||||
" "
|
path,
|
||||||
|
JSON.stringify(
|
||||||
|
{
|
||||||
|
type: "FeatureCollection",
|
||||||
|
features: features,
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
" "
|
||||||
|
)
|
||||||
)
|
)
|
||||||
)
|
ScriptUtils.erasableLog(
|
||||||
ScriptUtils.erasableLog(
|
"Written ",
|
||||||
"Written ",
|
path,
|
||||||
path,
|
"which has ",
|
||||||
"which has ",
|
tile.features.data.length,
|
||||||
tile.features.data.length,
|
"features"
|
||||||
"features"
|
)
|
||||||
)
|
},
|
||||||
},
|
})
|
||||||
})
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let args = [...process.argv]
|
new Slice().run()
|
||||||
args.splice(0, 2)
|
|
||||||
main(args).then((_) => {
|
|
||||||
console.log("All done!")
|
|
||||||
})
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue