From d5f4572e9a736fb79bfef2d467666035ffd5c7b0 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Wed, 27 Oct 2021 03:52:19 +0200 Subject: [PATCH] Add possibility to load external data as mercator tiles, add bbox tile possibilities, add CRAB and GRB as datasources in the GRB theme --- Logic/BBox.ts | 21 ++- Logic/FeatureSource/FeaturePipeline.ts | 10 +- Logic/FeatureSource/Sources/GeoJsonSource.ts | 16 ++- .../DynamicGeoJsonTileSource.ts | 42 +++--- Logic/GeoOperations.ts | 30 ++++- Models/ThemeConfig/Json/LayerConfigJson.ts | 4 +- Models/ThemeConfig/LayerConfig.ts | 1 + Models/ThemeConfig/SourceConfig.ts | 10 +- UI/SpecialVisualizations.ts | 2 +- assets/themes/grb_import/README.md | 20 +++ assets/themes/{ => grb_import}/grb.json | 39 +++++- scripts/slice.ts | 120 +++++++++++++----- 12 files changed, 254 insertions(+), 61 deletions(-) create mode 100644 assets/themes/grb_import/README.md rename assets/themes/{ => grb_import}/grb.json (80%) diff --git a/Logic/BBox.ts b/Logic/BBox.ts index 0205b15337..78634897b6 100644 --- a/Logic/BBox.ts +++ b/Logic/BBox.ts @@ -1,5 +1,6 @@ import * as turf from "@turf/turf"; import {TileRange, Tiles} from "../Models/TileRange"; +import {GeoOperations} from "./GeoOperations"; export class BBox { @@ -22,7 +23,7 @@ export class BBox { this.minLon = Math.min(this.minLon, coordinate[0]); this.minLat = Math.min(this.minLat, coordinate[1]); } - + this.maxLon = Math.min(this.maxLon, 180) this.maxLat = Math.min(this.maxLat, 90) this.minLon = Math.max(this.minLon, -180) @@ -117,12 +118,12 @@ export class BBox { } pad(factor: number, maxIncrease = 2): BBox { - + const latDiff = Math.min(maxIncrease / 2, Math.abs(this.maxLat - this.minLat) * factor) - const lonDiff =Math.min(maxIncrease / 2, Math.abs(this.maxLon - this.minLon) * factor) + const lonDiff = Math.min(maxIncrease / 2, Math.abs(this.maxLon - this.minLon) * factor) return new BBox([[ this.minLon - lonDiff, - this.minLat - latDiff + this.minLat - latDiff ], [this.maxLon + lonDiff, this.maxLat + latDiff]]) } @@ -161,4 +162,16 @@ export class BBox { const boundslr = Tiles.tile_bounds_lon_lat(lr.z, lr.x, lr.y) return new BBox([].concat(boundsul, boundslr)) } + + toMercator(): { minLat: number, maxLat: number, minLon: number, maxLon: number } { + const [minLon, minLat] = GeoOperations.ConvertWgs84To900913([this.minLon, this.minLat]) + const [maxLon, maxLat] = GeoOperations.ConvertWgs84To900913([this.maxLon, this.maxLat]) + + return { + minLon, maxLon, + minLat, maxLat + } + + + } } \ No newline at end of file diff --git a/Logic/FeatureSource/FeaturePipeline.ts b/Logic/FeatureSource/FeaturePipeline.ts index b5ea30bd3a..8cd9aae67c 100644 --- a/Logic/FeatureSource/FeaturePipeline.ts +++ b/Logic/FeatureSource/FeaturePipeline.ts @@ -98,7 +98,7 @@ export default class FeaturePipeline { this.osmSourceZoomLevel = state.osmApiTileSize.data; const useOsmApi = state.locationControl.map(l => l.zoom > (state.overpassMaxZoom.data ?? 12)) this.relationTracker = new RelationsTracker() - + state.changes.allChanges.addCallbackAndRun(allChanges => { allChanges.filter(ch => ch.id < 0 && ch.changes !== undefined) .map(ch => ch.changes) @@ -205,7 +205,9 @@ export default class FeaturePipeline { neededTiles: neededTilesFromOsm, handleTile: tile => { new RegisteringAllFromFeatureSourceActor(tile) - new SaveTileToLocalStorageActor(tile, tile.tileIndex) + if (tile.layer.layerDef.maxAgeOfCache > 0) { + new SaveTileToLocalStorageActor(tile, tile.tileIndex) + } perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile) tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile)) @@ -213,7 +215,9 @@ export default class FeaturePipeline { state: state, markTileVisited: (tileId) => state.filteredLayers.data.forEach(flayer => { - SaveTileToLocalStorageActor.MarkVisited(flayer.layerDef.id, tileId, new Date()) + if (flayer.layerDef.maxAgeOfCache > 0) { + SaveTileToLocalStorageActor.MarkVisited(flayer.layerDef.id, tileId, new Date()) + } self.freshnesses.get(flayer.layerDef.id).addTileLoad(tileId, new Date()) }) }) diff --git a/Logic/FeatureSource/Sources/GeoJsonSource.ts b/Logic/FeatureSource/Sources/GeoJsonSource.ts index be93dedc7a..68f8fab823 100644 --- a/Logic/FeatureSource/Sources/GeoJsonSource.ts +++ b/Logic/FeatureSource/Sources/GeoJsonSource.ts @@ -7,6 +7,7 @@ import {Utils} from "../../../Utils"; import {FeatureSourceForLayer, Tiled} from "../FeatureSource"; import {Tiles} from "../../../Models/TileRange"; import {BBox} from "../../BBox"; +import {GeoOperations} from "../../GeoOperations"; export default class GeoJsonSource implements FeatureSourceForLayer, Tiled { @@ -14,7 +15,6 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled { public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>; public readonly name; public readonly isOsmCache: boolean - private onFail: ((errorMsg: any, url: string) => void) = undefined; private readonly seenids: Set = new Set() public readonly layer: FilteredLayer; @@ -44,10 +44,20 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled { let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id); if (zxy !== undefined) { const [z, x, y] = zxy; + let tile_bbox = BBox.fromTile(z, x, y) + let bounds : { minLat: number, maxLat: number, minLon: number, maxLon: number } = tile_bbox + if(this.layer.layerDef.source.mercatorCrs){ + bounds = tile_bbox.toMercator() + } url = url .replace('{z}', "" + z) .replace('{x}', "" + x) .replace('{y}', "" + y) + .replace('{y_min}',""+bounds.minLat) + .replace('{y_max}',""+bounds.maxLat) + .replace('{x_min}',""+bounds.minLon) + .replace('{x_max}',""+bounds.maxLon) + this.tileIndex = Tiles.tile_index(z, x, y) this.bbox = BBox.fromTile(z, x, y) } else { @@ -71,6 +81,10 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled { if(json.features === undefined || json.features === null){ return; } + + if(self.layer.layerDef.source.mercatorCrs){ + json = GeoOperations.GeoJsonToWGS84(json) + } const time = new Date(); const newFeatures: { feature: any, freshness: Date } [] = [] diff --git a/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts b/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts index 5032f53eaa..21aeec1c1d 100644 --- a/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts +++ b/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts @@ -20,24 +20,28 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource { if (source.geojsonSource === undefined) { throw "Invalid layer: geojsonSource expected" } - - const whitelistUrl = source.geojsonSource - .replace("{z}", ""+source.geojsonZoomLevel) - .replace("{x}_{y}.geojson", "overview.json") - .replace("{layer}",layer.layerDef.id) - + let whitelist = undefined - Utils.downloadJson(whitelistUrl).then( - json => { - const data = new Map>(); - for (const x in json) { - data.set(Number(x), new Set(json[x])) + if (source.geojsonSource.indexOf("{x}_{y}.geojson") > 0) { + + const whitelistUrl = source.geojsonSource + .replace("{z}", "" + source.geojsonZoomLevel) + .replace("{x}_{y}.geojson", "overview.json") + .replace("{layer}", layer.layerDef.id) + + Utils.downloadJson(whitelistUrl).then( + json => { + const data = new Map>(); + for (const x in json) { + data.set(Number(x), new Set(json[x])) + } + console.log("The whitelist is", data, "based on ", json, "from", whitelistUrl) + whitelist = data } - whitelist = data - } - ).catch(err => { - console.warn("No whitelist found for ", layer.layerDef.id, err) - }) + ).catch(err => { + console.warn("No whitelist found for ", layer.layerDef.id, err) + }) + } const seenIds = new Set(); const blackList = new UIEventSource(seenIds) @@ -45,14 +49,14 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource { layer, source.geojsonZoomLevel, (zxy) => { - if(whitelist !== undefined){ + if (whitelist !== undefined) { const isWhiteListed = whitelist.get(zxy[1])?.has(zxy[2]) - if(!isWhiteListed){ + if (!isWhiteListed) { console.log("Not downloading tile", ...zxy, "as it is not on the whitelist") return undefined; } } - + const src = new GeoJsonSource( layer, zxy, diff --git a/Logic/GeoOperations.ts b/Logic/GeoOperations.ts index 87716003ee..09a2881178 100644 --- a/Logic/GeoOperations.ts +++ b/Logic/GeoOperations.ts @@ -226,7 +226,7 @@ export class GeoOperations { /** * Generates the closest point on a way from a given point - * + * * The properties object will contain three values: // - `index`: closest point was found on nth line part, // - `dist`: distance between pt and the closest point (in kilometer), @@ -283,6 +283,34 @@ export class GeoOperations { return headerValuesOrdered.map(v => JSON.stringify(v)).join(",") + "\n" + lines.join("\n") } + + private static readonly _earthRadius = 6378137; + private static readonly _originShift = 2 * Math.PI * GeoOperations._earthRadius / 2; + + //Converts given lat/lon in WGS84 Datum to XY in Spherical Mercator EPSG:900913 + public static ConvertWgs84To900913(lonLat: [number, number]): [number, number] { + const lon = lonLat[0]; + const lat = lonLat[1]; + const x = lon * GeoOperations._originShift / 180; + let y = Math.log(Math.tan((90 + lat) * Math.PI / 360)) / (Math.PI / 180); + y = y * GeoOperations._originShift / 180; + return [x, y]; + } + +//Converts XY point from (Spherical) Web Mercator EPSG:3785 (unofficially EPSG:900913) to lat/lon in WGS84 Datum + public static Convert900913ToWgs84(lonLat: [number, number]): [number, number] { + const lon = lonLat[0] + const lat = lonLat[1] + const x = 180 * lon / GeoOperations._originShift; + let y = 180 * lat / GeoOperations._originShift; + y = 180 / Math.PI * (2 * Math.atan(Math.exp(y * Math.PI / 180)) - Math.PI / 2); + return [x, y]; + } + + public static GeoJsonToWGS84(geojson){ + return turf.toWgs84(geojson) + } + /** * Calculates the intersection between two features. * Returns the length if intersecting a linestring and a (multi)polygon (in meters), returns a surface area (in m²) if intersecting two (multi)polygons diff --git a/Models/ThemeConfig/Json/LayerConfigJson.ts b/Models/ThemeConfig/Json/LayerConfigJson.ts index e13734c8a0..c15c90d4a1 100644 --- a/Models/ThemeConfig/Json/LayerConfigJson.ts +++ b/Models/ThemeConfig/Json/LayerConfigJson.ts @@ -53,6 +53,8 @@ export interface LayerConfigJson { * source: {geoJson: "https://my.source.net/some-tile-geojson-{layer}-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} * to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted by the location; {layer} is substituted with the id of the loaded layer * + * Some API's use a BBOX instead of a tile, this can be used by specifying {y_min}, {y_max}, {x_min} and {x_max} + * Some API's use a mercator-projection (EPSG:900913) instead of WGS84. Set the flag `mercatorCrs: true` in the source for this * * Note that both geojson-options might set a flag 'isOsmCache' indicating that the data originally comes from OSM too * @@ -61,7 +63,7 @@ export interface LayerConfigJson { * While still supported, this is considered deprecated */ source: ({ osmTags: AndOrTagConfigJson | string, overpassScript?: string } | - { osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number, isOsmCache?: boolean }) & ({ + { osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number, isOsmCache?: boolean, mercatorCrs?: boolean }) & ({ /** * The maximum amount of seconds that a tile is allowed to linger in the cache */ diff --git a/Models/ThemeConfig/LayerConfig.ts b/Models/ThemeConfig/LayerConfig.ts index ba6ca94b8d..9f49a6c210 100644 --- a/Models/ThemeConfig/LayerConfig.ts +++ b/Models/ThemeConfig/LayerConfig.ts @@ -124,6 +124,7 @@ export default class LayerConfig { geojsonSourceLevel: json.source["geoJsonZoomLevel"], overpassScript: json.source["overpassScript"], isOsmCache: json.source["isOsmCache"], + mercatorCrs: json.source["mercatorCrs"] }, this.id ); diff --git a/Models/ThemeConfig/SourceConfig.ts b/Models/ThemeConfig/SourceConfig.ts index 1d223bd2bd..b378d0acd3 100644 --- a/Models/ThemeConfig/SourceConfig.ts +++ b/Models/ThemeConfig/SourceConfig.ts @@ -1,4 +1,5 @@ import {TagsFilter} from "../../Logic/Tags/TagsFilter"; +import {RegexTag} from "../../Logic/Tags/RegexTag"; export default class SourceConfig { @@ -7,8 +8,10 @@ export default class SourceConfig { public readonly geojsonSource?: string; public readonly geojsonZoomLevel?: number; public readonly isOsmCacheLayer: boolean; + public readonly mercatorCrs: boolean; constructor(params: { + mercatorCrs?: boolean; osmTags?: TagsFilter, overpassScript?: string, geojsonSource?: string, @@ -33,10 +36,15 @@ export default class SourceConfig { console.error(params) throw `Source said it is a OSM-cached layer, but didn't define the actual source of the cache (in context ${context})` } - this.osmTags = params.osmTags; + if(params.geojsonSource !== undefined && params.geojsonSourceLevel !== undefined){ + if(! ["x","y","x_min","x_max","y_min","Y_max"].some(toSearch => params.geojsonSource.indexOf(toSearch) > 0)){ + throw `Source defines a geojson-zoomLevel, but does not specify {x} nor {y} (or equivalent), this is probably a bug (in context ${context})` + }} + this.osmTags = params.osmTags ?? new RegexTag("id",/.*/); this.overpassScript = params.overpassScript; this.geojsonSource = params.geojsonSource; this.geojsonZoomLevel = params.geojsonSourceLevel; this.isOsmCacheLayer = params.isOsmCache ?? false; + this.mercatorCrs = params.mercatorCrs ?? false; } } \ No newline at end of file diff --git a/UI/SpecialVisualizations.ts b/UI/SpecialVisualizations.ts index 1882152965..680b3e571c 100644 --- a/UI/SpecialVisualizations.ts +++ b/UI/SpecialVisualizations.ts @@ -457,7 +457,7 @@ There are also some technicalities in your theme to keep in mind: return new Combine([new FixedUiElement("The import button is disabled for unofficial themes to prevent accidents.").SetClass("alert"), new FixedUiElement("To test, add 'test=true' to the URL. The changeset will be printed in the console. Please open a PR to officialize this theme to actually enable the import button.")]) } - const tgsSpec = args[0].split(",").map(spec => { + const tgsSpec = args[0].split(";").map(spec => { const kv = spec.split("=").map(s => s.trim()); if (kv.length != 2) { throw "Invalid key spec: multiple '=' found in " + spec diff --git a/assets/themes/grb_import/README.md b/assets/themes/grb_import/README.md new file mode 100644 index 0000000000..2ed7bfbbfa --- /dev/null +++ b/assets/themes/grb_import/README.md @@ -0,0 +1,20 @@ + GRB Import helper +=================== + + +Preparing the CRAB dataset +-------------------------- + +```` +# The original data is downloaded from https://download.vlaanderen.be/Producten/Detail?id=447&title=CRAB_Adressenlijst# (the GML-file here ) +wget https://downloadagiv.blob.core.windows.net/crab-adressenlijst/GML/CRAB_Adressenlijst_GML.zip + +# Extract the zip file +unzip CRAB_Adressenlijst_GML.zip + +# convert the pesky GML file into geojson +ogr2ogr -progress -t_srs WGS84 -f \"GeoJson\" CRAB.geojson CrabAdr.gml + +# When done, this big file is sliced into tiles with the slicer script +node --max_old_space_size=8000 $(which ts-node) ~/git/MapComplete/scripts/slice.ts CRAB.geojson 18 ~/git/pietervdvn.github.io/CRAB_2021_10_26 +```` \ No newline at end of file diff --git a/assets/themes/grb.json b/assets/themes/grb_import/grb.json similarity index 80% rename from assets/themes/grb.json rename to assets/themes/grb_import/grb.json index 9c62526175..4f0d99d9db 100644 --- a/assets/themes/grb.json +++ b/assets/themes/grb_import/grb.json @@ -22,7 +22,7 @@ "socialImage": "", "layers": [ { - "id": "grb-fixmes", + "id": "osm-fixmes", "name": { "nl": "Fixmes op gebouwen" }, @@ -198,6 +198,43 @@ }, "wayHandling": 2, "presets": [] + }, + { + "id": "crab-addresses 2021-10-26", + "source": { + "osmTags": "HUISNR~*", + "geoJson": "https://raw.githubusercontent.com/pietervdvn/pietervdvn.github.io/master/CRAB_2021_10_26/tile_{z}_{x}_{y}.geojson", + "#geoJson": "https://pietervdvn.github.io/CRAB_2021_10_26/tile_{z}_{x}_{y}.geojson", + "geoJsonZoomLevel": 18, + "maxCacheAge": 0 + }, + "minzoom": 19, + "name": "CRAB-addressen", + "title": "CRAB-adres", + "icon": "circle:#bb3322", + "iconSize": "15,15,center", + "tagRenderings": [ + "all_tags", + { + "id": "import-button", + "render": "{import_button(addr:street=$STRAATNM; addr:housenumber=$HUISNR)}" + } + ] + }, + { + "id": "GRB", + "source": { + "geoJson": "https://betadata.grbosm.site/grb?bbox={x_min},{y_min},{x_max},{y_max}", + "geoJsonZoomLevel": 18, + "mercatorCrs": true, + "maxCacheAge": 0 + }, + "name": "GRB geometries", + "title": "GRB outline", + "minzoom": 19, + "tagRenderings": [ + "all_tags" + ] } ], "hideFromOverview": true, diff --git a/scripts/slice.ts b/scripts/slice.ts index d7ae5e186e..74673035b5 100644 --- a/scripts/slice.ts +++ b/scripts/slice.ts @@ -4,11 +4,84 @@ import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSou import * as readline from "readline"; import ScriptUtils from "./ScriptUtils"; +async function readFeaturesFromLineDelimitedJsonFile(inputFile: string): Promise { + const fileStream = fs.createReadStream(inputFile); + + const rl = readline.createInterface({ + input: fileStream, + crlfDelay: Infinity + }); + // Note: we use the crlfDelay option to recognize all instances of CR LF + // ('\r\n') in input.txt as a single line break. + + const allFeatures: any[] = [] + // @ts-ignore + for await (const line of rl) { + try { + allFeatures.push(JSON.parse(line)) + } catch (e) { + console.error("Could not parse", line) + break + } + if (allFeatures.length % 10000 === 0) { + ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now") + } + } + return allFeatures +} + +async function readGeojsonLineByLine(inputFile: string): Promise { + const fileStream = fs.createReadStream(inputFile); + + const rl = readline.createInterface({ + input: fileStream, + crlfDelay: Infinity + }); + // Note: we use the crlfDelay option to recognize all instances of CR LF + // ('\r\n') in input.txt as a single line break. + + const allFeatures: any[] = [] + let featuresSeen = false + // @ts-ignore + for await (let line: string of rl) { + if (!featuresSeen && line.startsWith("\"features\":")) { + featuresSeen = true; + continue; + } + if (!featuresSeen) { + continue + } + if (line.endsWith(",")) { + line = line.substring(0, line.length - 1) + } + + try { + allFeatures.push(JSON.parse(line)) + } catch (e) { + console.error("Could not parse", line) + break + } + if (allFeatures.length % 10000 === 0) { + ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now") + } + } + return allFeatures +} + +async function readFeaturesFromGeoJson(inputFile: string): Promise { + try { + return JSON.parse(fs.readFileSync(inputFile, "UTF-8")).features + } catch (e) { + // We retry, but with a line-by-line approach + return await readGeojsonLineByLine(inputFile) + } +} + async function main(args: string[]) { console.log("GeoJSON slicer") if (args.length < 3) { - console.log("USAGE: ") + console.log("USAGE: ") return } @@ -23,39 +96,24 @@ async function main(args: string[]) { console.log("Using directory ", outputDirectory) - const fileStream = fs.createReadStream(inputFile); - - const rl = readline.createInterface({ - input: fileStream, - crlfDelay: Infinity - }); - // Note: we use the crlfDelay option to recognize all instances of CR LF - // ('\r\n') in input.txt as a single line break. - - const allFeatures = [] - // @ts-ignore - for await (const line of rl) { - // Each line in input.txt will be successively available here as `line`. - try{ - allFeatures.push(JSON.parse(line)) - }catch (e) { - console.error("Could not parse", line) - break - } - if(allFeatures.length % 10000 === 0){ - ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now") - } + let allFeatures: any []; + if (inputFile.endsWith(".geojson")) { + allFeatures = await readFeaturesFromGeoJson(inputFile) + } else { + allFeatures = await readFeaturesFromLineDelimitedJsonFile(inputFile) } - + + console.log("Loaded all", allFeatures.length, "points") - - const keysToRemove = ["ID","STRAATNMID","NISCODE","GEMEENTE","POSTCODE","HERKOMST","APPTNR"] + + const keysToRemove = ["ID", "STRAATNMID", "NISCODE", "GEMEENTE", "POSTCODE", "HERKOMST"] for (const f of allFeatures) { for (const keyToRm of keysToRemove) { delete f.properties[keyToRm] } + delete f.bbox } - + //const knownKeys = Utils.Dedup([].concat(...allFeatures.map(f => Object.keys(f.properties)))) //console.log("Kept keys: ", knownKeys) @@ -67,11 +125,15 @@ async function main(args: string[]) { maxFeatureCount: Number.MAX_VALUE, registerTile: tile => { const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson` + const features = tile.features.data.map(ff => ff.feature) + features.forEach(f => { + delete f.bbox + }) fs.writeFileSync(path, JSON.stringify({ "type": "FeatureCollection", - "features": tile.features.data.map(ff => ff.feature) + "features": features }, null, " ")) - console.log("Written ", path, "which has ", tile.features.data.length, "features") + ScriptUtils.erasableLog("Written ", path, "which has ", tile.features.data.length, "features") } } )