ilja_space/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Fix multilayer geojson source

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-04-23 20:09:27 +02:00
parent a6e3b41b1d
commit f4dacab9ef
7 changed files with 97 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Customizations/JSON/LayerConfigJson.ts
View file

@ -30,7 +30,7 @@ export interface LayerConfigJson {
*
* source: {osmTags: "key=value"} will fetch all objects with given tags from OSM. Currently, this will create a query to overpass and fetch the data - in the future this might fetch from the OSM API
* source: {geoJson: "https://my.source.net/some-geo-data.geojson"} to fetch a geojson from a third party source
* source: {geoJson: "https://my.source.net/some-tile-geojson-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted
* source: {geoJson: "https://my.source.net/some-tile-geojson-{layer}-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted by the location; {layer} is substituted with the id of the loaded layer
*
* source: {overpassScript: "<custom overpass tags>"} when you want to do special things. _This should be really rare_.
* This means that the data will be pulled from overpass with this script, and will ignore the osmTags for the query

90
Logic/FeatureSource/GeoJsonSource.ts
View file

@ -16,15 +16,15 @@ export default class GeoJsonSource implements FeatureSource {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name;
private readonly onFail: ((errorMsg: any, url: string) => void) = undefined;
private onFail: ((errorMsg: any, url: string) => void) = undefined;
private readonly layerId: string;
private readonly seenids: Set<string> = new Set<string>()
constructor(locationControl: UIEventSource<Loc>,
private constructor(locationControl: UIEventSource<Loc>,
flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig },
onFail?: ((errorMsg: any) => void)) {
this.layerId = flayer.layerDef.id;
let url = flayer.layerDef.source.geojsonSource;
let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id);
this.name = "GeoJsonSource of " + url;
const zoomLevel = flayer.layerDef.source.geojsonZoomLevel;
@ -34,58 +34,66 @@ export default class GeoJsonSource implements FeatureSource {
// This is a classic, static geojson layer
if (onFail === undefined) {
onFail = errorMsg => {
console.warn(`Could not load geojson layer from`, url, "due to", errorMsg)
}
}
this.onFail = onFail;
this.LoadJSONFrom(url)
} else {
// This is a dynamic template with a fixed zoom level
url = url.replace("{z}", "" + zoomLevel)
const loadedTiles = new Set<string>();
const self = this;
this.onFail = (msg, url) => {
console.warn(`Could not load geojson layer from`, url, "due to", msg)
loadedTiles.delete(url)
}
this.ConfigureDynamicLayer(url, zoomLevel, locationControl, flayer)
}
}
private ConfigureDynamicLayer(url: string, zoomLevel: number, locationControl: UIEventSource<Loc>, flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }){
// This is a dynamic template with a fixed zoom level
url = url.replace("{z}", "" + zoomLevel)
const loadedTiles = new Set<string>();
const self = this;
this.onFail = (msg, url) => {
console.warn(`Could not load geojson layer from`, url, "due to", msg)
loadedTiles.add(url); // We add the url to the 'loadedTiles' in order to not reload it in the future
}
const neededTiles = locationControl.map(
location => {
if (!flayer.isDisplayed.data) {
return undefined;
const neededTiles = locationControl.map(
location => {
// Yup, this is cheating to just get the bounds here
const bounds = State.state.leafletMap.data.getBounds()
const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const needed = new Set<string>();
for (let x = tileRange.xstart; x <= tileRange.xend; x++) {
for (let y = tileRange.ystart; y <= tileRange.yend; y++) {
let neededUrl = url.replace("{x}", "" + x).replace("{y}", "" + y);
needed.add(neededUrl)
}
// Yup, this is cheating to just get the bounds here
const bounds = State.state.leafletMap.data.getBounds()
const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const needed = new Set<string>();
for (let x = tileRange.xstart; x <= tileRange.xend; x++) {
for (let y = tileRange.ystart; y <= tileRange.yend; y++) {
let neededUrl = url.replace("{x}", "" + x).replace("{y}", "" + y);
needed.add(neededUrl)
}
}
return needed;
}
);
neededTiles.stabilized(250).addCallback((needed: Set<string>) => {
if (needed === undefined) {
return needed;
}
, [flayer.isDisplayed]);
neededTiles.stabilized(250).addCallback((needed: Set<string>) => {
if (needed === undefined) {
return;
}
if (!flayer.isDisplayed.data) {
// No need to download! - the layer is disabled
return;
}
console.log("???", locationControl, flayer.layerDef)
if(locationControl.data.zoom < flayer.layerDef.minzoom){
console.log("Not downloading ", needed, "not sufficiently zoomed")
return;
}
needed.forEach(neededTile => {
if (loadedTiles.has(neededTile)) {
return;
}
needed.forEach(neededTile => {
if (loadedTiles.has(neededTile)) {
return;
}
loadedTiles.add(neededTile)
self.LoadJSONFrom(neededTile)
loadedTiles.add(neededTile)
self.LoadJSONFrom(neededTile)
})
})
})
}
}
/**
@ -98,7 +106,7 @@ export default class GeoJsonSource implements FeatureSource {
const flayersPerSource = new Map<string, { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }[]>();
for (const flayer of flayers) {
const url = flayer.layerDef.source.geojsonSource
const url = flayer.layerDef.source.geojsonSource?.replace(/{layer}/g, flayer.layerDef.id)
if (url === undefined) {
continue;
}

3
Logic/UIEventSource.ts
View file

@ -98,7 +98,8 @@ export class UIEventSource<T> {
const self = this;
const newSource = new UIEventSource<J>(
f(this.data)
f(this.data),
"map("+this.tag+")"
);
const update = function () {

2
Logic/Web/QueryParameters.ts
View file

@ -79,7 +79,7 @@ export class QueryParameters {
return QueryParameters.knownSources[key];
}
QueryParameters.addOrder(key);
const source = new UIEventSource<string>(deflt);
const source = new UIEventSource<string>(deflt, "&"+key);
QueryParameters.knownSources[key] = source;
source.addCallback(() => QueryParameters.Serialize())
return source;

2
Models/Constants.ts
View file

@ -2,7 +2,7 @@ import { Utils } from "../Utils";
export default class Constants {
public static vNumber = "0.6.11b";
public static vNumber = "0.6.11c";
// The user journey states thresholds when a new feature gets unlocked
public static userJourney = {

14
assets/themes/speelplekken/speelplekken.json
View file

@ -55,7 +55,12 @@
"minzoom":14
}
},
"sport_pitch",
{
"builtin": "sport_pitch",
"override": {
"minzoom": 14
}
},
{
"builtin": "slow_roads",
@ -174,9 +179,14 @@
}
}
],
"clustering": {
"maxZoom": 16,
"minNeededElements": 100
},
"overrideAll": {
"source": {
"geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
"geoJsonLocal": "http://127.0.0.1:8080/speelplekken_{layer}_{z}_{x}_{y}.geojson",
"geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{layer}_{z}_{x}_{y}.geojson",
"geoJsonZoomLevel": 14
}
},

36
scripts/generateCache.ts
View file

@ -2,6 +2,7 @@
* Generates a collection of geojson files based on an overpass query for a given theme
*/
import {TileRange, Utils} from "../Utils";
Utils.runningFromConsole = true
import {Overpass} from "../Logic/Osm/Overpass";
import {existsSync, readFileSync, writeFileSync} from "fs";
@ -15,7 +16,6 @@ import * as OsmToGeoJson from "osmtogeojson";
import MetaTagging from "../Logic/MetaTagging";
function createOverpassObject(theme: LayoutConfig) {
let filters: TagsFilter[] = [];
let extraScripts: string[] = [];
@ -105,8 +105,8 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/
console.log("Didn't get an answer yet - waiting more")
}
}
if(!success){
if (!success) {
failed++;
console.log("Hit the rate limit - waiting 90s")
for (let i = 0; i < 90; i++) {
@ -159,12 +159,37 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig)
// Extract the relationship information
const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm))
MetaTagging.addMetatags(featuresFreshness, relations, theme.layers);
writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson))
writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson, null, " "))
}
}
}
async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfig) {
let processed = 0;
const z = r.zoomlevel;
for (let x = r.xstart; x <= r.xend; x++) {
for (let y = r.ystart; y <= r.yend; y++) {
const file = readFileSync(geoJsonName(targetdir, x, y, z), "UTF8")
for (const layer of theme.layers) {
const geojson = JSON.parse(file)
geojson.features = geojson.features.filter(f => f._matching_layer_id === layer.id)
if(geojson.features.length == 0){
continue;
}
const new_path = geoJsonName(targetdir+"_"+layer.id, x, y, z);
writeFileSync(new_path, JSON.stringify(geojson, null, " "))
}
}
}
}
async function main(args: string[]) {
if (args.length == 0) {
@ -203,6 +228,7 @@ async function main(args: string[]) {
} while (failed > 0)
await postProcess(targetdir, tileRange, theme)
await splitPerLayer(targetdir, tileRange, theme)
}