midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

fix merge conflicts

Browse source
This commit is contained in:
Ward 2021-07-27 15:57:25 +02:00
commit 597dc9900a
6 changed files with 93 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

90
Logic/FeatureSource/GeoJsonSource.ts
View file

@ -25,7 +25,7 @@ export default class GeoJsonSource implements FeatureSource {
let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id);
this.name = "GeoJsonSource of " + url;
const zoomLevel = flayer.layerDef.source.geojsonZoomLevel;
this.isOsmCache = flayer.layerDef.source.isOsmCacheLayer;
this.features = new UIEventSource<{ feature: any; freshness: Date }[]>([])
@ -50,7 +50,7 @@ export default class GeoJsonSource implements FeatureSource {
* @param locationControl
* @constructor
*/
public static ConstructMultiSource(flayers: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }[], locationControl: UIEventSource<Loc>): GeoJsonSource[] {
public static ConstructMultiSource(flayers: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }[], locationControl: UIEventSource<Loc>): FeatureSource[] {
const flayersPerSource = new Map<string, { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }[]>();
for (const flayer of flayers) {
@ -65,7 +65,7 @@ export default class GeoJsonSource implements FeatureSource {
flayersPerSource.get(url).push(flayer)
}
const sources: GeoJsonSource[] = []
const sources: FeatureSource[] = []
flayersPerSource.forEach((flayers, key) => {
if (flayers.length == 1) {
@ -112,7 +112,19 @@ export default class GeoJsonSource implements FeatureSource {
}
const neededTiles = locationControl.map(
_ => {
location => {
if (!flayer.isDisplayed.data) {
// No need to download! - the layer is disabled
return undefined;
}
if (location.zoom < flayer.layerDef.minzoom ||
location.zoom > flayer.layerDef.maxzoom) {
// No need to download! - the layer is disabled
console.log("Not loading layers for " + url, "zoom" + location.zoom, " not between", flayer.layerDef.minzoom, "and", flayer.layerDef.maxzoom)
return undefined;
}
// Yup, this is cheating to just get the bounds here
const bounds = State.state.leafletMap.data.getBounds()
const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
@ -126,14 +138,6 @@ export default class GeoJsonSource implements FeatureSource {
if (needed === undefined) {
return;
}
if (!flayer.isDisplayed.data) {
// No need to download! - the layer is disabled
return;
}
if (locationControl.data.zoom < flayer.layerDef.minzoom) {
return;
}
needed.forEach(neededTile => {
if (loadedTiles.has(neededTile)) {
@ -153,42 +157,42 @@ export default class GeoJsonSource implements FeatureSource {
const self = this;
Utils.downloadJson(url)
.then(json => {
if (json.elements === [] && json.remarks.indexOf("runtime error") > 0) {
self.onFail("Runtime error (timeout)", url)
return;
}
const time = new Date();
const newFeatures: { feature: any, freshness: Date } [] = []
let i = 0;
let skipped = 0;
for (const feature of json.features) {
if (feature.properties.id === undefined) {
feature.properties.id = url + "/" + i;
feature.id = url + "/" + i;
i++;
if (json.elements === [] && json.remarks.indexOf("runtime error") > 0) {
self.onFail("Runtime error (timeout)", url)
return;
}
if (self.seenids.has(feature.properties.id)) {
skipped++;
continue;
}
self.seenids.add(feature.properties.id)
const time = new Date();
const newFeatures: { feature: any, freshness: Date } [] = []
let i = 0;
let skipped = 0;
for (const feature of json.features) {
if (feature.properties.id === undefined) {
feature.properties.id = url + "/" + i;
feature.id = url + "/" + i;
i++;
}
if (self.seenids.has(feature.properties.id)) {
skipped++;
continue;
}
self.seenids.add(feature.properties.id)
let freshness: Date = time;
if (feature.properties["_last_edit:timestamp"] !== undefined) {
freshness = new Date(feature.properties["_last_edit:timestamp"])
let freshness: Date = time;
if (feature.properties["_last_edit:timestamp"] !== undefined) {
freshness = new Date(feature.properties["_last_edit:timestamp"])
}
newFeatures.push({feature: feature, freshness: freshness})
}
console.debug("Downloaded " + newFeatures.length + " new features and " + skipped + " already seen features from " + url);
if (newFeatures.length == 0) {
return;
}
newFeatures.push({feature: feature, freshness: freshness})
}
console.debug("Downloaded " + newFeatures.length + " new features and " + skipped + " already seen features from " + url);
eventSource.setData(eventSource.data.concat(newFeatures))
if (newFeatures.length == 0) {
return;
}
eventSource.setData(eventSource.data.concat(newFeatures))
}).catch(msg => self.onFail(msg, url))
}).catch(msg => self.onFail(msg, url))
}
}

8
assets/themes/natuurpunt/natuurpunt.json
View file

@ -40,7 +40,7 @@
"geoJsonZoomLevel": 12,
"isOsmCache": true
},
"minzoom": "13",
"minzoom": 13,
"icon": {
"render": "circle:#FE6F32;./assets/themes/natuurpunt/nature_reserve.svg"
}
@ -56,9 +56,11 @@
"operator~.*[nN]atuurpunt.*"
]
},
"geoJson": "https://pietervdvn.github.io/natuurpunt_cache/natuurpunt_nature_reserve_points.geojson"
"geoJson": "https://pietervdvn.github.io/natuurpunt_cache/natuurpunt_nature_reserve_points.geojson",
"isOsmCache": "duplicate"
},
"minzoom": "0",
"minzoom": 1,
"maxzoom": 13,
"icon": {
"render": "circle:#FE6F32;./assets/themes/natuurpunt/nature_reserve.svg"
}

80
package-lock.json generated
View file

@ -2943,11 +2943,6 @@
"integrity": "sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E=",
"dev": true
},
"array-flatten": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-3.0.0.tgz",
"integrity": "sha512-zPMVc3ZYlGLNk4mpK1NzP2wg0ml9t7fUgDsayR5Y5rSzxQilzR9FGu/EH2jQOcKSAeAfWeylyW8juy3OkWRvNA=="
},
"array-unique": {
"version": "0.3.2",
"resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
@ -8539,9 +8534,9 @@
"integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ=="
},
"node-abi": {
"version": "2.26.0",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.26.0.tgz",
"integrity": "sha512-ag/Vos/mXXpWLLAYWsAoQdgS+gW7IwvgMLOgqopm/DbzAjazLltzgzpVMsFlgmo9TzG5hGXeaBZx2AI731RIsQ==",
"version": "2.30.0",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.30.0.tgz",
"integrity": "sha512-g6bZh3YCKQRdwuO/tSZZYJAw622SjsRfJ2X0Iy4sSOHZ34/sPPdVBn8fev2tj7njzLwuqPw9uMtGsGkO5kIQvg==",
"requires": {
"semver": "^5.4.1"
},
@ -8637,11 +8632,6 @@
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.71.tgz",
"integrity": "sha512-zR6HoT6LrLCRBwukmrVbHv0EpEQjksO6GmFcZQQuCAy139BEsoVKPYnf3jongYW83fAa1torLGYwxxky/p28sg=="
},
"noop-logger": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz",
"integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI="
},
"normalize-package-data": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz",
@ -10058,9 +10048,9 @@
"integrity": "sha512-eSToKjNLu0FiF76SSGMHjOFXYzAc/CJqi677Sq6hYvcvFCBtD6de/W5l+0IYPf7ypscqAfjCttxvTdMJt5Gj8Q=="
},
"prebuild-install": {
"version": "6.1.2",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-6.1.2.tgz",
"integrity": "sha512-PzYWIKZeP+967WuKYXlTOhYBgGOvTRSfaKI89XnfJ0ansRAH7hDU45X+K+FZeI1Wb/7p/NnuctPH3g0IqKUuSQ==",
"version": "6.1.3",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-6.1.3.tgz",
"integrity": "sha512-iqqSR84tNYQUQHRXalSKdIaM8Ov1QxOVuBNWI7+BzZWv6Ih9k75wOnH1rGQ9WWTaaLkTpxWKIciOF0KyfM74+Q==",
"requires": {
"detect-libc": "^1.0.3",
"expand-template": "^2.0.3",
@ -10069,25 +10059,12 @@
"mkdirp-classic": "^0.5.3",
"napi-build-utils": "^1.0.1",
"node-abi": "^2.21.0",
"noop-logger": "^0.1.1",
"npmlog": "^4.0.1",
"pump": "^3.0.0",
"rc": "^1.2.7",
"simple-get": "^3.0.3",
"tar-fs": "^2.0.0",
"tunnel-agent": "^0.6.0"
},
"dependencies": {
"simple-get": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz",
"integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==",
"requires": {
"decompress-response": "^4.2.0",
"once": "^1.3.1",
"simple-concat": "^1.0.0"
}
}
}
},
"prelude-ls": {
@ -10818,18 +10795,16 @@
}
},
"sharp": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.27.2.tgz",
"integrity": "sha512-w3FVoONPG/x5MXCc3wsjOS+b9h3CI60qkus6EPQU4dkT0BDm0PyGhDCK6KhtfT3/vbeOMOXAKFNSw+I3QGWkMA==",
"version": "0.28.3",
"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.28.3.tgz",
"integrity": "sha512-21GEP45Rmr7q2qcmdnjDkNP04Ooh5v0laGS5FDpojOO84D1DJwUijLiSq8XNNM6e8aGXYtoYRh3sVNdm8NodMA==",
"requires": {
"array-flatten": "^3.0.0",
"color": "^3.1.3",
"detect-libc": "^1.0.3",
"node-addon-api": "^3.1.0",
"npmlog": "^4.1.2",
"prebuild-install": "^6.0.1",
"semver": "^7.3.4",
"simple-get": "^4.0.0",
"node-addon-api": "^3.2.0",
"prebuild-install": "^6.1.2",
"semver": "^7.3.5",
"simple-get": "^3.1.0",
"tar-fs": "^2.1.1",
"tunnel-agent": "^0.6.0"
},
@ -10843,9 +10818,9 @@
}
},
"node-addon-api": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.1.0.tgz",
"integrity": "sha512-flmrDNB06LIl5lywUz7YlNGZH/5p0M7W28k8hzd9Lshtdh1wshD2Y+U4h9LD6KObOy1f+fEVdgprPrEymjM5uw=="
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz",
"integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A=="
},
"semver": {
"version": "7.3.5",
@ -10886,28 +10861,13 @@
"integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q=="
},
"simple-get": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.0.tgz",
"integrity": "sha512-ZalZGexYr3TA0SwySsr5HlgOOinS4Jsa8YB2GJ6lUNAazyAu4KG/VmzMTwAt2YVXzzVj8QmefmAonZIK2BSGcQ==",
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz",
"integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==",
"requires": {
"decompress-response": "^6.0.0",
"decompress-response": "^4.2.0",
"once": "^1.3.1",
"simple-concat": "^1.0.0"
},
"dependencies": {
"decompress-response": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
"integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
"requires": {
"mimic-response": "^3.1.0"
}
},
"mimic-response": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
"integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="
}
}
},
"simple-swizzle": {

2
package.json
View file

@ -82,7 +82,7 @@
"parcel": "^1.2.4",
"postcss": "^7.0.36",
"prompt-sync": "^4.2.0",
"sharp": "^0.27.0",
"sharp": "^0.28.3",
"tslint": "^6.1.3"
},
"devDependencies": {

4
scripts/ScriptUtils.ts
View file

@ -86,6 +86,10 @@ export default class ScriptUtils {
})
}
public static erasableLog(...text) {
process.stdout.write("\r "+text.join(" ")+" \r")
}
public static sleep(ms) {
if (ms <= 0) {

22
scripts/generateCache.ts
View file

@ -34,7 +34,7 @@ function createOverpassObject(theme: LayoutConfig) {
if (layer.source.geojsonSource !== undefined) {
// This layer defines a geoJson-source
// SHould it be cached?
if (!layer.source.isOsmCacheLayer) {
if (layer.source.isOsmCacheLayer !== true) {
continue;
}
}
@ -131,7 +131,7 @@ async function downloadExtraData(theme: LayoutConfig)/* : any[] */ {
if (source === undefined) {
continue;
}
if (layer.source.isOsmCacheLayer) {
if (layer.source.isOsmCacheLayer !== undefined) {
// Cached layers are not considered here
continue;
}
@ -148,7 +148,7 @@ function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig, extra
for (let y = r.ystart; y <= r.yend; y++) {
processed++;
const filename = rawJsonName(targetdir, x, y, r.zoomlevel)
console.log(" Post processing", processed, "/", r.total, filename)
ScriptUtils.erasableLog(" Post processing", processed, "/", r.total, filename)
if (!existsSync(filename)) {
console.error("Not found - and not downloaded. Run this script again!: " + filename)
continue;
@ -234,9 +234,8 @@ function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfig) {
})
const new_path = geoJsonName(targetdir + "_" + layer.id, x, y, z);
console.log(new_path, " has ", geojson.features.length, " features after filtering (dropped ", oldLength - geojson.features.length, ")")
ScriptUtils.erasableLog(new_path, " has ", geojson.features.length, " features after filtering (dropped ", oldLength - geojson.features.length, ")")
if (geojson.features.length == 0) {
console.log("Not writing geojson file as it is empty", new_path)
continue;
}
writeFileSync(new_path, JSON.stringify(geojson, null, " "))
@ -289,9 +288,20 @@ async function createOverview(targetdir: string, r: TileRange, z: number, layern
}
}
const featuresDedup = []
const seen = new Set<string>()
for (const feature of allFeatures) {
const id = feature.properties.id
if(seen.has(id)){
continue
}
seen.add(id)
featuresDedup.push(feature)
}
const geojson = {
"type": "FeatureCollection",
"features": allFeatures
"features": featuresDedup
}
writeFileSync(targetdir + "_" + layername + "_points.geojson", JSON.stringify(geojson, null, " "))
}