ilja_space/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Enable caching for some natuurpunt layers

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-07-18 19:08:14 +02:00
parent 2ff279566d
commit 6d73330f6b
2 changed files with 18 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -20,7 +20,7 @@
"generate:layouts": "ts-node scripts/generateLayouts.ts",
"generate:docs": "ts-node scripts/generateDocs.ts && ts-node scripts/generateTaginfoProjectFiles.ts",
"generate:cache:speelplekken": "npm run generate:layeroverview && ts-node scripts/generateCache.ts speelplekken 14 ../pietervdvn.github.io/speelplekken_cache/ 51.20 4.35 51.09 4.56",
"generate:cache:natuurpunt": "npm run generate:layeroverview && ts-node scripts/generateCache.ts natuurpunt 6 ../pietervdvn.github.io/natuurpunt_cache/ 50.40 2.1 51.54 6.4",
"generate:cache:natuurpunt": "npm run generate:layeroverview && ts-node scripts/generateCache.ts natuurpunt 12 ../pietervdvn.github.io/natuurpunt_cache/ 50.40 2.1 51.54 6.4",
"generate:layeroverview": "npm run generate:licenses && echo {\\\"layers\\\":[], \\\"themes\\\":[]} > ./assets/generated/known_layers_and_themes.json && ts-node scripts/generateLayerOverview.ts --no-fail",
"generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail",
"generate:report": "cd Docs/Tools && ./compileStats.sh && git commit . -m 'New statistics ands graphs' && git push",

23
scripts/generateCache.ts
View file

@ -88,13 +88,24 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/
await ScriptUtils.DownloadJSON(url)
.then(json => {
if (json.elements.length === 0) {
console.log("Got an empty response!")
if ((<string>json.remark ?? "").startsWith("runtime error")) {
console.error("Got a runtime error: ", json.remark)
failed++;
return
}
}
console.log("Got the response - writing to ", filename)
writeFileSync(filename, JSON.stringify(json, null, " "));
}
)
.catch(err => {
console.log(url)
console.log("Could not download - probably hit the rate limit; waiting a bit. ("+err+")")
console.log("Could not download - probably hit the rate limit; waiting a bit. (" + err + ")")
failed++;
return ScriptUtils.sleep(60000).then(() => console.log("Waiting is done"))
})
@ -143,7 +154,7 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig,
// We read the raw OSM-file and convert it to a geojson
const rawOsm = JSON.parse(readFileSync(filename, "UTF8"))
// Create and save the geojson file - which is the main chunk of the data
const geojson = OsmToGeoJson.default(rawOsm);
const osmTime = new Date(rawOsm.osm3s.timestamp_osm_base);
@ -168,7 +179,7 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig,
// Extract the relationship information
const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm))
MetaTagging.addMetatags(featuresFreshness, new UIEventSource<{feature: any; freshness: Date}[]>(featuresFreshness) , relations, theme.layers, false);
MetaTagging.addMetatags(featuresFreshness, new UIEventSource<{ feature: any; freshness: Date }[]>(featuresFreshness), relations, theme.layers, false);
for (const feature of geojson.features) {
@ -192,7 +203,7 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig,
delete feature["bbox"]
}
const targetPath = geoJsonName(targetdir+".unfiltered", x, y, r.zoomlevel)
const targetPath = geoJsonName(targetdir + ".unfiltered", x, y, r.zoomlevel)
// This is the geojson file containing all features
writeFileSync(targetPath, JSON.stringify(geojson, null, " "))
@ -204,7 +215,7 @@ async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfi
const z = r.zoomlevel;
for (let x = r.xstart; x <= r.xend; x++) {
for (let y = r.ystart; y <= r.yend; y++) {
const file = readFileSync(geoJsonName(targetdir+".unfiltered", x, y, z), "UTF8")
const file = readFileSync(geoJsonName(targetdir + ".unfiltered", x, y, z), "UTF8")
for (const layer of theme.layers) {
if (!layer.source.isOsmCacheLayer) {
@ -222,7 +233,7 @@ async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfi
return true;
})
const new_path = geoJsonName(targetdir + "_" + layer.id, x, y, z);
console.log(new_path, " has ", geojson.features.length, " features after filtering (dropped ", oldLength - geojson.features.length,")" )
console.log(new_path, " has ", geojson.features.length, " features after filtering (dropped ", oldLength - geojson.features.length, ")")
if (geojson.features.length == 0) {
console.log("Not writing geojson file as it is empty", new_path)
continue;