From 3c51c28157aed25dcdac7dbda76bb604bbd0a043 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Fri, 10 Sep 2021 00:00:48 +0200 Subject: [PATCH] Attempt to fix caching process --- assets/layers/binocular/binocular.json | 4 ++-- assets/themes/speelplekken/speelplekken.json | 13 +++++++------ package.json | 1 + scripts/generateCache.ts | 17 ++++++++++------- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/assets/layers/binocular/binocular.json b/assets/layers/binocular/binocular.json index 690a6f38d..3c31bcfac 100644 --- a/assets/layers/binocular/binocular.json +++ b/assets/layers/binocular/binocular.json @@ -84,8 +84,8 @@ "nl": "verrekijker" }, "description": { - "en": "A telescope or pair of binoculars mounted on a pole, available to the public to look around. ", - "nl": "Een telescoop of verrekijker die op een vaste plaats gemonteerd staat waar iedereen door mag kijken. " + "en": "A telescope or pair of binoculars mounted on a pole, available to the public to look around. ", + "nl": "Een telescoop of verrekijker die op een vaste plaats gemonteerd staat waar iedereen door mag kijken. " }, "preciseInput": { "preferredBackground": "photo" diff --git a/assets/themes/speelplekken/speelplekken.json b/assets/themes/speelplekken/speelplekken.json index f5f1b98d9..14e03b4d9 100644 --- a/assets/themes/speelplekken/speelplekken.json +++ b/assets/themes/speelplekken/speelplekken.json @@ -30,7 +30,8 @@ "id": "shadow", "source": { "geoJson": "https://raw.githubusercontent.com/pietervdvn/MapComplete/master/assets/themes/speelplekken/shadow.geojson", - "osmTags": "shadow=yes" + "osmTags": "shadow=yes", + "isOsmCache": false }, "color": "#444444", "width": { @@ -49,7 +50,7 @@ "minzoom": 12, "calculatedTags": [ "_is_shadowed=feat.overlapWith('shadow').length > 0 ? 'yes': ''", - "_video:id=new URL(feat.properties.video).searchParams.get('v')" + "_video:id= feat.properties.video === undefined ? undefined : new URL(feat.properties.video).searchParams.get('v')" ] } }, @@ -66,7 +67,7 @@ }, "calculatedTags": [ "_is_shadowed=feat.overlapWith('shadow').length > 0 ? 'yes': ''", - "_video:id=new URL(feat.properties.video).searchParams.get('v')" + "_video:id=feat.properties.video === undefined ? undefined : new URL(feat.properties.video).searchParams.get('v')" ] } }, @@ -83,7 +84,7 @@ }, "calculatedTags": [ "_is_shadowed=feat.overlapWith('shadow').length > 0 ? 'yes': ''", - "_video:id=new URL(feat.properties.video).searchParams.get('v')" + "_video:id=feat.properties.video === undefined ? undefined : new URL(feat.properties.video).searchParams.get('v')" ] } }, @@ -100,7 +101,7 @@ }, "calculatedTags": [ "_is_shadowed=feat.overlapWith('shadow').length > 0 ? 'yes': ''", - "_video:id=new URL(feat.properties.video).searchParams.get('v')" + "_video:id=feat.properties.video === undefined ? undefined : new URL(feat.properties.video).searchParams.get('v')" ] } }, @@ -116,7 +117,7 @@ }, "calculatedTags": [ "_is_shadowed=feat.overlapWith('shadow').length > 0 ? 'yes': ''", - "_video:id=new URL(feat.properties.video).searchParams.get('v')" + "_video:id=feat.properties.video === undefined ? undefined : new URL(feat.properties.video).searchParams.get('v')" ] } }, diff --git a/package.json b/package.json index ad24fb535..8baeb62fd 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "reset:translations": "ts-node scripts/generateTranslations.ts --ignore-weblate", "generate:layouts": "ts-node scripts/generateLayouts.ts", "generate:docs": "ts-node scripts/generateDocs.ts && ts-node scripts/generateTaginfoProjectFiles.ts", + "generate:cache:speelplekken:mini": "npm run generate:layeroverview && ts-node scripts/generateCache.ts speelplekken 14 ../pietervdvn.github.io/speelplekken_cache/ 51.181710380278176 4.423413276672363 51.193007664772495 4.444141387939452", "generate:cache:speelplekken": "npm run generate:layeroverview && ts-node scripts/generateCache.ts speelplekken 14 ../pietervdvn.github.io/speelplekken_cache/ 51.20 4.35 51.09 4.56", "generate:cache:natuurpunt": "npm run generate:layeroverview && ts-node scripts/generateCache.ts natuurpunt 12 ../pietervdvn.github.io/natuurpunt_cache/ 50.40 2.1 51.54 6.4 --generate-point-overview nature_reserve,visitor_information_centre", "generate:layeroverview": "npm run generate:licenses && echo {\\\"layers\\\":[], \\\"themes\\\":[]} > ./assets/generated/known_layers_and_themes.json && ts-node scripts/generateLayerOverview.ts --no-fail", diff --git a/scripts/generateCache.ts b/scripts/generateCache.ts index 62dc5039f..c043bc224 100644 --- a/scripts/generateCache.ts +++ b/scripts/generateCache.ts @@ -2,13 +2,13 @@ * Generates a collection of geojson files based on an overpass query for a given theme */ import {Utils} from "../Utils"; +Utils.runningFromConsole = true import {Overpass} from "../Logic/Osm/Overpass"; import * as fs from "fs"; import {existsSync, readFileSync, writeFileSync} from "fs"; import {TagsFilter} from "../Logic/Tags/TagsFilter"; import {Or} from "../Logic/Tags/Or"; import {AllKnownLayouts} from "../Customizations/AllKnownLayouts"; -import ScriptUtils from "./ScriptUtils"; import ExtractRelations from "../Logic/Osm/ExtractRelations"; import * as OsmToGeoJson from "osmtogeojson"; import MetaTagging from "../Logic/MetaTagging"; @@ -17,8 +17,9 @@ import {UIEventSource} from "../Logic/UIEventSource"; import {TileRange} from "../Models/TileRange"; import LayoutConfig from "../Models/ThemeConfig/LayoutConfig"; import LayerConfig from "../Models/ThemeConfig/LayerConfig"; +import ScriptUtils from "./ScriptUtils"; -Utils.runningFromConsole = true +ScriptUtils.fixUtils() function createOverpassObject(theme: LayoutConfig) { @@ -52,7 +53,7 @@ function createOverpassObject(theme: LayoutConfig) { if (filters.length + extraScripts.length === 0) { throw "Nothing to download! The theme doesn't declare anything to download" } - return new Overpass(new Or(filters), extraScripts, new UIEventSource("https://overpass-api.de/api/interpreter"), + return new Overpass(new Or(filters), extraScripts, new UIEventSource("https://overpass.kumi.systems/api/interpreter"), //https://overpass-api.de/api/interpreter"), new UIEventSource(60)); } @@ -112,9 +113,11 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/ failed++; return ScriptUtils.sleep(60000).then(() => console.log("Waiting is done")) }) - // Cooldown - console.debug("Cooling down 10s") - await ScriptUtils.sleep(10000) + + if (x < r.xend || y < r.yend) { + console.debug("Cooling down 10s") + await ScriptUtils.sleep(10000) + } } } @@ -132,7 +135,7 @@ async function downloadExtraData(theme: LayoutConfig)/* : any[] */ { if (source === undefined) { continue; } - if (layer.source.isOsmCacheLayer !== undefined) { + if (layer.source.isOsmCacheLayer !== undefined && layer.source.isOsmCacheLayer !== false) { // Cached layers are not considered here continue; }