Merge branch 'develop' into alpha

InitUiElements.ts

@@ -426,10 +426,10 @@ export class InitUiElements {
 
 
 
-
+        const clustering = State.state.layoutToUse.clustering
         const clusterCounter = TileHierarchyAggregator.createHierarchy()
         new ShowDataLayer({
-            features: clusterCounter.getCountsForZoom(State.state.locationControl, State.state.layoutToUse.clustering.minNeededElements),
+            features: clusterCounter.getCountsForZoom(clustering, State.state.locationControl, State.state.layoutToUse.clustering.minNeededElements),
             leafletMap: State.state.leafletMap,
             layerToShow: ShowTileInfo.styling,
             enablePopups: false
@@ -440,7 +440,7 @@ export class InitUiElements {
 
                 clusterCounter.addTile(source)
 
-                const clustering = State.state.layoutToUse.clustering
+                // Do show features indicates if the 'showDataLayer' should be shown
                 const doShowFeatures = source.features.map(
                     f => {
                         const z = State.state.locationControl.data.zoom
@@ -449,6 +449,18 @@ export class InitUiElements {
                             return false;
                         }
 
+                        const bounds = State.state.currentBounds.data
+                        if(bounds === undefined){
+                            // Map is not yet displayed
+                            return false;
+                        }
+
+                        if (!source.bbox.overlapsWith(bounds)) {
+                            // Not within range -> features are hidden
+                            return false
+                        }
+
+
                         if (z < source.layer.layerDef.minzoom) {
                             // Layer is always hidden for this zoom level
                             return false;
@@ -473,20 +485,12 @@ export class InitUiElements {
                             }
 
                             if (clusterCounter.getTile(Tiles.tile_index(tileZ, tileX, tileY))?.totalValue > clustering.minNeededElements) {
+                                // To much elements
                                 return false
                             }
                         }
 
 
-                        const bounds = State.state.currentBounds.data
-                        if(bounds === undefined){
-                            // Map is not yet displayed
-                            return false;
-                        }
-                        if (!source.bbox.overlapsWith(bounds)) {
-                            // Not within range
-                            return false
-                        }
                       
                         return true
                     }, [State.state.currentBounds, source.layer.isDisplayed]

Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts

@@ -21,7 +21,9 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
             throw "Invalid layer: geojsonSource expected"
         }
         
-        const whitelistUrl = source.geojsonSource.replace("{z}_{x}_{y}.geojson", "overview.json")
+        const whitelistUrl = source.geojsonSource
+            .replace("{z}", ""+source.geojsonZoomLevel)
+            .replace("{x}_{y}.geojson", "overview.json")
             .replace("{layer}",layer.layerDef.id)
         
         let whitelist = undefined
@@ -46,7 +48,8 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
                 if(whitelist !== undefined){
                     const isWhiteListed = whitelist.get(zxy[1])?.has(zxy[2])
                     if(!isWhiteListed){
-                        return undefined;
+                        console.log("Not whitelisted:",zxy, isWhiteListed, whitelist)
+                    //    return undefined;
                     }
                 }
                 

Models/ThemeConfig/Json/LayoutConfigJson.ts

@@ -1,6 +1,5 @@
 import {TagRenderingConfigJson} from "./TagRenderingConfigJson";
 import {LayerConfigJson} from "./LayerConfigJson";
-import UnitConfigJson from "./UnitConfigJson";
 
 /**
  * Defines the entire theme.
@@ -250,7 +249,7 @@ export interface LayoutConfigJson {
      * If set to [[lat0, lon0], [lat1, lon1]], the map will not scroll outside of those bounds.
      * Off by default, which will enable panning to the entire world
      */
-    lockLocation?: boolean | [[number, number], [number, number]];
+    lockLocation?: boolean | [[number, number], [number, number]] | number[][];
 
     enableUserBadge?: boolean;
     enableShareScreen?: boolean;

Models/ThemeConfig/LayoutConfig.ts

@@ -29,7 +29,7 @@ export default class LayoutConfig {
     public layers: LayerConfig[];
     public readonly clustering?: {
         maxZoom: number,
-        minNeededElements: number
+        minNeededElements: number,
     };
     public readonly hideFromOverview: boolean;
     public lockLocation: boolean | [[number, number], [number, number]];
@@ -137,12 +137,17 @@ export default class LayoutConfig {
 
         this.clustering = {
             maxZoom: 16,
-            minNeededElements: 25
+            minNeededElements: 25,
         };
-        if (json.clustering) {
+        if(json.clustering === false){
+            this.clustering = {
+                maxZoom: 0,
+                minNeededElements: 100000,
+            };
+        }else         if (json.clustering) {
             this.clustering = {
                 maxZoom: json.clustering.maxZoom ?? 18,
-                minNeededElements: json.clustering.minNeededElements ?? 25
+                minNeededElements: json.clustering.minNeededElements ?? 25,
             }
         }
 
@@ -151,7 +156,7 @@ export default class LayoutConfig {
         if (json.hideInOverview) {
             throw "The json for " + this.id + " contains a 'hideInOverview'. Did you mean hideFromOverview instead?"
         }
-        this.lockLocation = json.lockLocation ?? undefined;
+        this.lockLocation = <[[number, number], [number, number]]> json.lockLocation ?? undefined;
         this.enableUserBadge = json.enableUserBadge ?? true;
         this.enableShareScreen = json.enableShareScreen ?? true;
         this.enableMoreQuests = json.enableMoreQuests ?? true;

UI/ShowDataLayer/TileHierarchyAggregator.ts

@@ -22,7 +22,7 @@ export class TileHierarchyAggregator implements FeatureSource {
     public readonly name;
 
     private readonly featuresStatic = []
-    private readonly featureProperties: { count: string, tileId: string, id: string };
+    private readonly featureProperties: { count: string, kilocount: string, tileId: string, id: string };
 
     private constructor(parent: TileHierarchyAggregator, z: number, x: number, y: number) {
         this._parent = parent;
@@ -36,7 +36,8 @@ export class TileHierarchyAggregator implements FeatureSource {
         const totals = {
             id: ""+this._tileIndex,
             tileId: ""+this._tileIndex,
-            count: ""+0
+            count: `0`,
+            kilocount: "0"
         }
         this.featureProperties = totals
 
@@ -108,6 +109,7 @@ export class TileHierarchyAggregator implements FeatureSource {
             this.features.setData(TileHierarchyAggregator.empty)
         } else {
             this.featureProperties.count = "" + total;
+            this.featureProperties.kilocount = "" +Math.floor(total/1000);
             this.features.data = this.featuresStatic
             this.features.ping()
         }
@@ -153,13 +155,18 @@ export class TileHierarchyAggregator implements FeatureSource {
         }
     }
     
-    getCountsForZoom(locationControl: UIEventSource<{ zoom : number }>, cutoff: number = 0) : FeatureSource{
+    getCountsForZoom(clusteringConfig: {maxZoom: number}, locationControl: UIEventSource<{ zoom : number }>, cutoff: number = 0) : FeatureSource{
         const self = this
-        
+        const empty = []
         return new StaticFeatureSource(
             locationControl.map(loc => {
-                const features = []
                 const targetZoom = loc.zoom
+                
+                if(targetZoom > clusteringConfig.maxZoom){
+                    return empty
+                }
+                
+                const features = []
                 self.visitSubTiles(aggr => {
                     if(aggr.totalValue < cutoff) {
                         return false

Utils.ts

@@ -78,18 +78,6 @@ export class Utils {
         return res;
     }
 
-    static DoEvery(millis: number, f: (() => void)) {
-        if (Utils.runningFromConsole) {
-            return;
-        }
-        window.setTimeout(
-            function () {
-                f();
-                Utils.DoEvery(millis, f);
-            }
-            , millis)
-    }
-
     public static NoNull<T>(array: T[]): T[] {
         const ls: T[] = [];
         for (const t of array) {
@@ -448,5 +436,19 @@ export class Utils {
             window.setTimeout(resolve, timeMillis);
         })
     }
+    
+    public static toHumanTime(seconds): string{
+        seconds = Math.floor(seconds)
+        let minutes = Math.floor(seconds / 60)
+        seconds = seconds % 60
+        let hours = Math.floor(minutes / 60)
+        minutes = minutes % 60
+        let days = Math.floor(hours / 24)
+        hours = hours % 24
+        if(days > 0){
+            return days+"days"+" "+hours+"h"
+        }
+            return hours+":"+Utils.TwoDigits(minutes)+":"+Utils.TwoDigits(seconds)
+    }
 }
 

assets/layers/cluster_style/cluster_style.json

@@ -28,8 +28,8 @@
         "render": "<div class='rounded-full text-xl font-bold' style='width: 2rem; height: 2rem; background: white'>{count}</div>",
         "mappings": [
             {
-                "if": "count>99",
+                "if": "count>1000",
-                "then": "<div class='rounded-full text-xl font-bold flex flex-col' style='width: 2.5rem; height: 2.5rem; background: white'>&gt;99</div>"
+                "then": "<div class='rounded-full text-xl font-bold flex flex-col' style='width: 2.5rem; height: 2.5rem; background: white'>{kilocount}K</div>"
             }
         ]
     }

assets/themes/natuurpunt/natuurpunt.json

@@ -32,8 +32,8 @@
   "enablePdfDownload": true,
   "enableDownload": true,
   "hideFromOverview": true,
-  "clustering": {
     "#": "Disable clustering for this theme",
+  "clustering": {
     "maxZoom": 0
   },
   "layers": [
@@ -112,7 +112,7 @@
           "geoJsonZoomLevel": 12,
           "isOsmCache": true
         },
-        "minzoom": "13",
+        "minzoom": 10,
         "icon": {
           "render": "circle:#FE6F32;./assets/themes/natuurpunt/trail.svg",
           "mappings": [

index.ts

@@ -63,7 +63,6 @@ if (path !== "index.html" && path !== "") {
 defaultLayout = QueryParameters.GetQueryParameter("layout", defaultLayout, "The layout to load into MapComplete").data;
 let layoutToUse: LayoutConfig = AllKnownLayouts.allKnownLayouts.get(defaultLayout.toLowerCase());
 
-
 const userLayoutParam = QueryParameters.GetQueryParameter("userlayout", "false", "If not 'false', a custom (non-official) theme is loaded. This custom layout can be done in multiple ways: \n\n- The hash of the URL contains a base64-encoded .json-file containing the theme definition\n- The hash of the URL contains a lz-compressed .json-file, as generated by the custom theme generator\n- The parameter itself is an URL, in which case that URL will be downloaded. It should point to a .json of a theme");
 
 // Workaround/legacy to keep the old paramters working as I renamed some of them

langs/en.json

@@ -36,9 +36,6 @@
         "splitTitle": "Choose on the map where to split this road",
         "hasBeenSplit": "This way has been split"
     },
-    "multi_apply": {
-        "autoApply": "When changing the attributes {attr_names}, these attributes will automatically be changed on {count} other objects too"
-    },
     "delete": {
         "delete": "Delete",
         "cancel": "Cancel",
@@ -123,7 +120,7 @@
         },
         "morescreen": {
             "intro": "<h3>More thematic maps?</h3>Do you enjoy collecting geodata? <br/>There are more themes available.",
-            "requestATheme": "If you want a custom-built quest, request it in the issue tracker",
+            "requestATheme": "If you want a custom-built theme, request it in the issue tracker",
             "streetcomplete": "Another, similar application is <a href='https://play.google.com/store/apps/details?id=de.westnordost.streetcomplete' class='underline hover:text-blue-800' class='underline hover:text-blue-800' target='_blank'>StreetComplete</a>.",
             "createYourOwnTheme": "Create your own MapComplete theme from scratch"
         },
@@ -253,5 +250,8 @@
         "tos": "If you create a review, you agree to <a href='https://mangrove.reviews/terms' target='_blank'>the TOS and privacy policy of Mangrove.reviews</a>",
         "attribution": "Reviews are powered by <a href='https://mangrove.reviews/' target='_blank'>Mangrove Reviews</a> and are available under <a href='https://mangrove.reviews/terms#8-licensing-of-content' target='_blank'>CC-BY 4.0</a>.",
         "plz_login": "Login to leave a review"
+    },
+    "multi_apply": {
+        "autoApply": "When changing the attributes {attr_names}, these attributes will automatically be changed on {count} other objects too"
     }
 }

scripts/generateCache.ts

@@ -20,6 +20,7 @@ import FeatureSource, {FeatureSourceForLayer} from "../Logic/FeatureSource/Featu
 import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource";
 import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource";
 import Constants from "../Models/Constants";
+import {GeoOperations} from "../Logic/GeoOperations";
 
 
 ScriptUtils.fixUtils()
@@ -73,6 +74,7 @@ async function downloadRaw(targetdir: string, r: TileRange, theme: LayoutConfig,
     let downloaded = 0
     let failed = 0
     let skipped = 0
+    const startTime = new Date().getTime()
     for (let x = r.xstart; x <= r.xend; x++) {
         for (let y = r.ystart; y <= r.yend; y++) {
             downloaded++;
@@ -82,7 +84,11 @@ async function downloadRaw(targetdir: string, r: TileRange, theme: LayoutConfig,
                 skipped++
                 continue;
             }
-            console.log("x:", (x - r.xstart), "/", (r.xend - r.xstart), "; y:", (y - r.ystart), "/", (r.yend - r.ystart), "; total: ", downloaded, "/", r.total, "failed: ", failed, "skipped: ", skipped)
+            const runningSeconds = (new Date().getTime() - startTime) / 1000
+            const resting = failed + (r.total - downloaded)
+            const perTile=   (runningSeconds / (downloaded - skipped))
+            const estimated =Math.floor(resting * perTile)
+            console.log("total: ", downloaded, "/", r.total, "failed: ", failed, "skipped: ", skipped, "running time: ",Utils.toHumanTime(runningSeconds)+"s", "estimated left: ", Utils.toHumanTime(estimated), "("+Math.floor(perTile)+"s/tile)")
 
             const boundsArr = Tiles.tile_bounds(r.zoomlevel, x, y)
             const bounds = {
@@ -91,7 +97,7 @@ async function downloadRaw(targetdir: string, r: TileRange, theme: LayoutConfig,
                 east: Math.max(boundsArr[0][1], boundsArr[1][1]),
                 west: Math.min(boundsArr[0][1], boundsArr[1][1])
             }
-            const overpass = createOverpassObject(theme, relationTracker, Constants.defaultOverpassUrls[(downloaded + failed) % Constants.defaultOverpassUrls.length])
+            const overpass = createOverpassObject(theme, relationTracker, Constants.defaultOverpassUrls[(failed) % Constants.defaultOverpassUrls.length])
             const url = overpass.buildQuery("[bbox:" + bounds.south + "," + bounds.west + "," + bounds.north + "," + bounds.east + "]")
 
             try {
@@ -170,11 +176,11 @@ function loadAllTiles(targetdir: string, r: TileRange, theme: LayoutConfig, extr
 /**
  * Load all the tiles into memory from disk
  */
-function postProcess(allFeatures: FeatureSource, theme: LayoutConfig, relationsTracker: RelationsTracker, targetdir: string) {
+function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relationsTracker: RelationsTracker, targetdir: string, pointsOnlyLayers: string[]) {
-
-
     function handleLayer(source: FeatureSourceForLayer) {
         const layer = source.layer.layerDef;
+        const targetZoomLevel = layer.source.geojsonZoomLevel ?? 0
+        
         const layerId = layer.id
         if (layer.source.isOsmCacheLayer !== true) {
             return;
@@ -200,13 +206,10 @@ function postProcess(allFeatures: FeatureSource, theme: LayoutConfig, relationsT
         // At this point, we have all the features of the entire area.
         // However, we want to export them per tile of a fixed size, so we use a dynamicTileSOurce to split it up
         TiledFeatureSource.createHierarchy(source, {
-            minZoomLevel: 14,
+            minZoomLevel: targetZoomLevel,
-            maxZoomLevel: 14,
+            maxZoomLevel: targetZoomLevel,
             maxFeatureCount: undefined,
             registerTile: tile => {
-                if (tile.z < 12) {
-                    return;
-                }
                 if (tile.features.data.length === 0) {
                     return
                 }
@@ -229,7 +232,7 @@ function postProcess(allFeatures: FeatureSource, theme: LayoutConfig, relationsT
 
         // All the tiles are written at this point
         // Only thing left to do is to create the index
-        const path = targetdir + "_" + layerId + "_overview.json"
+        const path = targetdir + "_" + layerId + "_" + targetZoomLevel + "_overview.json"
         const perX = {}
         createdTiles.map(i => Tiles.tile_from_index(i)).forEach(([z, x, y]) => {
             const key = "" + x
@@ -240,7 +243,18 @@ function postProcess(allFeatures: FeatureSource, theme: LayoutConfig, relationsT
         })
         writeFileSync(path, JSON.stringify(perX))
 
-
+        // And, if needed, to create a points-only layer
+        if(pointsOnlyLayers.indexOf(layer.id) >= 0){
+            const features = source.features.data.map(f => f.feature)
+            const points = features.map(feature => GeoOperations.centerpoint(feature))
+            console.log("Writing points overview for ", layerId)
+            const targetPath = targetdir+"_"+layerId+"_points.geojson"
+            // This is the geojson file containing all features for this tile
+            writeFileSync(targetPath, JSON.stringify({
+                type: "FeatureCollection",
+                features: points
+            }, null, " "))
+        }
     }
 
     new PerLayerFeatureSourceSplitter(
@@ -255,10 +269,11 @@ function postProcess(allFeatures: FeatureSource, theme: LayoutConfig, relationsT
 }
 
 
+
 async function main(args: string[]) {
 
     if (args.length == 0) {
-        console.error("Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1 [--generate-point-overview layer-name]")
+        console.error("Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1 [--generate-point-overview layer-name,layer-name,...]")
         return;
     }
     const themeName = args[0]
@@ -269,6 +284,12 @@ async function main(args: string[]) {
     const lat1 = Number(args[5])
     const lon1 = Number(args[6])
     
+    let generatePointLayersFor = []
+    if(args[7] == "--generate-point-overview"){
+        generatePointLayersFor = args[8].split(",")
+    }
+    
+
     const tileRange = Tiles.TileRangeBetween(zoomlevel, lat0, lon0, lat1, lon1)
 
     const theme = AllKnownLayouts.allKnownLayouts.get(themeName)
@@ -293,7 +314,7 @@ async function main(args: string[]) {
 
     const extraFeatures = await downloadExtraData(theme);
     const allFeaturesSource = loadAllTiles(targetdir, tileRange, theme, extraFeatures)
-    postProcess(allFeaturesSource, theme, relationTracker, targetdir)
+    sliceToTiles(allFeaturesSource, theme, relationTracker, targetdir, generatePointLayersFor)
 
 }