Merge pull request 'Add local clustering' (#2468) from feature/local-clustering into develop

Reviewed-on: MapComplete/MapComplete#2468

src/Logic/FeatureSource/FeatureSource.ts

@@ -1,25 +1,27 @@
 import { Store, UIEventSource } from "../UIEventSource"
 import FilteredLayer from "../../Models/FilteredLayer"
-import { Feature } from "geojson"
+import { Feature, Geometry } from "geojson"
+import { OsmTags } from "../../Models/OsmFeature"
 
-export interface FeatureSource<T extends Feature = Feature> {
+export interface FeatureSource<T extends Feature = Feature<Geometry, OsmTags>> {
     features: Store<T[]>
 }
 
-export interface UpdatableFeatureSource<T extends Feature = Feature> extends FeatureSource<T> {
+export interface UpdatableFeatureSource<T extends Feature = Feature<Geometry, OsmTags>> extends FeatureSource<T> {
     /**
      * Forces an update and downloads the data, even if the feature source is supposed to be active
      */
-    updateAsync()
+    updateAsync(): void
 }
-export interface WritableFeatureSource<T extends Feature = Feature> extends FeatureSource<T> {
+
+export interface WritableFeatureSource<T extends Feature = Feature<Geometry, OsmTags>> extends FeatureSource<T> {
     features: UIEventSource<T[]>
 }
 
 /**
  * A feature source which only contains features for the defined layer
  */
-export interface FeatureSourceForLayer<T extends Feature = Feature> extends FeatureSource<T> {
+export interface FeatureSourceForLayer<T extends Feature = Feature<Geometry, OsmTags>> extends FeatureSource<T> {
     readonly layer: FilteredLayer
 }
 

src/Logic/FeatureSource/PerLayerFeatureSourceSplitter.ts

@@ -5,21 +5,23 @@ import { Feature } from "geojson"
 import { UIEventSource } from "../UIEventSource"
 
 /**
+ * Constructs multiple featureStores based on the given layers, where every constructed feature source will contain features only matching the given layer
+ *
  * In some rare cases, some elements are shown on multiple layers (when 'passthrough' is enabled)
  * If this is the case, multiple objects with a different _matching_layer_id are generated.
  * In any case, this featureSource marks the objects with _matching_layer_id
  */
-export default class PerLayerFeatureSourceSplitter<T extends FeatureSource = FeatureSource> {
-    public readonly perLayer: ReadonlyMap<string, T>
+export default class PerLayerFeatureSourceSplitter<T extends Feature, SRC extends FeatureSource<T>> {
+    public readonly perLayer: ReadonlyMap<string, SRC>
     constructor(
         layers: FilteredLayer[],
-        upstream: FeatureSource,
+        upstream: FeatureSource<T>,
         options?: {
-            constructStore?: (features: UIEventSource<Feature[]>, layer: FilteredLayer) => T
-            handleLeftovers?: (featuresWithoutLayer: Feature[]) => void
+            constructStore?: (features: UIEventSource<T[]>, layer: FilteredLayer) => SRC
+            handleLeftovers?: (featuresWithoutLayer: T[]) => void
         }
     ) {
-        const knownLayers = new Map<string, T>()
+        const knownLayers = new Map<string, SRC>()
         /**
          * Keeps track of the ids that are included per layer.
          * Used to know if the downstream feature source needs to be pinged
@@ -30,12 +32,12 @@ export default class PerLayerFeatureSourceSplitter<T extends FeatureSource = Fea
         const constructStore =
             options?.constructStore ?? ((store, layer) => new SimpleFeatureSource(layer, store))
         for (const layer of layers) {
-            const src = new UIEventSource<Feature[]>([])
+            const src = new UIEventSource<T[]>([])
             layerSources.set(layer.layerDef.id, src)
-            knownLayers.set(layer.layerDef.id, <T>constructStore(src, layer))
+            knownLayers.set(layer.layerDef.id, <SRC>constructStore(src, layer))
         }
 
-        upstream.features.addCallbackAndRunD((features) => {
+        upstream.features.addCallbackAndRunD((features: T[]) => {
             if (layers === undefined) {
                 return
             }
@@ -49,7 +51,7 @@ export default class PerLayerFeatureSourceSplitter<T extends FeatureSource = Fea
              */
             const hasChanged: boolean[] = layers.map(() => false)
             const newIndices: Set<string>[] = layers.map(() => new Set<string>())
-            const noLayerFound: Feature[] = []
+            const noLayerFound: T[] = []
 
             for (const layer of layers) {
                 featuresPerLayer.set(layer.layerDef.id, [])
@@ -113,7 +115,7 @@ export default class PerLayerFeatureSourceSplitter<T extends FeatureSource = Fea
         })
     }
 
-    public forEach(f: (featureSource: T) => void) {
+    public forEach(f: ((src: SRC) => void)) {
         for (const fs of this.perLayer.values()) {
             f(fs)
         }

src/Logic/FeatureSource/Sources/FilteringFeatureSource.ts

@@ -1,12 +1,13 @@
 import { Store, UIEventSource } from "../../UIEventSource"
 import FilteredLayer from "../../../Models/FilteredLayer"
 import { FeatureSource } from "../FeatureSource"
-import { Feature } from "geojson"
+import { Feature, Geometry } from "geojson"
 import { GlobalFilter } from "../../../Models/GlobalFilter"
+import { OsmTags } from "../../../Models/OsmFeature"
 
-export default class FilteringFeatureSource implements FeatureSource {
-    public features: UIEventSource<Feature[]> = new UIEventSource([])
-    private readonly upstream: FeatureSource
+export default class FilteringFeatureSource<T extends Feature = Feature<Geometry, OsmTags>> implements FeatureSource<T> {
+    public readonly features: UIEventSource<T[]> = new UIEventSource([])
+    private readonly upstream: FeatureSource<T>
     private readonly _fetchStore?: (id: string) => Store<Record<string, string>>
     private readonly _globalFilters?: Store<GlobalFilter[]>
     private readonly _alreadyRegistered = new Set<Store<any>>()
@@ -18,7 +19,7 @@ export default class FilteringFeatureSource implements FeatureSource {
 
     constructor(
         layer: FilteredLayer,
-        upstream: FeatureSource,
+        upstream: FeatureSource<T>,
         fetchStore?: (id: string) => Store<Record<string, string>>,
         globalFilters?: Store<GlobalFilter[]>,
         metataggingUpdated?: Store<any>,
@@ -40,7 +41,7 @@ export default class FilteringFeatureSource implements FeatureSource {
         })
 
         layer.appliedFilters.forEach((value) =>
-            value.addCallback((_) => {
+            value.addCallback(() => {
                 this.update()
             })
         )
@@ -68,7 +69,7 @@ export default class FilteringFeatureSource implements FeatureSource {
 
     private update() {
         const layer = this._layer
-        const features: Feature[] = this.upstream.features.data ?? []
+        const features: T[] = this.upstream.features.data ?? []
         const includedFeatureIds = new Set<string>()
         const globalFilters = this._globalFilters?.data?.map((f) => f)
         const zoomlevel = this._zoomlevel?.data
@@ -121,10 +122,9 @@ export default class FilteringFeatureSource implements FeatureSource {
         }
         this._alreadyRegistered.add(src)
 
-        const self = this
         // Add a callback as a changed tag might change the filter
-        src.addCallbackAndRunD((_) => {
-            self._is_dirty.setData(true)
+        src.addCallbackAndRunD(() => {
+            this._is_dirty.setData(true)
         })
     }
 }

src/Logic/FeatureSource/Sources/IfVisibleFeatureSource.ts

@@ -0,0 +1,38 @@
+import { FeatureSource } from "../FeatureSource"
+import { Feature } from "geojson"
+import { Store, UIEventSource } from "../../UIEventSource"
+
+export class IfVisibleFeatureSource<T extends Feature> implements FeatureSource<T> {
+
+    private readonly _features: UIEventSource<T[]> = new UIEventSource<T[]>([])
+    public readonly features: Store<T[]> = this._features
+
+    constructor(upstream: FeatureSource<T>, visible: Store<boolean>) {
+
+        let dirty = false
+        upstream.features.addCallbackAndRun(features => {
+            if (!visible.data) {
+                dirty = true
+                this._features.set([])
+                return
+            }
+            this._features.set(features)
+            dirty = false
+        })
+
+        visible.addCallbackAndRun(isVisible => {
+            if (isVisible && dirty) {
+                this._features.set(upstream.features.data)
+                dirty = false
+            }
+            if (!visible) {
+                this._features.set([])
+            }
+
+        })
+
+
+    }
+
+
+}

src/Logic/FeatureSource/Sources/MvtSource.ts

@@ -1,24 +1,19 @@
-import { Feature as GeojsonFeature, Geometry } from "geojson"
+import { Feature, Feature as GeojsonFeature, Geometry } from "geojson"
 
 import { Store, UIEventSource } from "../../UIEventSource"
 import { FeatureSourceForTile, UpdatableFeatureSource } from "../FeatureSource"
 import { MvtToGeojson } from "mvt-to-geojson"
+import { OsmTags } from "../../../Models/OsmFeature"
 
 export default class MvtSource implements FeatureSourceForTile, UpdatableFeatureSource {
-    public readonly features: Store<GeojsonFeature<Geometry, { [name: string]: any }>[]>
+    public readonly features: Store<GeojsonFeature<Geometry, OsmTags>[]>
     public readonly x: number
     public readonly y: number
     public readonly z: number
     private readonly _url: string
-    private readonly _layerName: string
     private readonly _features: UIEventSource<
-        GeojsonFeature<
-            Geometry,
-            {
-                [name: string]: any
-            }
-        >[]
-    > = new UIEventSource<GeojsonFeature<Geometry, { [p: string]: any }>[]>([])
+        GeojsonFeature<Geometry, OsmTags>[]
+    > = new UIEventSource<GeojsonFeature<Geometry, OsmTags>[]>([])
     private currentlyRunning: Promise<any>
 
     constructor(
@@ -26,11 +21,9 @@ export default class MvtSource implements FeatureSourceForTile, UpdatableFeature
         x: number,
         y: number,
         z: number,
-        layerName?: string,
         isActive?: Store<boolean>
     ) {
         this._url = url
-        this._layerName = layerName
         this.x = x
         this.y = y
         this.z = z
@@ -61,7 +54,7 @@ export default class MvtSource implements FeatureSourceForTile, UpdatableFeature
                 return
             }
             const buffer = await result.arrayBuffer()
-            const features = MvtToGeojson.fromBuffer(buffer, this.x, this.y, this.z)
+            const features = <Feature<Geometry, OsmTags>[]>MvtToGeojson.fromBuffer(buffer, this.x, this.y, this.z)
             for (const feature of features) {
                 const properties = feature.properties
                 if (!properties["osm_type"]) {

src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts

@@ -8,7 +8,8 @@ import { Utils } from "../../../Utils"
 import { TagsFilter } from "../../Tags/TagsFilter"
 import { BBox } from "../../BBox"
 import { OsmTags } from "../../../Models/OsmFeature"
-;("use strict")
+
+("use strict")
 
 /**
  * A wrapper around the 'Overpass'-object.
@@ -138,7 +139,6 @@ export default class OverpassFeatureSource implements UpdatableFeatureSource {
                     return undefined
                 }
                 this.runningQuery.setData(true)
-                console.trace("Overpass feature source: querying geojson")
                 data = (await overpass.queryGeoJson(bounds))[0]
             } catch (e) {
                 this.retries.data++

src/Logic/FeatureSource/Sources/SimpleFeatureSource.ts

@@ -1,14 +1,15 @@
 import { UIEventSource } from "../../UIEventSource"
 import FilteredLayer from "../../../Models/FilteredLayer"
 import { FeatureSourceForLayer } from "../FeatureSource"
-import { Feature } from "geojson"
+import { Feature, Geometry } from "geojson"
+import { OsmTags } from "../../../Models/OsmFeature"
 
-export default class SimpleFeatureSource implements FeatureSourceForLayer {
-    public readonly features: UIEventSource<Feature[]>
+export default class SimpleFeatureSource<T extends Feature = Feature<Geometry, OsmTags>> implements FeatureSourceForLayer<T> {
+    public readonly features: UIEventSource<T[]>
     public readonly layer: FilteredLayer
 
-    constructor(layer: FilteredLayer, featureSource?: UIEventSource<Feature[]>) {
+    constructor(layer: FilteredLayer, featureSource?: UIEventSource<T[]>) {
         this.layer = layer
-        this.features = featureSource ?? new UIEventSource<Feature[]>([])
+        this.features = featureSource ?? new UIEventSource<T[]>([])
     }
 }

src/Logic/FeatureSource/TiledFeatureSource/ClusteringFeatureSource.ts

@@ -0,0 +1,170 @@
+import { FeatureSource } from "../FeatureSource"
+import { Feature, Point } from "geojson"
+import { ImmutableStore, Store, UIEventSource } from "../../UIEventSource"
+import { GeoOperations } from "../../GeoOperations"
+import { Tiles } from "../../../Models/TileRange"
+
+export interface ClusteringOptions {
+    /**
+     * If the zoomlevel is (strictly) above the specified value, don't cluster no matter how many features the tile contains.
+     */
+    dontClusterAboveZoom?: number
+    /**
+     * If the number of features in a _tile_ is equal or more then this number,
+     * drop those features and emit a summary tile instead
+     */
+    cutoff?: 20 | number
+
+    showSummaryAt?: "tilecenter" | "average"
+}
+
+interface SummaryProperties {
+    id: string,
+    total: number,
+    tile_id: number
+}
+
+export class ClusteringFeatureSource<T extends Feature<Point> = Feature<Point>> implements FeatureSource<T> {
+
+    private readonly id: string
+    private readonly showSummaryAt: "tilecenter" | "average"
+    features: Store<T[]>
+
+    /**
+     *The clustering feature source works _only_ on points and is a preprocessing step for the ShowDataLayer.
+     * If a tile contains many points, a 'summary' point is emitted instead in 'summaryPoints'.
+     * The points from the summary will _not_ be emitted in 'this.features' in that case.
+     *
+     * We ignore the polygons, as polygons get smaller when zoomed out and thus don't clutter the map too much
+     */
+    constructor(upstream: FeatureSource<T>,
+                currentZoomlevel: Store<number>,
+                id: string,
+                options?: ClusteringOptions) {
+        this.id = id
+        this.showSummaryAt = options?.showSummaryAt ?? "average"
+        const clusterCutoff = options?.dontClusterAboveZoom ?? 17
+        const doCluster = options?.dontClusterAboveZoom === undefined ? new ImmutableStore(true) : currentZoomlevel.map(zoom => zoom <= clusterCutoff)
+        const cutoff = options?.cutoff ?? 20
+        const summaryPoints = new UIEventSource<Feature<Point, SummaryProperties>[]>([])
+        currentZoomlevel = currentZoomlevel.stabilized(500).map(z => Math.floor(z))
+        this.features = (upstream.features.map(features => {
+            if (!doCluster.data) {
+                summaryPoints.set([])
+                return features
+            }
+
+            const z = currentZoomlevel.data
+            const perTile = GeoOperations.spreadIntoBboxes(features, z)
+            const resultingFeatures = []
+            const summary: Feature<Point, SummaryProperties>[] = []
+            for (const tileIndex of perTile.keys()) {
+                const tileFeatures: Feature<Point>[] = perTile.get(tileIndex)
+                if (tileFeatures.length > cutoff) {
+                    summary.push(this.createSummaryFeature(tileFeatures, tileIndex))
+                } else {
+                    resultingFeatures.push(...tileFeatures)
+                }
+            }
+            summaryPoints.set(summary)
+            return resultingFeatures
+
+        }, [doCluster, currentZoomlevel]))
+
+        ClusterGrouping.singleton.registerSource(summaryPoints)
+
+    }
+
+
+    private createSummaryFeature(features: Feature<Point>[], tileId: number): Feature<Point, SummaryProperties> {
+
+        let lon: number
+        let lat: number
+        const [z, x, y] = Tiles.tile_from_index(tileId)
+        if (this.showSummaryAt === "tilecenter") {
+            [lon, lat] = Tiles.centerPointOf(z, x, y)
+        } else {
+            let lonSum = 0
+            let latSum = 0
+            for (const feature of features) {
+                const [lon, lat] = feature.geometry.coordinates
+                lonSum += lon
+                latSum += lat
+            }
+            lon = lonSum / features.length
+            lat = latSum / features.length
+        }
+        return {
+            type: "Feature",
+            geometry: {
+                type: "Point",
+                coordinates: [lon, lat]
+            },
+            properties: {
+                id: "summary_" + this.id + "_" + tileId,
+                tile_id: tileId,
+                total: features.length
+            }
+        }
+    }
+}
+
+/**
+ * Groups multiple summaries together
+ */
+export class ClusterGrouping implements FeatureSource<Feature<Point, { total_metric: string }>> {
+    private readonly _features: UIEventSource<Feature<Point, { total_metric: string }>[]> = new UIEventSource([])
+    public readonly features: Store<Feature<Point, { total_metric: string }>[]> = this._features
+
+    public static readonly singleton = new ClusterGrouping()
+
+    public readonly isDirty = new UIEventSource(false)
+
+    private constructor() {
+        this.isDirty.stabilized(200).addCallback(dirty => {
+            if (dirty) {
+                this.update()
+            }
+        })
+    }
+
+    private allSource: Store<Feature<Point, { total: number, tile_id: number }>[]>[] = []
+
+    private update() {
+        const countPerTile = new Map<number, number>()
+        for (const source of this.allSource) {
+            for (const f of source.data) {
+                const id = f.properties.tile_id
+                const count = f.properties.total + (countPerTile.get(id) ?? 0)
+                countPerTile.set(id, count)
+            }
+        }
+        const features: Feature<Point, { total_metric: string, id: string }>[] = []
+        const now = new Date().getTime() + ""
+        for (const tileId of countPerTile.keys()) {
+            const coordinates = Tiles.centerPointOf(tileId)
+            features.push({
+                type: "Feature",
+                properties: {
+                    total_metric: "" + countPerTile.get(tileId),
+                    id: "clustered_all_" + tileId + "_" + now // We add the date to force a fresh ID every time, this makes sure values are updated
+                },
+                geometry: {
+                    type: "Point",
+                    coordinates
+                }
+            })
+        }
+        this._features.set(features)
+        this.isDirty.set(false)
+    }
+
+    public registerSource(features: Store<Feature<Point, SummaryProperties>[]>) {
+        this.allSource.push(features)
+        features.addCallbackAndRun(() => {
+            //this.isDirty.set(true)
+            this.update()
+        })
+    }
+
+}

src/Logic/GeoOperations.ts

@@ -10,11 +10,12 @@ import {
     MultiPolygon,
     Point,
     Polygon,
-    Position,
+    Position
 } from "geojson"
 import { Tiles } from "../Models/TileRange"
 import { Utils } from "../Utils"
-;("use strict")
+
+("use strict")
 
 export class GeoOperations {
     private static readonly _earthRadius: number = 6378137
@@ -536,10 +537,23 @@ export class GeoOperations {
      * @param features
      * @param zoomlevel
      */
-    public static spreadIntoBboxes(features: Feature[], zoomlevel: number): Map<number, Feature[]> {
-        const perBbox = new Map<number, Feature[]>()
-
+    public static spreadIntoBboxes<T extends Feature = Feature>(features: T[], zoomlevel: number): Map<number, T[]> {
+        const perBbox = new Map<number, T[]>()
+        const z = zoomlevel
         for (const feature of features) {
+            if (feature.geometry.type === "Point") {
+                const [lon, lat] = feature.geometry.coordinates
+                const tileXYZ = Tiles.embedded_tile(lat, lon, z)
+                const tileNumber = Tiles.tile_index(z, tileXYZ.x, tileXYZ.y)
+                let newFeatureList = perBbox.get(tileNumber)
+                if (newFeatureList === undefined) {
+                    newFeatureList = []
+                    perBbox.set(tileNumber, newFeatureList)
+                }
+                newFeatureList.push(feature)
+
+                continue
+            }
             const bbox = BBox.get(feature)
             const tilerange = bbox.expandToTileBounds(zoomlevel).containingTileRange(zoomlevel)
             Tiles.MapRange(tilerange, (x, y) => {