From 215aebce1983ebd2f027327547678f9ba5fd68fc Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Mon, 27 Sep 2021 14:45:48 +0200 Subject: [PATCH] More work on clustering, more or less finished --- InitUiElements.ts | 74 ++++-- Logic/FeatureSource/FeaturePipeline.ts | 32 ++- Models/ThemeConfig/Json/LayoutConfigJson.ts | 4 +- Models/ThemeConfig/LayoutConfig.ts | 9 +- UI/ShowDataLayer/PerTileCountAggregator.ts | 258 ++++++++++++-------- UI/ShowDataLayer/ShowDataLayer.ts | 77 +++--- assets/layers/tree_node/tree_node.json | 2 +- assets/themes/trees/trees.json | 5 +- 8 files changed, 276 insertions(+), 185 deletions(-) diff --git a/InitUiElements.ts b/InitUiElements.ts index 0bc6506d1e..c3e57766a1 100644 --- a/InitUiElements.ts +++ b/InitUiElements.ts @@ -39,7 +39,8 @@ import Combine from "./UI/Base/Combine"; import {SubtleButton} from "./UI/Base/SubtleButton"; import ShowTileInfo from "./UI/ShowDataLayer/ShowTileInfo"; import {Tiles} from "./Models/TileRange"; -import PerTileCountAggregator from "./UI/ShowDataLayer/PerTileCountAggregator"; +import {TileHierarchyAggregator} from "./UI/ShowDataLayer/PerTileCountAggregator"; +import {BBox} from "./Logic/GeoOperations"; export class InitUiElements { static InitAll( @@ -430,48 +431,67 @@ export class InitUiElements { return flayers; }); - const clusterCounter = new PerTileCountAggregator(State.state.locationControl.map(l => { - const z = l.zoom + 1 - if(z < 7){ - return 7 - } - return z - })) - const clusterShow = Math.min(...State.state.layoutToUse.data.layers.map(layer => layer.minzoomVisible ?? layer.minzoom)) + const layers = State.state.layoutToUse.data.layers + const clusterShow = Math.min(...layers.map(layer => layer.minzoom)) + + + const clusterCounter = TileHierarchyAggregator.createHierarchy() new ShowDataLayer({ - features: clusterCounter, + features: clusterCounter.getCountsForZoom(State.state.locationControl, State.state.layoutToUse.data.clustering.minNeededElements), leafletMap: State.state.leafletMap, layerToShow: ShowTileInfo.styling, - doShowLayer: State.state.locationControl.map(l => l.zoom < clusterShow) + doShowLayer: layers.length === 1 ? undefined : State.state.locationControl.map(l => l.zoom < clusterShow) }) + State.state.featurePipeline = new FeaturePipeline( source => { + + clusterCounter.addTile(source) + const clustering = State.state.layoutToUse.data.clustering const doShowFeatures = source.features.map( f => { const z = State.state.locationControl.data.zoom - if(z >= clustering.maxZoom){ - return true - } - if(z < source.layer.layerDef.minzoom){ + + if (z < source.layer.layerDef.minzoom) { + // Layer is always hidden for this zoom level return false; } - if(f.length > clustering.minNeededElements){ - console.log("Activating clustering for tile ", Tiles.tile_from_index(source.tileIndex)," as it has ", f.length, "features (clustering starts at)", clustering.minNeededElements) + + if (z >= clustering.maxZoom) { + return true + } + + if (f.length > clustering.minNeededElements) { + // This tile alone has too much features return false } - + + let [tileZ, tileX, tileY] = Tiles.tile_from_index(source.tileIndex); + if (tileZ >= z) { + + while (tileZ > z) { + tileZ-- + tileX = Math.floor(tileX / 2) + tileY = Math.floor(tileY / 2) + } + + if (clusterCounter.getTile(Tiles.tile_index(tileZ, tileX, tileY))?.totalValue > clustering.minNeededElements) { + return false + } + } + + + const bounds = State.state.currentBounds.data + const tilebbox = BBox.fromTileIndex(source.tileIndex) + if (!tilebbox.overlapsWith(bounds)) { + return false + } + return true - }, [State.state.locationControl] + }, [State.state.locationControl, State.state.currentBounds] ) - clusterCounter.addTile(source, doShowFeatures.map(b => !b)) - - /* - new ShowTileInfo({source: source, - leafletMap: State.state.leafletMap, - layer: source.layer.layerDef, - doShowLayer: doShowFeatures.map(b => !b) - })*/ + new ShowDataLayer( { features: source, diff --git a/Logic/FeatureSource/FeaturePipeline.ts b/Logic/FeatureSource/FeaturePipeline.ts index 1e033dad77..a24d6f516c 100644 --- a/Logic/FeatureSource/FeaturePipeline.ts +++ b/Logic/FeatureSource/FeaturePipeline.ts @@ -134,6 +134,8 @@ export default class FeaturePipeline implements FeatureSourceState { layer: source.layer, minZoomLevel: 14, dontEnforceMinZoom: true, + maxFeatureCount: state.layoutToUse.data.clustering.minNeededElements, + maxZoomLevel: state.layoutToUse.data.clustering.maxZoom, registerTile: (tile) => { // We save the tile data for the given layer to local storage new SaveTileToLocalStorageActor(tile, tile.tileIndex) @@ -171,20 +173,26 @@ export default class FeaturePipeline implements FeatureSourceState { private applyMetaTags(src: FeatureSourceForLayer){ const self = this - console.log("Applying metatagging onto ", src.name) - MetaTagging.addMetatags( - src.features.data, - { - memberships: this.relationTracker, - getFeaturesWithin: (layerId, bbox: BBox) => self.GetFeaturesWithin(layerId, bbox) + console.debug("Applying metatagging onto ", src.name) + window.setTimeout( + () => { + MetaTagging.addMetatags( + src.features.data, + { + memberships: this.relationTracker, + getFeaturesWithin: (layerId, bbox: BBox) => self.GetFeaturesWithin(layerId, bbox) + }, + src.layer.layerDef, + { + includeDates: true, + // We assume that the non-dated metatags are already set by the cache generator + includeNonDates: !src.layer.layerDef.source.isOsmCacheLayer + } + ) }, - src.layer.layerDef, - { - includeDates: true, - // We assume that the non-dated metatags are already set by the cache generator - includeNonDates: !src.layer.layerDef.source.isOsmCacheLayer - } + 15 ) + } private updateAllMetaTagging() { diff --git a/Models/ThemeConfig/Json/LayoutConfigJson.ts b/Models/ThemeConfig/Json/LayoutConfigJson.ts index 856ab2736f..887649fc4d 100644 --- a/Models/ThemeConfig/Json/LayoutConfigJson.ts +++ b/Models/ThemeConfig/Json/LayoutConfigJson.ts @@ -228,8 +228,8 @@ export interface LayoutConfigJson { */ maxZoom?: number, /** - * The number of elements that should be showed (in total) before clustering starts to happen. - * If clustering is defined, defaults to 0 + * The number of elements per tile needed to start clustering + * If clustering is defined, defaults to 25 */ minNeededElements?: number }, diff --git a/Models/ThemeConfig/LayoutConfig.ts b/Models/ThemeConfig/LayoutConfig.ts index b632d8a29b..9e63250390 100644 --- a/Models/ThemeConfig/LayoutConfig.ts +++ b/Models/ThemeConfig/LayoutConfig.ts @@ -129,17 +129,12 @@ export default class LayoutConfig { this.clustering = { maxZoom: 16, - minNeededElements: 500 + minNeededElements: 25 }; if (json.clustering) { this.clustering = { maxZoom: json.clustering.maxZoom ?? 18, - minNeededElements: json.clustering.minNeededElements ?? 1 - } - for (const layer of this.layers) { - if (layer.wayHandling !== LayerConfig.WAYHANDLING_CENTER_ONLY) { - console.debug("WARNING: In order to allow clustering, every layer must be set to CENTER_ONLY. Layer", layer.id, "does not respect this for layout", this.id); - } + minNeededElements: json.clustering.minNeededElements ?? 25 } } diff --git a/UI/ShowDataLayer/PerTileCountAggregator.ts b/UI/ShowDataLayer/PerTileCountAggregator.ts index 82e247f742..e4085f2cd6 100644 --- a/UI/ShowDataLayer/PerTileCountAggregator.ts +++ b/UI/ShowDataLayer/PerTileCountAggregator.ts @@ -3,120 +3,178 @@ import {BBox} from "../../Logic/GeoOperations"; import LayerConfig from "../../Models/ThemeConfig/LayerConfig"; import {UIEventSource} from "../../Logic/UIEventSource"; import {Tiles} from "../../Models/TileRange"; +import StaticFeatureSource from "../../Logic/FeatureSource/Sources/StaticFeatureSource"; +export class TileHierarchyAggregator implements FeatureSource { + private _parent: TileHierarchyAggregator; + private _root: TileHierarchyAggregator; + private _z: number; + private _x: number; + private _y: number; + private _tileIndex: number + private _counter: SingleTileCounter -/** - * A feature source containing meta features. - * It will contain exactly one point for every tile of the specified (dynamic) zoom level - */ -export default class PerTileCountAggregator implements FeatureSource { - public readonly features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]); - public readonly name: string = "PerTileCountAggregator" + private _subtiles: [TileHierarchyAggregator, TileHierarchyAggregator, TileHierarchyAggregator, TileHierarchyAggregator] = [undefined, undefined, undefined, undefined] + public totalValue: number = 0 - private readonly perTile: Map = new Map() - private readonly _requestedZoomLevel: UIEventSource; + private static readonly empty = [] + public readonly features = new UIEventSource<{ feature: any, freshness: Date }[]>(TileHierarchyAggregator.empty) + public readonly name; - constructor(requestedZoomLevel: UIEventSource) { - this._requestedZoomLevel = requestedZoomLevel; - const self = this; - this._requestedZoomLevel.addCallbackAndRun(_ => self.update()) + private readonly featuresStatic = [] + private readonly featureProperties: { count: number, tileId: number }; + + private constructor(parent: TileHierarchyAggregator, z: number, x: number, y: number) { + this._parent = parent; + this._root = parent?._root ?? this + this._z = z; + this._x = x; + this._y = y; + this._tileIndex = Tiles.tile_index(z, x, y) + this.name = "Count(" + this._tileIndex + ")" + + const totals = { + tileId: this._tileIndex, + count: 0 + } + this.featureProperties = totals + + const now = new Date() + const feature = { + "type": "Feature", + "properties": totals, + "geometry": { + "type": "Point", + "coordinates": Tiles.centerPointOf(z, x, y) + } + } + this.featuresStatic.push({feature: feature, freshness: now}) + + const bbox = BBox.fromTile(z, x, y) + const box = { + "type": "Feature", + "properties": totals, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [bbox.minLon, bbox.minLat], + [bbox.minLon, bbox.maxLat], + [bbox.maxLon, bbox.maxLat], + [bbox.maxLon, bbox.minLat], + [bbox.minLon, bbox.minLat] + ] + ] + } + } + this.featuresStatic.push({feature: box, freshness: now}) + } + + public getTile(tileIndex): TileHierarchyAggregator { + if (tileIndex === this._tileIndex) { + return this; + } + let [tileZ, tileX, tileY] = Tiles.tile_from_index(tileIndex) + while (tileZ - 1 > this._z) { + tileX = Math.floor(tileX / 2) + tileY = Math.floor(tileY / 2) + tileZ-- + } + const xDiff = tileX - (2 * this._x) + const yDiff = tileY - (2 * this._y) + const subtileIndex = yDiff * 2 + xDiff; + return this._subtiles[subtileIndex]?.getTile(tileIndex) } private update() { - const now = new Date() - const allCountsAsFeatures : {feature: any, freshness: Date}[] = [] - const aggregate = this.calculatePerTileCount() - aggregate.forEach((totalsPerLayer, tileIndex) => { - const totals = {} - let totalCount = 0 - totalsPerLayer.forEach((total, layerId) => { - totals[layerId] = total - totalCount += total - }) - totals["tileId"] = tileIndex - totals["count"] = totalCount - const feature = { - "type": "Feature", - "properties": totals, - "geometry": { - "type": "Point", - "coordinates": Tiles.centerPointOf(...Tiles.tile_from_index(tileIndex)) - } - } - allCountsAsFeatures.push({feature: feature, freshness: now}) - - const bbox= BBox.fromTileIndex(tileIndex) - const box = { - "type": "Feature", - "properties":totals, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [bbox.minLon, bbox.minLat], - [bbox.minLon, bbox.maxLat], - [bbox.maxLon, bbox.maxLat], - [bbox.maxLon, bbox.minLat], - [bbox.minLon, bbox.minLat] - ] - ] - } - } - allCountsAsFeatures.push({feature:box, freshness: now}) + const newMap = new Map() + let total = 0 + this?._counter?.countsPerLayer?.data?.forEach((count, layerId) => { + newMap.set(layerId, count) + total += count }) - this.features.setData(allCountsAsFeatures) - } - /** - * Calculates an aggregate count per tile and per subtile - * @private - */ - private calculatePerTileCount() { - const perTileCount = new Map>() - const targetZoom = this._requestedZoomLevel.data; - // We only search for tiles of the same zoomlevel or a higher zoomlevel, which is embedded - for (const singleTileCounter of Array.from(this.perTile.values())) { - - let tileZ = singleTileCounter.z - let tileX = singleTileCounter.x - let tileY = singleTileCounter.y - if (tileZ < targetZoom) { + for (const tile of this._subtiles) { + if (tile === undefined) { continue; } + total += tile.totalValue + } + this.totalValue = total + this._parent?.update() + + if (total === 0) { + this.features.setData(TileHierarchyAggregator.empty) + } else { + this.featureProperties.count = total; + this.features.data = this.featuresStatic + this.features.ping() + } + } - while (tileZ > targetZoom) { + public addTile(source: FeatureSourceForLayer & Tiled) { + const self = this; + if (source.tileIndex === this._tileIndex) { + if (this._counter === undefined) { + this._counter = new SingleTileCounter(this._tileIndex) + this._counter.countsPerLayer.addCallbackAndRun(_ => self.update()) + } + this._counter.addTileCount(source) + } else { + + // We have to give it to one of the subtiles + let [tileZ, tileX, tileY] = Tiles.tile_from_index(source.tileIndex) + while (tileZ - 1 > this._z) { tileX = Math.floor(tileX / 2) tileY = Math.floor(tileY / 2) tileZ-- } - const tileI = Tiles.tile_index(tileZ, tileX, tileY) - let counts = perTileCount.get(tileI) - if (counts === undefined) { - counts = new Map() - perTileCount.set(tileI, counts) + const xDiff = tileX - (2 * this._x) + const yDiff = tileY - (2 * this._y) + + const subtileIndex = yDiff * 2 + xDiff; + if (this._subtiles[subtileIndex] === undefined) { + this._subtiles[subtileIndex] = new TileHierarchyAggregator(this, tileZ, tileX, tileY) } - singleTileCounter.countsPerLayer.data.forEach((count, layerId) => { - if (counts.has(layerId)) { - counts.set(layerId, count + counts.get(layerId)) - } else { - counts.set(layerId, count) - } + this._subtiles[subtileIndex].addTile(source) + } + + } + + public static createHierarchy() { + return new TileHierarchyAggregator(undefined, 0, 0, 0) + } + + + private visitSubTiles(f : (aggr: TileHierarchyAggregator) => boolean){ + const visitFurther = f(this) + if(visitFurther){ + this._subtiles.forEach(tile => tile?.visitSubTiles(f)) + } + } + + getCountsForZoom(locationControl: UIEventSource<{ zoom : number }>, cutoff: number) : FeatureSource{ + const self = this + return new StaticFeatureSource( + locationControl.map(loc => { + const features = [] + const targetZoom = loc.zoom + self.visitSubTiles(aggr => { + if(aggr.totalValue < cutoff) { + return false + } + if(aggr._z === targetZoom){ + features.push(...aggr.features.data) + return false + } + return aggr._z <= targetZoom; + + }) + + return features }) - } - return perTileCount; + , true); } - - public addTile(tile: FeatureSourceForLayer & Tiled, shouldBeCounted: UIEventSource) { - let counter = this.perTile.get(tile.tileIndex) - if (counter === undefined) { - counter = new SingleTileCounter(tile.tileIndex) - this.perTile.set(tile.tileIndex, counter) - // We do **NOT** add a callback on the perTile index, even though we could! It'll update just fine without it - } - counter.addTileCount(tile, shouldBeCounted) - } - - } /** @@ -131,6 +189,7 @@ class SingleTileCounter implements Tiled { public readonly x: number public readonly y: number + constructor(tileIndex: number) { this.tileIndex = tileIndex this.bbox = BBox.fromTileIndex(tileIndex) @@ -140,17 +199,16 @@ class SingleTileCounter implements Tiled { this.y = y } - public addTileCount(source: FeatureSourceForLayer, shouldBeCounted: UIEventSource) { + public addTileCount(source: FeatureSourceForLayer) { const layer = source.layer.layerDef this.registeredLayers.set(layer.id, layer) const self = this + source.features.map(f => { - /*if (!shouldBeCounted.data) { - return; - }*/ self.countsPerLayer.data.set(layer.id, f.length) self.countsPerLayer.ping() - }, [shouldBeCounted]) + }) + } } \ No newline at end of file diff --git a/UI/ShowDataLayer/ShowDataLayer.ts b/UI/ShowDataLayer/ShowDataLayer.ts index 278ea35974..0d02bcd4e1 100644 --- a/UI/ShowDataLayer/ShowDataLayer.ts +++ b/UI/ShowDataLayer/ShowDataLayer.ts @@ -17,6 +17,7 @@ export default class ShowDataLayer { // Used to generate a fresh ID when needed private _cleanCount = 0; private geoLayer = undefined; + private isDirty = false; /** * If the selected element triggers, this is used to lookup the correct layer and to open the popup @@ -37,9 +38,30 @@ export default class ShowDataLayer { this._layerToShow = options.layerToShow; const self = this; + options.leafletMap.addCallbackAndRunD(_ => { + self.update(options) + } + ); + features.addCallback(_ => self.update(options)); - options.leafletMap.addCallback(_ => self.update(options)); - this.update(options); + options.doShowLayer?.addCallbackAndRun(doShow => { + const mp = options.leafletMap.data; + if (mp == undefined) { + return; + } + if (doShow) { + if (self.isDirty) { + self.update(options) + } else { + mp.addLayer(this.geoLayer) + } + } else { + if(this.geoLayer !== undefined){ + mp.removeLayer(this.geoLayer) + } + } + + }) State.state.selectedElement.addCallbackAndRunD(selected => { if (self._leafletMap.data === undefined) { @@ -68,26 +90,16 @@ export default class ShowDataLayer { } }) - options.doShowLayer?.addCallbackAndRun(doShow => { - const mp = options.leafletMap.data; - if (this.geoLayer == undefined || mp == undefined) { - return; - } - if (doShow) { - mp.addLayer(this.geoLayer) - } else { - mp.removeLayer(this.geoLayer) - } - - - }) - } - private update(options) { + private update(options: ShowDataLayerOptions) { if (this._features.data === undefined) { return; } + this.isDirty = true; + if (options?.doShowLayer?.data === false) { + return; + } const mp = options.leafletMap.data; if (mp === undefined) { @@ -99,7 +111,18 @@ export default class ShowDataLayer { mp.removeLayer(this.geoLayer); } - this.geoLayer = this.CreateGeojsonLayer() + const self = this; + const data = { + type: "FeatureCollection", + features: [] + } + // @ts-ignore + this.geoLayer = L.geoJSON(data, { + style: feature => self.createStyleFor(feature), + pointToLayer: (feature, latLng) => self.pointToLayer(feature, latLng), + onEachFeature: (feature, leafletLayer) => self.postProcessFeature(feature, leafletLayer) + }); + const allFeats = this._features.data; for (const feat of allFeats) { if (feat === undefined) { @@ -123,6 +146,7 @@ export default class ShowDataLayer { if (options.doShowLayer?.data ?? true) { mp.addLayer(this.geoLayer) } + this.isDirty = false; } @@ -143,7 +167,7 @@ export default class ShowDataLayer { return; } - const tagSource = feature.properties.id === undefined ? new UIEventSource(feature.properties) : + const tagSource = feature.properties.id === undefined ? new UIEventSource(feature.properties) : State.state.allElements.getEventSourceById(feature.properties.id) const clickable = !(layer.title === undefined && (layer.tagRenderings ?? []).length === 0) const style = layer.GenerateLeafletStyle(tagSource, clickable); @@ -218,19 +242,4 @@ export default class ShowDataLayer { } - private CreateGeojsonLayer(): L.Layer { - const self = this; - const data = { - type: "FeatureCollection", - features: [] - } - // @ts-ignore - return L.geoJSON(data, { - style: feature => self.createStyleFor(feature), - pointToLayer: (feature, latLng) => self.pointToLayer(feature, latLng), - onEachFeature: (feature, leafletLayer) => self.postProcessFeature(feature, leafletLayer) - }); - - } - } \ No newline at end of file diff --git a/assets/layers/tree_node/tree_node.json b/assets/layers/tree_node/tree_node.json index bba09af121..59c1d03c14 100644 --- a/assets/layers/tree_node/tree_node.json +++ b/assets/layers/tree_node/tree_node.json @@ -7,7 +7,7 @@ "ru": "Дерево", "fr": "Arbre" }, - "minzoom": 14, + "minzoom": 16, "source": { "osmTags": { "and": [ diff --git a/assets/themes/trees/trees.json b/assets/themes/trees/trees.json index 52961293a3..c609a43f11 100644 --- a/assets/themes/trees/trees.json +++ b/assets/themes/trees/trees.json @@ -45,10 +45,11 @@ "startLat": 50.642, "startLon": 4.482, "startZoom": 8, - "widenFactor": 1.5, + "widenFactor": 1.01, "socialImage": "./assets/themes/trees/logo.svg", "clustering": { - "maxZoom": 18 + "maxZoom": 19, + "minNeededElements": 25 }, "layers": [ "tree_node"