From 8fca373437deaf7adaa04ed3a62d1a46e7a6d6fe Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Fri, 15 Oct 2021 18:48:33 +0200 Subject: [PATCH] Optimizing trees theme and clustering --- Logic/FeatureSource/FeaturePipeline.ts | 13 +- Logic/State/FeaturePipelineState.ts | 11 +- Logic/State/UserRelatedState.ts | 1 + Models/ThemeConfig/LayoutConfig.ts | 2 +- UI/ShowDataLayer/ShowDataLayer.ts | 5 +- UI/ShowDataLayer/TileHierarchyAggregator.ts | 125 ++++++++++++------ Utils.ts | 5 + .../layers/cluster_style/cluster_style.json | 16 ++- assets/layers/tree_node/tree_node.json | 9 +- assets/themes/trees/trees.json | 3 +- index.ts | 2 +- 11 files changed, 125 insertions(+), 67 deletions(-) diff --git a/Logic/FeatureSource/FeaturePipeline.ts b/Logic/FeatureSource/FeaturePipeline.ts index b14a42c31..ffe194043 100644 --- a/Logic/FeatureSource/FeaturePipeline.ts +++ b/Logic/FeatureSource/FeaturePipeline.ts @@ -58,7 +58,7 @@ export default class FeaturePipeline { private readonly freshnesses = new Map(); private readonly oldestAllowedDate: Date = new Date(new Date().getTime() - 60 * 60 * 24 * 30 * 1000); - private readonly osmSourceZoomLevel = 15 + private readonly osmSourceZoomLevel constructor( handleFeatureSource: (source: FeatureSourceForLayer & Tiled) => void, @@ -74,10 +74,12 @@ export default class FeaturePipeline { readonly overpassMaxZoom: UIEventSource; readonly osmConnection: OsmConnection readonly currentBounds: UIEventSource, + readonly osmApiTileSize: UIEventSource }) { this.state = state; const self = this + this.osmSourceZoomLevel = state.osmApiTileSize.data; // milliseconds const useOsmApi = state.locationControl.map(l => l.zoom > (state.overpassMaxZoom.data ?? 12)) this.relationTracker = new RelationsTracker() @@ -275,18 +277,19 @@ export default class FeaturePipeline { private getNeededTilesFromOsm(isSufficientlyZoomed: UIEventSource): UIEventSource { const self = this return this.state.currentBounds.map(bbox => { + console.log("Current bbox is", bbox) if (bbox === undefined) { - return undefined + return [] } if (!isSufficientlyZoomed.data) { - return undefined; + return []; } const osmSourceZoomLevel = self.osmSourceZoomLevel const range = bbox.containingTileRange(osmSourceZoomLevel) const tileIndexes = [] if (range.total >= 100) { // Too much tiles! - return [] + return undefined } Tiles.MapRange(range, (x, y) => { const i = Tiles.tile_index(osmSourceZoomLevel, x, y); @@ -299,7 +302,7 @@ export default class FeaturePipeline { tileIndexes.push(i) }) return tileIndexes - }) + }, [isSufficientlyZoomed]) } private initOverpassUpdater(state: { diff --git a/Logic/State/FeaturePipelineState.ts b/Logic/State/FeaturePipelineState.ts index d20800035..53f21341b 100644 --- a/Logic/State/FeaturePipelineState.ts +++ b/Logic/State/FeaturePipelineState.ts @@ -8,10 +8,6 @@ import {UIEventSource} from "../UIEventSource"; import MapState from "./MapState"; import SelectedFeatureHandler from "../Actors/SelectedFeatureHandler"; import Hash from "../Web/Hash"; -import ScrollableFullScreen from "../../UI/Base/ScrollableFullScreen"; -import Translations from "../../UI/i18n/Translations"; -import SimpleAddUI from "../../UI/BigComponents/SimpleAddUI"; -import StrayClickHandler from "../Actors/StrayClickHandler"; export default class FeaturePipelineState extends MapState { @@ -25,7 +21,7 @@ export default class FeaturePipelineState extends MapState { super(layoutToUse); const clustering = layoutToUse.clustering - this.featureAggregator = TileHierarchyAggregator.createHierarchy(); + this.featureAggregator = TileHierarchyAggregator.createHierarchy(this); const clusterCounter = this.featureAggregator const self = this; this.featurePipeline = new FeaturePipeline( @@ -117,12 +113,9 @@ export default class FeaturePipelineState extends MapState { features: this.featureAggregator.getCountsForZoom(clustering, this.locationControl, clustering.minNeededElements), leafletMap: leafletMap, layerToShow: ShowTileInfo.styling, - enablePopups: false, + enablePopups: this.featureSwitchIsDebugging.data, }) } - - - } \ No newline at end of file diff --git a/Logic/State/UserRelatedState.ts b/Logic/State/UserRelatedState.ts index e431bb7f7..622d7f394 100644 --- a/Logic/State/UserRelatedState.ts +++ b/Logic/State/UserRelatedState.ts @@ -83,6 +83,7 @@ export default class UserRelatedState extends ElementsState { this.InitializeLanguage(); + this.initHomeLocation() new SelectedElementTagsUpdater(this) } diff --git a/Models/ThemeConfig/LayoutConfig.ts b/Models/ThemeConfig/LayoutConfig.ts index 7527dd2cc..f83b0cb5b 100644 --- a/Models/ThemeConfig/LayoutConfig.ts +++ b/Models/ThemeConfig/LayoutConfig.ts @@ -90,7 +90,7 @@ export default class LayoutConfig { this.startZoom = json.startZoom; this.startLat = json.startLat; this.startLon = json.startLon; - if(json.widenFactor < 1){ + if(json.widenFactor < 0.02){ if(official){ throw "Widenfactor too small" }else{ diff --git a/UI/ShowDataLayer/ShowDataLayer.ts b/UI/ShowDataLayer/ShowDataLayer.ts index d50b6711f..2f957ed15 100644 --- a/UI/ShowDataLayer/ShowDataLayer.ts +++ b/UI/ShowDataLayer/ShowDataLayer.ts @@ -38,7 +38,8 @@ export default class ShowDataLayer { ShowDataLayer.dataLayerIds++ this._enablePopups = options.enablePopups ?? true; if (options.features === undefined) { - throw "Invalid ShowDataLayer invocation" + console.error("Invalid ShowDataLayer invocation: options.features is undefed") + throw "Invalid ShowDataLayer invocation: options.features is undefed" } const features = options.features.features.map(featFreshes => featFreshes.map(ff => ff.feature)); this._features = features; @@ -166,7 +167,7 @@ export default class ShowDataLayer { private createStyleFor(feature) { - const tagsSource = this.allElements?.addOrGetElement(feature) ?? new UIEventSource(feature.properties.id); + const tagsSource = this.allElements?.addOrGetElement(feature) ?? new UIEventSource(feature.properties); // Every object is tied to exactly one layer const layer = this._layerToShow return layer?.GenerateLeafletStyle(tagsSource, true); diff --git a/UI/ShowDataLayer/TileHierarchyAggregator.ts b/UI/ShowDataLayer/TileHierarchyAggregator.ts index c8a19ab5f..42878131c 100644 --- a/UI/ShowDataLayer/TileHierarchyAggregator.ts +++ b/UI/ShowDataLayer/TileHierarchyAggregator.ts @@ -4,7 +4,11 @@ import {UIEventSource} from "../../Logic/UIEventSource"; import {Tiles} from "../../Models/TileRange"; import StaticFeatureSource from "../../Logic/FeatureSource/Sources/StaticFeatureSource"; import {BBox} from "../../Logic/BBox"; +import FilteredLayer from "../../Models/FilteredLayer"; +/** + * A feature source containing but a single feature, which keeps stats about a tile + */ export class TileHierarchyAggregator implements FeatureSource { private _parent: TileHierarchyAggregator; private _root: TileHierarchyAggregator; @@ -16,28 +20,39 @@ export class TileHierarchyAggregator implements FeatureSource { private _subtiles: [TileHierarchyAggregator, TileHierarchyAggregator, TileHierarchyAggregator, TileHierarchyAggregator] = [undefined, undefined, undefined, undefined] public totalValue: number = 0 + public showCount: number = 0 + public hiddenCount: number = 0 private static readonly empty = [] public readonly features = new UIEventSource<{ feature: any, freshness: Date }[]>(TileHierarchyAggregator.empty) public readonly name; private readonly featuresStatic = [] - private readonly featureProperties: { count: string, kilocount: string, tileId: string, id: string }; - - private constructor(parent: TileHierarchyAggregator, z: number, x: number, y: number) { + private readonly featureProperties: { count: string, kilocount: string, tileId: string, id: string, showCount: string, totalCount: string }; + private readonly _state: { filteredLayers: UIEventSource }; + private readonly updateSignal = new UIEventSource(undefined) + + private constructor(parent: TileHierarchyAggregator, + state: { + filteredLayers: UIEventSource + }, + z: number, x: number, y: number) { this._parent = parent; + this._state = state; this._root = parent?._root ?? this this._z = z; this._x = x; this._y = y; this._tileIndex = Tiles.tile_index(z, x, y) this.name = "Count(" + this._tileIndex + ")" - + const totals = { - id: ""+this._tileIndex, - tileId: ""+this._tileIndex, + id: "" + this._tileIndex, + tileId: "" + this._tileIndex, count: `0`, - kilocount: "0" + kilocount: "0", + showCount: "0", + totalCount: "0" } this.featureProperties = totals @@ -91,25 +106,57 @@ export class TileHierarchyAggregator implements FeatureSource { private update() { const newMap = new Map() let total = 0 + let hiddenCount = 0 + let showCount = 0 + let isShown: Map = new Map() + for (const filteredLayer of this._state.filteredLayers.data) { + isShown.set(filteredLayer.layerDef.id, filteredLayer) + } this?._counter?.countsPerLayer?.data?.forEach((count, layerId) => { - newMap.set(layerId, count) + newMap.set("layer:" + layerId, count) total += count + this.featureProperties["direct_layer:" + layerId] = count + const flayer = isShown.get(layerId) + if (flayer.isDisplayed.data && this._z >= flayer.layerDef.minzoom) { + showCount += count + } else { + hiddenCount += count; + } }) + for (const tile of this._subtiles) { if (tile === undefined) { continue; } total += tile.totalValue + + showCount += tile.showCount + hiddenCount += tile.hiddenCount + + for (const key in tile.featureProperties) { + if (key.startsWith("layer:")) { + newMap.set(key, (newMap.get(key) ?? 0) + Number(tile.featureProperties[key] ?? 0)) + } + } } + this.totalValue = total + this.showCount = showCount + this.hiddenCount = hiddenCount this._parent?.update() - + if (total === 0) { this.features.setData(TileHierarchyAggregator.empty) } else { this.featureProperties.count = "" + total; - this.featureProperties.kilocount = "" +Math.floor(total/1000); + this.featureProperties.kilocount = "" + Math.floor(total / 1000); + this.featureProperties.showCount = "" + showCount + this.featureProperties.totalCount = "" + total + newMap.forEach((value, key) => { + this.featureProperties[key] = "" + value + }) + this.features.data = this.featuresStatic this.features.ping() } @@ -137,49 +184,48 @@ export class TileHierarchyAggregator implements FeatureSource { const subtileIndex = yDiff * 2 + xDiff; if (this._subtiles[subtileIndex] === undefined) { - this._subtiles[subtileIndex] = new TileHierarchyAggregator(this, tileZ, tileX, tileY) + this._subtiles[subtileIndex] = new TileHierarchyAggregator(this, this._state, tileZ, tileX, tileY) } this._subtiles[subtileIndex].addTile(source) } - + this.updateSignal.setData(source) } - public static createHierarchy() { - return new TileHierarchyAggregator(undefined, 0, 0, 0) + public static createHierarchy(state: { filteredLayers: UIEventSource }) { + return new TileHierarchyAggregator(undefined, state, 0, 0, 0) } - private visitSubTiles(f : (aggr: TileHierarchyAggregator) => boolean){ + private visitSubTiles(f: (aggr: TileHierarchyAggregator) => boolean) { const visitFurther = f(this) - if(visitFurther){ + if (visitFurther) { this._subtiles.forEach(tile => tile?.visitSubTiles(f)) } } - - getCountsForZoom(clusteringConfig: {maxZoom: number}, locationControl: UIEventSource<{ zoom : number }>, cutoff: number = 0) : FeatureSource{ + + getCountsForZoom(clusteringConfig: { maxZoom: number }, locationControl: UIEventSource<{ zoom: number }>, cutoff: number = 0): FeatureSource { const self = this const empty = [] - return new StaticFeatureSource( - locationControl.map(loc => loc.zoom).map(targetZoom => { - if(targetZoom-1 > clusteringConfig.maxZoom){ - return empty + const features = locationControl.map(loc => loc.zoom).map(targetZoom => { + if (targetZoom - 1 > clusteringConfig.maxZoom) { + return empty + } + + const features = [] + self.visitSubTiles(aggr => { + if (aggr.showCount < cutoff) { + return false } - - const features = [] - self.visitSubTiles(aggr => { - if(aggr.totalValue < cutoff) { - return false - } - if(aggr._z === targetZoom){ - features.push(...aggr.features.data) - return false - } - return aggr._z <= targetZoom; - - }) - - return features + if (aggr._z === targetZoom) { + features.push(...aggr.features.data) + return false + } + return aggr._z <= targetZoom; }) - , true); + + return features + }, [this.updateSignal.stabilized(500)]) + + return new StaticFeatureSource(features, true); } } @@ -209,14 +255,11 @@ class SingleTileCounter implements Tiled { const layer = source.layer.layerDef this.registeredLayers.set(layer.id, layer) const self = this - source.features.map(f => { const isDisplayed = source.layer.isDisplayed.data self.countsPerLayer.data.set(layer.id, isDisplayed ? f.length : 0) self.countsPerLayer.ping() }, [source.layer.isDisplayed]) - - } } \ No newline at end of file diff --git a/Utils.ts b/Utils.ts index 0dd2a8363..51aaf072b 100644 --- a/Utils.ts +++ b/Utils.ts @@ -168,7 +168,12 @@ export class Utils { if (!tags.hasOwnProperty(key)) { continue } + try{ txt = txt.replace(new RegExp("{" + key + "}", "g"), tags[key] ?? "") + }catch(e){ + console.error("WEIRD" , e) + throw e + } } txt = txt.replace(new RegExp('{.*}', "g"), "") return txt; diff --git a/assets/layers/cluster_style/cluster_style.json b/assets/layers/cluster_style/cluster_style.json index 982baa79b..7e93c506a 100644 --- a/assets/layers/cluster_style/cluster_style.json +++ b/assets/layers/cluster_style/cluster_style.json @@ -1,22 +1,26 @@ { "id": "cluster_style", - "description": "The style for the clustering in all themes.", + "description": "The style for the clustering in all themes. Enable `debug=true` to peak into clustered tiles", "source": { "osmTags": "tileId~*" }, + "title": "Clustered data", + "tagRenderings": [ + "all_tags" + ], "color": { "render": "#3c3", "mappings": [ { - "if": "count>200", + "if": "showCount>200", "then": "#f33" }, { - "if": "count>100", + "if": "showCount>100", "then": "#c93" }, { - "if": "count>50", + "if": "showCount>50", "then": "#cc3" } ] @@ -25,10 +29,10 @@ "render": "1" }, "label": { - "render": "
{count}
", + "render": "
{showCount}
", "mappings": [ { - "if": "count>1000", + "if": "showCount>1000", "then": "
{kilocount}K
" } ] diff --git a/assets/layers/tree_node/tree_node.json b/assets/layers/tree_node/tree_node.json index 59c1d03c1..a504f74b1 100644 --- a/assets/layers/tree_node/tree_node.json +++ b/assets/layers/tree_node/tree_node.json @@ -563,5 +563,12 @@ "ru": "Если вы не уверены в том, лиственное это дерево или хвойное." } } - ] + ], + "allowMove": { + "enableRelocation": false, + "enableImproveAccuraccy": true + }, + "deletion": { + "minNeededChangesets": 5 + } } \ No newline at end of file diff --git a/assets/themes/trees/trees.json b/assets/themes/trees/trees.json index c609a43f1..7c2baa8ac 100644 --- a/assets/themes/trees/trees.json +++ b/assets/themes/trees/trees.json @@ -45,12 +45,13 @@ "startLat": 50.642, "startLon": 4.482, "startZoom": 8, - "widenFactor": 1.01, + "widenFactor": 0.2, "socialImage": "./assets/themes/trees/logo.svg", "clustering": { "maxZoom": 19, "minNeededElements": 25 }, + "osmApiTileSize": 18, "layers": [ "tree_node" ], diff --git a/index.ts b/index.ts index 61ebf59e6..92ba9da66 100644 --- a/index.ts +++ b/index.ts @@ -106,7 +106,7 @@ DetermineLayout.GetLayout().then(value => { console.log("Got ", value) Init.Init(value[0], value[1]) }).catch(err => { - console.error(err) + console.error("Error while initializing: ", err, err.stack) })