midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Add initial clustering per tile, very broken

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-09-26 17:36:39 +02:00
parent 2b78c4b53f
commit c5e9448720
88 changed files with 1080 additions and 651 deletions
Download patch file Download diff file Expand all files Collapse all files

31
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -37,7 +37,7 @@ export default class FeaturePipeline implements FeatureSourceState {
private readonly perLayerHierarchy: Map<string, TileHierarchyMerger>;
constructor(
handleFeatureSource: (source: FeatureSourceForLayer) => void,
handleFeatureSource: (source: FeatureSourceForLayer & Tiled) => void,
state: {
filteredLayers: UIEventSource<FilteredLayer[]>,
locationControl: UIEventSource<Loc>,
@ -52,7 +52,6 @@ export default class FeaturePipeline implements FeatureSourceState {
const self = this
const updater = new OverpassFeatureSource(state);
updater.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(updater))
this.overpassUpdater = updater;
this.sufficientlyZoomed = updater.sufficientlyZoomed
this.runningQuery = updater.runningQuery
@ -65,14 +64,15 @@ export default class FeaturePipeline implements FeatureSourceState {
const perLayerHierarchy = new Map<string, TileHierarchyMerger>()
this.perLayerHierarchy = perLayerHierarchy
const patchedHandleFeatureSource = function (src: FeatureSourceForLayer & IndexedFeatureSource) {
const patchedHandleFeatureSource = function (src: FeatureSourceForLayer & IndexedFeatureSource & Tiled) {
// This will already contain the merged features for this tile. In other words, this will only be triggered once for every tile
const srcFiltered =
new FilteringFeatureSource(state,
new FilteringFeatureSource(state, src.tileIndex,
new WayHandlingApplyingFeatureSource(
new ChangeGeometryApplicator(src, state.changes)
)
)
handleFeatureSource(srcFiltered)
self.somethingLoaded.setData(true)
};
@ -102,10 +102,12 @@ export default class FeaturePipeline implements FeatureSourceState {
if (source.geojsonZoomLevel === undefined) {
// This is a 'load everything at once' geojson layer
// We split them up into tiles
// We split them up into tiles anyway
const src = new GeoJsonSource(filteredLayer)
TiledFeatureSource.createHierarchy(src, {
layer: src.layer,
minZoomLevel:14,
dontEnforceMinZoom: true,
registerTile: (tile) => {
new RegisteringAllFromFeatureSourceActor(tile)
addToHierarchy(tile, id)
@ -115,14 +117,11 @@ export default class FeaturePipeline implements FeatureSourceState {
} else {
new DynamicGeoJsonTileSource(
filteredLayer,
src => TiledFeatureSource.createHierarchy(src, {
layer: src.layer,
registerTile: (tile) => {
tile => {
new RegisteringAllFromFeatureSourceActor(tile)
addToHierarchy(tile, id)
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
}
}),
},
state
)
}
@ -133,13 +132,17 @@ export default class FeaturePipeline implements FeatureSourceState {
new PerLayerFeatureSourceSplitter(state.filteredLayers,
(source) => TiledFeatureSource.createHierarchy(source, {
layer: source.layer,
minZoomLevel: 14,
dontEnforceMinZoom: true,
registerTile: (tile) => {
// We save the tile data for the given layer to local storage
new SaveTileToLocalStorageActor(tile, tile.tileIndex)
addToHierarchy(tile, source.layer.layerDef.id);
addToHierarchy(new RememberingSource(tile), source.layer.layerDef.id);
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
}
}),
new RememberingSource(updater))
updater)
// Also load points/lines that are newly added.
@ -152,6 +155,8 @@ export default class FeaturePipeline implements FeatureSourceState {
addToHierarchy(perLayer, perLayer.layer.layerDef.id)
// AT last, we always apply the metatags whenever possible
perLayer.features.addCallbackAndRunD(_ => self.applyMetaTags(perLayer))
perLayer.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(perLayer))
},
newGeometry
)
@ -166,6 +171,7 @@ export default class FeaturePipeline implements FeatureSourceState {
private applyMetaTags(src: FeatureSourceForLayer){
const self = this
console.log("Applying metatagging onto ", src.name)
MetaTagging.addMetatags(
src.features.data,
{
@ -183,6 +189,7 @@ export default class FeaturePipeline implements FeatureSourceState {
private updateAllMetaTagging() {
const self = this;
console.log("Reupdating all metatagging")
this.perLayerHierarchy.forEach(hierarchy => {
hierarchy.loadedTiles.forEach(src => {
self.applyMetaTags(src)

3
Logic/FeatureSource/Sources/FeatureSourceMerger.ts
View file

@ -7,6 +7,7 @@ import FeatureSource, {FeatureSourceForLayer, IndexedFeatureSource, Tiled} from
import FilteredLayer from "../../../Models/FilteredLayer";
import {BBox} from "../../GeoOperations";
import {Utils} from "../../../Utils";
import {Tiles} from "../../../Models/TileRange";
export default class FeatureSourceMerger implements FeatureSourceForLayer, Tiled, IndexedFeatureSource {
@ -23,7 +24,7 @@ export default class FeatureSourceMerger implements FeatureSourceForLayer, Tiled
this.bbox = bbox;
this._sources = sources;
this.layer = layer;
this.name = "FeatureSourceMerger("+layer.layerDef.id+", "+Utils.tile_from_index(tileIndex).join(",")+")"
this.name = "FeatureSourceMerger("+layer.layerDef.id+", "+Tiles.tile_from_index(tileIndex).join(",")+")"
const self = this;
const handledSources = new Set<FeatureSource>();

26
Logic/FeatureSource/Sources/FilteringFeatureSource.ts
View file

@ -1,24 +1,29 @@
import {UIEventSource} from "../../UIEventSource";
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig";
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer} from "../FeatureSource";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import Hash from "../../Web/Hash";
import {BBox} from "../../GeoOperations";
export default class FilteringFeatureSource implements FeatureSourceForLayer {
export default class FilteringFeatureSource implements FeatureSourceForLayer , Tiled {
public features: UIEventSource<{ feature: any; freshness: Date }[]> =
new UIEventSource<{ feature: any; freshness: Date }[]>([]);
public readonly name;
public readonly layer: FilteredLayer;
public readonly tileIndex : number
public readonly bbox : BBox
constructor(
state: {
locationControl: UIEventSource<{ zoom: number }>,
selectedElement: UIEventSource<any>,
},
tileIndex,
upstream: FeatureSourceForLayer
) {
const self = this;
this.name = "FilteringFeatureSource("+upstream.name+")"
this.tileIndex = tileIndex
this.bbox = BBox.fromTileIndex(tileIndex)
this.layer = upstream.layer;
const layer = upstream.layer;
@ -51,7 +56,7 @@ export default class FilteringFeatureSource implements FeatureSourceForLayer {
return false;
}
}
if (!FilteringFeatureSource.showLayer(layer, state.locationControl.data)) {
if (!layer.isDisplayed) {
// The layer itself is either disabled or hidden due to zoom constraints
// We should return true, but it might still match some other layer
return false;
@ -66,10 +71,7 @@ export default class FilteringFeatureSource implements FeatureSourceForLayer {
update();
});
let isShown = state.locationControl.map((l) => FilteringFeatureSource.showLayer(layer, l),
[layer.isDisplayed])
isShown.addCallback(isShown => {
layer.isDisplayed.addCallback(isShown => {
if (isShown) {
update();
} else {
@ -78,7 +80,7 @@ export default class FilteringFeatureSource implements FeatureSourceForLayer {
});
layer.appliedFilters.addCallback(_ => {
if(!isShown.data){
if(!layer.isDisplayed.data){
// Currently not shown.
// Note that a change in 'isSHown' will trigger an update as well, so we don't have to watch it another time
return;
@ -93,10 +95,8 @@ export default class FilteringFeatureSource implements FeatureSourceForLayer {
layer: {
isDisplayed: UIEventSource<boolean>;
layerDef: LayerConfig;
},
location: { zoom: number }) {
return layer.isDisplayed.data &&
layer.layerDef.minzoomVisible <= location.zoom;
}) {
return layer.isDisplayed.data;
}
}

6
Logic/FeatureSource/Sources/GeoJsonSource.ts
View file

@ -6,6 +6,7 @@ import FilteredLayer from "../../../Models/FilteredLayer";
import {Utils} from "../../../Utils";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {BBox} from "../../GeoOperations";
import {Tiles} from "../../../Models/TileRange";
export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
@ -35,10 +36,10 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
.replace('{z}', "" + z)
.replace('{x}', "" + x)
.replace('{y}', "" + y)
this.tileIndex = Utils.tile_index(z, x, y)
this.tileIndex = Tiles.tile_index(z, x, y)
this.bbox = BBox.fromTile(z, x, y)
} else {
this.tileIndex = Utils.tile_index(0, 0, 0)
this.tileIndex = Tiles.tile_index(0, 0, 0)
this.bbox = BBox.global;
}
@ -89,7 +90,6 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
newFeatures.push({feature: feature, freshness: freshness})
}
console.debug("Downloaded " + newFeatures.length + " new features and " + skipped + " already seen features from " + url);
if (newFeatures.length == 0) {
return;

14
Logic/FeatureSource/Sources/RememberingSource.ts
View file

@ -2,17 +2,23 @@
* Every previously added point is remembered, but new points are added.
* Data coming from upstream will always overwrite a previous value
*/
import FeatureSource from "../FeatureSource";
import FeatureSource, {Tiled} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import {BBox} from "../../GeoOperations";
export default class RememberingSource implements FeatureSource {
export default class RememberingSource implements FeatureSource , Tiled{
public readonly features: UIEventSource<{ feature: any, freshness: Date }[]>;
public readonly name;
constructor(source: FeatureSource) {
public readonly tileIndex : number
public readonly bbox : BBox
constructor(source: FeatureSource & Tiled) {
const self = this;
this.name = "RememberingSource of " + source.name;
this.tileIndex= source.tileIndex
this.bbox = source.bbox;
const empty = [];
this.features = source.features.map(features => {
const oldFeatures = self.features?.data ?? empty;

3
Logic/FeatureSource/Sources/SimpleFeatureSource.ts
View file

@ -3,13 +3,14 @@ import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {BBox} from "../../GeoOperations";
import {Utils} from "../../../Utils";
import {Tiles} from "../../../Models/TileRange";
export default class SimpleFeatureSource implements FeatureSourceForLayer, Tiled {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]);
public readonly name: string = "SimpleFeatureSource";
public readonly layer: FilteredLayer;
public readonly bbox: BBox = BBox.global;
public readonly tileIndex: number = Utils.tile_index(0, 0, 0);
public readonly tileIndex: number = Tiles.tile_index(0, 0, 0);
constructor(layer: FilteredLayer) {
this.name = "SimpleFeatureSource(" + layer.layerDef.id + ")"

7
Logic/FeatureSource/Sources/StaticFeatureSource.ts
View file

@ -8,12 +8,13 @@ export default class StaticFeatureSource implements FeatureSource {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name: string = "StaticFeatureSource"
constructor(features: any[] | UIEventSource<any[]>, useFeaturesDirectly = false) {
constructor(features: any[] | UIEventSource<any[] | UIEventSource<{ feature: any, freshness: Date }>>, useFeaturesDirectly) {
const now = new Date();
if(useFeaturesDirectly){
if (useFeaturesDirectly) {
// @ts-ignore
this.features = features
}else if (features instanceof UIEventSource) {
} else if (features instanceof UIEventSource) {
// @ts-ignore
this.features = features.map(features => features.map(f => ({feature: f, freshness: now})))
} else {
this.features = new UIEventSource(features.map(f => ({

5
Logic/FeatureSource/Sources/WayHandlingApplyingFeatureSource.ts
View file

@ -12,10 +12,11 @@ export default class WayHandlingApplyingFeatureSource implements FeatureSourceFo
public readonly layer;
constructor(upstream: FeatureSourceForLayer) {
this.name = "Wayhandling(" + upstream.name+")";
this.name = "Wayhandling(" + upstream.name + ")";
this.layer = upstream.layer
const layer = upstream.layer.layerDef;
if (layer.wayHandling === LayerConfig.WAYHANDLING_DEFAULT) {
// We don't have to do anything fancy
// lets just wire up the upstream

4
Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts
View file

@ -1,5 +1,5 @@
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer} from "../FeatureSource";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import Loc from "../../../Models/Loc";
import DynamicTileSource from "./DynamicTileSource";
@ -8,7 +8,7 @@ import GeoJsonSource from "../Sources/GeoJsonSource";
export default class DynamicGeoJsonTileSource extends DynamicTileSource {
constructor(layer: FilteredLayer,
registerLayer: (layer: FeatureSourceForLayer) => void,
registerLayer: (layer: FeatureSourceForLayer & Tiled) => void,
state: {
locationControl: UIEventSource<Loc>
leafletMap: any

7
Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts
View file

@ -6,6 +6,7 @@ import {Utils} from "../../../Utils";
import {UIEventSource} from "../../UIEventSource";
import Loc from "../../../Models/Loc";
import TileHierarchy from "./TileHierarchy";
import {Tiles} from "../../../Models/TileRange";
/***
* A tiled source which dynamically loads the required tiles at a fixed zoom level
@ -46,9 +47,9 @@ export default class DynamicTileSource implements TileHierarchy<FeatureSourceFor
// We'll retry later
return undefined
}
const tileRange = Utils.TileRangeBetween(zoomlevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const tileRange = Tiles.TileRangeBetween(zoomlevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const needed = Utils.MapRange(tileRange, (x, y) => Utils.tile_index(zoomlevel, x, y)).filter(i => !self._loadedTiles.has(i))
const needed = Tiles.MapRange(tileRange, (x, y) => Tiles.tile_index(zoomlevel, x, y)).filter(i => !self._loadedTiles.has(i))
if (needed.length === 0) {
return undefined
}
@ -63,7 +64,7 @@ export default class DynamicTileSource implements TileHierarchy<FeatureSourceFor
}
for (const neededIndex of neededIndexes) {
self._loadedTiles.add(neededIndex)
const src = constructTile( Utils.tile_from_index(neededIndex))
const src = constructTile(Tiles.tile_from_index(neededIndex))
if(src !== undefined){
self.loadedTiles.set(neededIndex, src)
}

5
Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts
View file

@ -5,6 +5,7 @@ import FilteredLayer from "../../../Models/FilteredLayer";
import {Utils} from "../../../Utils";
import {BBox} from "../../GeoOperations";
import FeatureSourceMerger from "../Sources/FeatureSourceMerger";
import {Tiles} from "../../../Models/TileRange";
export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
@ -13,7 +14,7 @@ export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer
public readonly layer: FilteredLayer;
private _handleTile: (src: FeatureSourceForLayer & IndexedFeatureSource, index: number) => void;
constructor(layer: FilteredLayer, handleTile: (src: FeatureSourceForLayer & IndexedFeatureSource, index: number) => void) {
constructor(layer: FilteredLayer, handleTile: (src: FeatureSourceForLayer & IndexedFeatureSource & Tiled, index: number) => void) {
this.layer = layer;
this._handleTile = handleTile;
}
@ -37,7 +38,7 @@ export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer
// We have to setup
const sources = new UIEventSource<FeatureSource[]>([src])
this.sources.set(index, sources)
const merger = new FeatureSourceMerger(this.layer, index, BBox.fromTile(...Utils.tile_from_index(index)), sources)
const merger = new FeatureSourceMerger(this.layer, index, BBox.fromTile(...Tiles.tile_from_index(index)), sources)
this.loadedTiles.set(index, merger)
this._handleTile(merger, index)
}

17
Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource.ts
View file

@ -4,7 +4,7 @@ import {Utils} from "../../../Utils";
import {BBox} from "../../GeoOperations";
import FilteredLayer from "../../../Models/FilteredLayer";
import TileHierarchy from "./TileHierarchy";
import {feature} from "@turf/turf";
import {Tiles} from "../../../Models/TileRange";
/**
* Contains all features in a tiled fashion.
@ -41,12 +41,12 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
this.x = x;
this.y = y;
this.bbox = BBox.fromTile(z, x, y)
this.tileIndex = Utils.tile_index(z, x, y)
this.tileIndex = Tiles.tile_index(z, x, y)
this.name = `TiledFeatureSource(${z},${x},${y})`
this.parent = parent;
this.layer = options.layer
options = options ?? {}
this.maxFeatureCount = options?.maxFeatureCount ?? 500;
this.maxFeatureCount = options?.maxFeatureCount ?? 250;
this.maxzoom = options.maxZoomLevel ?? 18
this.options = options;
if (parent === undefined) {
@ -61,7 +61,7 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
} else {
this.root = this.parent.root;
this.loadedTiles = this.root.loadedTiles;
const i = Utils.tile_index(z, x, y)
const i = Tiles.tile_index(z, x, y)
this.root.loadedTiles.set(i, this)
}
this.features = new UIEventSource<any[]>([])
@ -143,9 +143,7 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
for (const feature of features) {
const bbox = BBox.get(feature.feature)
if (this.options.minZoomLevel === undefined) {
if (this.options.dontEnforceMinZoom || this.options.minZoomLevel === undefined) {
if (bbox.isContainedIn(this.upper_left.bbox)) {
ulf.push(feature)
} else if (bbox.isContainedIn(this.upper_right.bbox)) {
@ -186,6 +184,11 @@ export interface TiledFeatureSourceOptions {
readonly maxFeatureCount?: number,
readonly maxZoomLevel?: number,
readonly minZoomLevel?: number,
/**
* IF minZoomLevel is set, and if a feature runs through a tile boundary, it would normally be duplicated.
* Setting 'dontEnforceMinZoomLevel' will still allow bigger zoom levels for those features
*/
readonly dontEnforceMinZoom?: boolean,
readonly registerTile?: (tile: TiledFeatureSource & Tiled) => void,
readonly layer?: FilteredLayer
}

13
Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts
View file

@ -6,6 +6,7 @@ import TileHierarchy from "./TileHierarchy";
import {Utils} from "../../../Utils";
import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor";
import {BBox} from "../../GeoOperations";
import {Tiles} from "../../../Models/TileRange";
export default class TiledFromLocalStorageSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
@ -17,6 +18,7 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
leafletMap: any
}) {
const undefinedTiles = new Set<number>()
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-"
// @ts-ignore
const indexes: number[] = Object.keys(localStorage)
@ -27,7 +29,7 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
return Number(key.substring(prefix.length));
})
console.log("Layer", layer.layerDef.id, "has following tiles in available in localstorage", indexes.map(i => Utils.tile_from_index(i).join("/")).join(", "))
console.log("Layer", layer.layerDef.id, "has following tiles in available in localstorage", indexes.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
const zLevels = indexes.map(i => i % 100)
const indexesSet = new Set(indexes)
@ -57,9 +59,9 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
const needed = []
for (let z = minZoom; z <= maxZoom; z++) {
const tileRange = Utils.TileRangeBetween(z, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const neededZ = Utils.MapRange(tileRange, (x, y) => Utils.tile_index(z, x, y))
.filter(i => !self.loadedTiles.has(i) && indexesSet.has(i))
const tileRange = Tiles.TileRangeBetween(z, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const neededZ = Tiles.MapRange(tileRange, (x, y) => Tiles.tile_index(z, x, y))
.filter(i => !self.loadedTiles.has(i) && !undefinedTiles.has(i) && indexesSet.has(i))
needed.push(...neededZ)
}
@ -84,12 +86,13 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
features: new UIEventSource<{ feature: any; freshness: Date }[]>(features),
name: "FromLocalStorage(" + key + ")",
tileIndex: neededIndex,
bbox: BBox.fromTile(...Utils.tile_from_index(neededIndex))
bbox: BBox.fromTileIndex(neededIndex)
}
handleFeatureSource(src, neededIndex)
self.loadedTiles.set(neededIndex, src)
} catch (e) {
console.error("Could not load data tile from local storage due to", e)
undefinedTiles.add(neededIndex)
}
}