midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

More refactoring: using a decent, configurable datapipeline now

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-01-03 03:09:52 +01:00
parent 6ac8ec84e4
commit e42a668c4a
17 changed files with 434 additions and 265 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Logic/Actors/StrayClickHandler.ts
View file

@ -2,7 +2,6 @@ import * as L from "leaflet";
import {UIElement} from "../../UI/UIElement";
import Svg from "../../Svg";
import {UIEventSource} from "../UIEventSource";
import {FilteredLayer} from "../FilteredLayer";
import Img from "../../UI/Base/Img";
/**
@ -16,7 +15,7 @@ export class StrayClickHandler {
constructor(
lastClickLocation: UIEventSource<{ lat: number, lon:number }>,
selectedElement: UIEventSource<string>,
filteredLayers: UIEventSource<FilteredLayer[]>,
filteredLayers: UIEventSource<{ readonly isDisplayed: UIEventSource<boolean>}[]>,
leafletMap: UIEventSource<L.Map>,
fullscreenMessage: UIEventSource<UIElement>,
uiToShow: (() => UIElement)) {

24
Logic/UpdateFromOverpass.ts → Logic/Actors/UpdateFromOverpass.ts
View file

@ -1,22 +1,19 @@
import {Or, TagsFilter} from "./Tags";
import {UIEventSource} from "./UIEventSource";
import Bounds from "../Models/Bounds";
import {Overpass} from "./Osm/Overpass";
import Loc from "../Models/Loc";
import LayoutConfig from "../Customizations/JSON/LayoutConfig";
import FeatureSource from "./Actors/FeatureSource";
import {UIEventSource} from "../UIEventSource";
import Loc from "../../Models/Loc";
import {Or, TagsFilter} from "../Tags";
import LayoutConfig from "../../Customizations/JSON/LayoutConfig";
import {Overpass} from "../Osm/Overpass";
import Bounds from "../../Models/Bounds";
import FeatureSource from "../FeatureSource/FeatureSource";
export default class UpdateFromOverpass implements FeatureSource{
/**
* The last loaded features of the geojson
*/
public readonly features: UIEventSource<any[]> = new UIEventSource<any[]>(undefined);
public readonly features: UIEventSource<{feature:any, freshness: Date}[]> = new UIEventSource<any[]>(undefined);
/**
* The time of updating according to Overpass
*/
public readonly freshness:UIEventSource<Date> = new UIEventSource<Date>(undefined);
public readonly sufficientlyZoomed: UIEventSource<boolean>;
public readonly runningQuery: UIEventSource<boolean> = new UIEventSource<boolean>(false);
@ -142,8 +139,7 @@ export default class UpdateFromOverpass implements FeatureSource{
function (data, date) {
self._previousBounds.get(z).push(queryBounds);
self.retries.setData(0);
self.freshness.setData(date);
self.features.setData(data.features);
self.features.setData(data.features.map(f => ({feature: f, freshness: date})));
self.runningQuery.setData(false);
},
function (reason) {

5
Logic/Actors/FeatureSource.ts → Logic/FeatureSource/FeatureSource.ts
View file

@ -1,8 +1,5 @@
import {UIEventSource} from "../UIEventSource";
export default interface FeatureSource {
features : UIEventSource<any[]>;
freshness: UIEventSource<Date>;
features: UIEventSource<{feature: any, freshness: Date}[]>;
}

40
Logic/FeatureSource/FeatureSourceMerger.ts Normal file
View file

@ -0,0 +1,40 @@
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
export default class FeatureSourceMerger implements FeatureSource {
public features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{feature: any; freshness: Date}[]>([]);
private readonly _sources: FeatureSource[];
constructor(sources: FeatureSource[]) {
this._sources = sources;
const self = this;
for (const source of sources) {
source.features.addCallback(() => self.Update());
}
}
private Update() {
let all = {}; // Mapping 'id' -> {feature, freshness}
for (const source of this._sources) {
for (const f of source.features.data) {
const id = f.feature.properties.id+f.feature.geometry.type;
const oldV = all[id];
if(oldV === undefined){
all[id] = f;
}else{
if(oldV.freshness < f.freshness){
all[id]=f;
}
}
}
}
const newList = [];
for (const id in all) {
newList.push(all[id]);
}
this.features.setData(newList);
}
}

52
Logic/FeatureSource/FilteringFeatureSource.ts Normal file
View file

@ -0,0 +1,52 @@
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import LayerConfig from "../../Customizations/JSON/LayerConfig";
import Loc from "../../Models/Loc";
export default class FilteringFeatureSource implements FeatureSource {
public features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]);
constructor(layers: {
isDisplayed: UIEventSource<boolean>,
layerDef: LayerConfig
}[],
location: UIEventSource<Loc>,
upstream: FeatureSource) {
const layerDict = {};
const self = this;
function update() {
console.log("UPdating...")
const features: { feature: any, freshness: Date }[] = upstream.features.data;
const newFeatures = features.filter(f => {
const layerId = f.feature.properties._matching_layer_id;
if (layerId === undefined) {
console.error(f)
throw "feature._matching_layer_id is undefined"
}
const layer: {
isDisplayed: UIEventSource<boolean>,
layerDef: LayerConfig
} = layerDict[layerId];
if (layer === undefined) {
throw "No layer found with id " + layerId;
}
return layer.isDisplayed.data && (layer.layerDef.minzoom <= location.data.zoom);
});
self.features.setData(newFeatures);
}
for (const layer of layers) {
layerDict[layer.layerDef.id] = layer;
layer.isDisplayed.addCallback(update)
}
upstream.features.addCallback(update);
location.map(l => l.zoom).addCallback(update);
}
}

86
Logic/FeatureSource/NoOverlapSource.ts Normal file
View file

@ -0,0 +1,86 @@
import LayerConfig from "../../Customizations/JSON/LayerConfig";
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import {GeoOperations} from "../GeoOperations";
/**
* The no overlap source takes a featureSource and applies a filter on it.
* First, it'll figure out for each feature to which layer it belongs
* Then, it'll check any feature of any 'lower' layer
*/
export default class NoOverlapSource {
features: UIEventSource<{ feature: any, freshness: Date }[]> = new UIEventSource<{ feature: any, freshness: Date }[]>([]);
constructor(layers: {
layerDef: LayerConfig
}[],
upstream: FeatureSource) {
const layerDict = {};
let noOverlapRemoval = true;
const layerIds = []
for (const layer of layers) {
layerDict[layer.layerDef.id] = layer;
layerIds.push(layer.layerDef.id);
if ((layer.layerDef.hideUnderlayingFeaturesMinPercentage ?? 0) !== 0) {
noOverlapRemoval = false;
}
}
if (noOverlapRemoval) {
this.features = upstream.features;
return;
}
this.features = upstream.features.map(
features => {
if (features === undefined) {
return;
}
// There is overlap removal active
// We partition all the features with their respective layerIDs
const partitions = {};
for (const layerId of layerIds) {
partitions[layerId] = []
}
for (const feature of features) {
partitions[feature.feature.properties._matching_layer_id].push(feature);
}
// With this partitioning in hand, we run over every layer and remove every underlying feature if needed
for (let i = 0; i < layerIds.length; i++) {
let layerId = layerIds[i];
const percentage = layerDict[layerId].layerDef.hideUnderlayingFeaturesMinPercentage ?? 0;
if (percentage === 0) {
// We don't have to remove underlying features!
continue;
}
const guardPartition = partitions[layerId];
for (let j = i + 1; j < layerIds.length; j++) {
let layerJd = layerIds[j];
let partitionToShrink: { feature: any, freshness: Date }[] = partitions[layerJd];
let newPartition = [];
for (const mightBeDeleted of partitionToShrink) {
const doesOverlap = GeoOperations.featureIsContainedInAny(
mightBeDeleted.feature,
guardPartition.map(f => f.feature),
percentage
);
if(!doesOverlap){
newPartition.push(mightBeDeleted);
}
}
partitions[layerJd] = newPartition;
}
}
// At last, we create the actual new features
let newFeatures: { feature: any, freshness: Date }[] = [];
for (const layerId of layerIds) {
newFeatures = newFeatures.concat(partitions[layerId]);
}
return newFeatures;
});
}
}

24
Logic/FeatureSource/RememberingSource.ts Normal file
View file

@ -0,0 +1,24 @@
/**
* Every previously added point is remembered, but new points are added
*/
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
export default class RememberingSource implements FeatureSource{
features: UIEventSource<{feature: any, freshness: Date}[]> = new UIEventSource<{feature: any, freshness: Date}[]>([]);
constructor(source: FeatureSource) {
const self = this;
source.features.addCallbackAndRun(features => {
if(features === undefined){
return;
}
const ids = new Set<string>( features.map(f => f.feature.properties.id+f.feature.geometry.type));
const newList = features.concat(
self.features.data.filter(old => !ids.has(old.feature.properties.id+old.feature.geometry.type))
)
self.features.setData(newList);
})
}
}

66
Logic/FeatureSource/WayHandlingApplyingFeatureSource.ts Normal file
View file

@ -0,0 +1,66 @@
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import LayerConfig from "../../Customizations/JSON/LayerConfig";
import {GeoOperations} from "../GeoOperations";
export default class WayHandlingApplyingFeatureSource implements FeatureSource {
features: UIEventSource<{ feature: any; freshness: Date }[]>;
constructor(layers: {
layerDef: LayerConfig
}[],
upstream: FeatureSource) {
const layerDict = {};
let allDefaultWayHandling = true;
for (const layer of layers) {
layerDict[layer.layerDef.id] = layer;
if (layer.layerDef.wayHandling !== LayerConfig.WAYHANDLING_DEFAULT) {
allDefaultWayHandling = false;
}
}
if (allDefaultWayHandling) {
this.features = upstream.features;
return;
}
this.features = upstream.features.map(
features => {
if(features === undefined){
return;
}
const newFeatures: { feature: any, freshness: Date }[] = [];
for (const f of features) {
const feat = f.feature;
const layerId = feat.properties._matching_layer_id;
const layer: LayerConfig = layerDict[layerId].layerDef;
if (layer === undefined) {
throw "No layer found with id " + layerId;
}
if(layer.wayHandling === LayerConfig.WAYHANDLING_DEFAULT){
newFeatures.push(f);
continue;
}
if (feat.geometry.type === "Point") {
newFeatures.push(f);
// it is a point, nothing to do here
continue;
}
const centerPoint = GeoOperations.centerpoint(feat);
newFeatures.push({feature: centerPoint, freshness: f.freshness});
if(layer.wayHandling === LayerConfig.WAYHANDLING_CENTER_AND_WAY){
newFeatures.push(f);
}
}
return newFeatures;
}
);
}
}

130
Logic/FilteredLayer.ts
View file

@ -10,29 +10,20 @@ import Hash from "./Web/Hash";
import LazyElement from "../UI/Base/LazyElement";
/***
* A filtered layer is a layer which offers a 'set-data' function
* It is initialized with a tagfilter.
*
* When geojson-data is given to 'setData', all the geojson matching the filter, is rendered on this layer.
* If it is not rendered, it is returned in a 'leftOver'-geojson; which can be consumed by the next layer.
*
* This also makes sure that no objects are rendered twice if they are applicable on two layers
*
*/
export class FilteredLayer {
public readonly name: string | UIElement;
public readonly filters: TagsFilter;
public readonly isDisplayed: UIEventSource<boolean> = new UIEventSource(true);
public readonly layerDef: LayerConfig;
private readonly combinedIsDisplayed: UIEventSource<boolean>;
private readonly filters: TagsFilter;
private readonly _maxAllowedOverlap: number;
/** The featurecollection from overpass
*/
private _dataFromOverpass: any[];
/** List of new elements, geojson features
*/
private _newElements = [];
/**
* The leaflet layer object which should be removed on rerendering
*/
@ -51,22 +42,7 @@ export class FilteredLayer {
this.name = name;
this.filters = layerDef.overpassTags;
this._maxAllowedOverlap = layerDef.hideUnderlayingFeaturesMinPercentage;
const self = this;
this.combinedIsDisplayed = this.isDisplayed.map<boolean>(isDisplayed => {
return isDisplayed && State.state.locationControl.data.zoom >= self.layerDef.minzoom
},
[State.state.locationControl]
);
this.combinedIsDisplayed.addCallback(function (isDisplayed) {
const map = State.state.leafletMap.data;
if (self._geolayer !== undefined && self._geolayer !== null) {
if (isDisplayed) {
self._geolayer.addTo(map);
} else {
map.removeLayer(self._geolayer);
}
}
})
}
/**
@ -88,29 +64,11 @@ export class FilteredLayer {
}
this.RenderLayer(selfFeatures)
const notShadowed = [];
for (const feature of leftoverFeatures) {
if (this._maxAllowedOverlap !== undefined && this._maxAllowedOverlap > 0) {
if (GeoOperations.featureIsContainedInAny(feature, selfFeatures, this._maxAllowedOverlap)) {
// This feature is filtered away
continue;
}
}
notShadowed.push(feature);
}
return notShadowed;
return leftoverFeatures;
}
public AddNewElement(element) {
this._newElements.push(element);
this.RenderLayer(this._dataFromOverpass); // Update the layer
}
private RenderLayer(features) {
private RenderLayer(features: any[]) {
if (this._geolayer !== undefined && this._geolayer !== null) {
// Remove the old geojson layer from the map - we'll reshow all the elements later on anyway
@ -118,12 +76,9 @@ export class FilteredLayer {
}
// We fetch all the data we have to show:
let fusedFeatures = this.ApplyWayHandling(this.FuseData(features));
// And we copy some features as points - if needed
const data = {
type: "FeatureCollection",
features: fusedFeatures
features: features
}
let self = this;
@ -144,13 +99,7 @@ export class FilteredLayer {
radius: 25,
color: style.color
});
} else if (style.icon.iconUrl.startsWith("$circle")) {
marker = L.circle(latLng, {
radius: 25,
color: style.color
});
} else {
style.icon.html.ListenTo(self.isDisplayed)
marker = L.marker(latLng, {
icon: L.divIcon({
html: style.icon.html.Render(),
@ -206,72 +155,9 @@ export class FilteredLayer {
}
});
if (this.combinedIsDisplayed.data) {
this._geolayer.addTo(State.state.leafletMap.data);
}
this._geolayer.addTo(State.state.leafletMap.data);
}
private ApplyWayHandling(fusedFeatures: any[]) {
if (this.layerDef.wayHandling === LayerConfig.WAYHANDLING_DEFAULT) {
// We don't have to do anything special
return fusedFeatures;
}
// We have to convert all the ways into centerpoints
const existingPoints = [];
const newPoints = [];
const existingWays = [];
for (const feature of fusedFeatures) {
if (feature.geometry.type === "Point") {
existingPoints.push(feature);
continue;
}
existingWays.push(feature);
const centerPoint = GeoOperations.centerpoint(feature);
newPoints.push(centerPoint);
}
fusedFeatures = existingPoints.concat(newPoints);
if (this.layerDef.wayHandling === LayerConfig.WAYHANDLING_CENTER_AND_WAY) {
fusedFeatures = fusedFeatures.concat(existingWays)
}
return fusedFeatures;
}
//*Fuses the old and the new datasets*/
private FuseData(data: any[]) {
const oldData = this._dataFromOverpass ?? [];
// We keep track of all the ids that are freshly loaded in order to avoid adding duplicates
const idsFromOverpass: Set<number> = new Set<number>();
// A list of all the features to show
const fusedFeatures = [];
// First, we add all the fresh data:
for (const feature of data) {
idsFromOverpass.add(feature.properties.id);
fusedFeatures.push(feature);
}
// Now we add all the stale data
for (const feature of oldData) {
if (idsFromOverpass.has(feature.properties.id)) {
continue; // Feature already loaded and a fresher version is available
}
idsFromOverpass.add(feature.properties.id);
fusedFeatures.push(feature);
}
this._dataFromOverpass = fusedFeatures;
for (const feature of this._newElements) {
if (!idsFromOverpass.has(feature.properties.id)) {
// This element is not yet uploaded or not yet visible in overpass
// We include it in the layer
fusedFeatures.push(feature);
}
}
return fusedFeatures;
}
}

115
Logic/Osm/Changes.ts
View file

@ -1,71 +1,25 @@
/**
* Handles all changes made to OSM.
* Needs an authenticator via OsmConnection
*/
import {OsmNode, OsmObject} from "./OsmObject";
import {And, Tag, TagsFilter} from "../Tags";
import State from "../../State";
import {Utils} from "../../Utils";
import {UIEventSource} from "../UIEventSource";
import Constants from "../../Models/Constants";
import FeatureSource from "../FeatureSource/FeatureSource";
export class Changes {
/**
* Handles all changes made to OSM.
* Needs an authenticator via OsmConnection
*/
export class Changes implements FeatureSource{
private static _nextId = -1; // New assined ID's are negative
addTag(elementId: string, tagsFilter: TagsFilter,
tags?: UIEventSource<any>) {
const changes = this.tagToChange(tagsFilter);
if (changes.length == 0) {
return;
}
const eventSource = tags ?? State.state?.allElements.getEventSourceById(elementId);
const elementTags = eventSource.data;
const pending : {elementId:string, key: string, value: string}[] = [];
for (const change of changes) {
if (elementTags[change.k] !== change.v) {
elementTags[change.k] = change.v;
pending.push({elementId: elementTags.id, key: change.k, value: change.v});
}
}
if(pending.length === 0){
return;
}
console.log("Sending ping",eventSource)
eventSource.ping();
this.uploadAll([], pending);
}
private tagToChange(tagsFilter: TagsFilter) {
let changes: { k: string, v: string }[] = [];
if (tagsFilter instanceof Tag) {
const tag = tagsFilter as Tag;
if (typeof tag.value !== "string") {
throw "Invalid value"
}
return [this.checkChange(tag.key, tag.value)];
}
if (tagsFilter instanceof And) {
const and = tagsFilter as And;
for (const tag of and.and) {
changes = changes.concat(this.tagToChange(tag));
}
return changes;
}
console.log("Unsupported tagsfilter element to addTag", tagsFilter);
throw "Unsupported tagsFilter element";
}
public features = new UIEventSource<{feature: any, freshness: Date}[]>([]);
private static _nextId = -1; // Newly assigned ID's are negative
/**
* Adds a change to the pending changes
* @param elementId
* @param key
* @param value
*/
private checkChange(key: string, value: string): { k: string, v: string } {
private static checkChange(key: string, value: string): { k: string, v: string } {
if (key === undefined || key === null) {
console.log("Invalid key");
return undefined;
@ -85,12 +39,35 @@ export class Changes {
return {k: key, v: value};
}
addTag(elementId: string, tagsFilter: TagsFilter,
tags?: UIEventSource<any>) {
const changes = this.tagToChange(tagsFilter);
if (changes.length == 0) {
return;
}
const eventSource = tags ?? State.state?.allElements.getEventSourceById(elementId);
const elementTags = eventSource.data;
const pending: { elementId: string, key: string, value: string }[] = [];
for (const change of changes) {
if (elementTags[change.k] !== change.v) {
elementTags[change.k] = change.v;
pending.push({elementId: elementTags.id, key: change.k, value: change.v});
}
}
if (pending.length === 0) {
return;
}
console.log("Sending ping", eventSource)
eventSource.ping();
this.uploadAll([], pending);
}
/**
* Create a new node element at the given lat/long.
* An internal OsmObject is created to upload later on, a geojson represention is returned.
* Note that the geojson version shares the tags (properties) by pointer, but has _no_ id in properties
*/
createElement(basicTags:Tag[], lat: number, lon: number) {
public createElement(basicTags: Tag[], lat: number, lon: number) {
console.log("Creating a new element with ", basicTags)
const osmNode = new OsmNode(Changes._nextId);
Changes._nextId--;
@ -113,6 +90,9 @@ export class Changes {
}
}
this.features.data.push({feature:geojson, freshness: new Date()});
this.features.ping();
// The basictags are COPIED, the id is included in the properties
// The tags are not yet written into the OsmObject, but this is applied onto a
const changes = [];
@ -128,6 +108,27 @@ export class Changes {
return geojson;
}
private tagToChange(tagsFilter: TagsFilter) {
let changes: { k: string, v: string }[] = [];
if (tagsFilter instanceof Tag) {
const tag = tagsFilter as Tag;
if (typeof tag.value !== "string") {
throw "Invalid value"
}
return [Changes.checkChange(tag.key, tag.value)];
}
if (tagsFilter instanceof And) {
const and = tagsFilter as And;
for (const tag of and.and) {
changes = changes.concat(this.tagToChange(tag));
}
return changes;
}
console.log("Unsupported tagsfilter element to addTag", tagsFilter);
throw "Unsupported tagsFilter element";
}
private uploadChangesWithLatestVersions(
knownElements, newElements: OsmObject[], pending: { elementId: string; key: string; value: string }[]) {