Osmwithspace/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Huge refactoring of the feature pipeline, WIP

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-09-20 17:14:55 +02:00
parent 7793297348
commit 973b5d8bbe
25 changed files with 522 additions and 591 deletions
Download patch file Download diff file Expand all files Collapse all files

0
Logic/Actors/LayerResetter.ts → Logic/Actors/BackgroundLayerResetter.ts
View file

35
Logic/FeatureSource/Actors/LocalStorageSaverActor.ts Normal file
View file

@ -0,0 +1,35 @@
import {FeatureSourceForLayer} from "./FeatureSource";
import {Utils} from "../../Utils";
/***
* Saves all the features that are passed in to localstorage, so they can be retrieved on the next run
*
* Technically, more an Actor then a featuresource, but it fits more neatly this ay
*/
export default class LocalStorageSaverActor {
public static readonly storageKey: string = "cached-features";
constructor(source: FeatureSourceForLayer, x: number, y: number, z: number) {
source.features.addCallbackAndRunD(features => {
const index = Utils.tile_index(z, x, y)
const key = `${LocalStorageSaverActor.storageKey}-${source.layer.layerDef.id}-${index}`
const now = new Date().getTime()
if (features.length == 0) {
return;
}
try {
localStorage.setItem(key, JSON.stringify(features));
console.log("Saved ", features.length, "elements to", key)
localStorage.setItem(key + "-time", JSON.stringify(now))
} catch (e) {
console.warn("Could not save the features to local storage:", e)
}
})
}
}

2
Logic/FeatureSource/RegisteringFeatureSource.ts → Logic/FeatureSource/Actors/RegisteringAllFromFeatureSourceActor.ts
View file

@ -2,7 +2,7 @@ import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource"; import {UIEventSource} from "../UIEventSource";
import State from "../../State"; import State from "../../State";
export default class RegisteringFeatureSource implements FeatureSource { export default class RegisteringAllFromFeatureSourceActor {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>; public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name; public readonly name;

64
Logic/FeatureSource/FeatureDuplicatorPerLayer.ts
View file

@ -1,64 +0,0 @@
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import FilteredLayer from "../../Models/FilteredLayer";
/**
* In some rare cases, some elements are shown on multiple layers (when 'passthrough' is enabled)
* If this is the case, multiple objects with a different _matching_layer_id are generated.
* In any case, this featureSource marks the objects with _matching_layer_id
*/
export default class FeatureDuplicatorPerLayer implements FeatureSource {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name;
constructor(layers: UIEventSource<FilteredLayer[]>, upstream: FeatureSource) {
this.name = "FeatureDuplicator of " + upstream.name;
this.features = upstream.features.map(features => {
const newFeatures: { feature: any, freshness: Date }[] = [];
if (features === undefined) {
return newFeatures;
}
for (const f of features) {
if (f.feature._matching_layer_id) {
// Already matched previously
// We simply add it
newFeatures.push(f);
continue;
}
let foundALayer = false;
for (const layer of layers.data) {
if (layer.layerDef.source.osmTags.matchesProperties(f.feature.properties)) {
foundALayer = true;
if (layer.layerDef.passAllFeatures) {
// We copy the feature; the "properties" field is kept identical though!
// Keeping "properties" identical is needed, as it might break the 'allElementStorage' otherwise
const newFeature = {
geometry: f.feature.geometry,
id: f.feature.id,
type: f.feature.type,
properties: f.feature.properties,
_matching_layer_id: layer.layerDef.id
}
newFeatures.push({feature: newFeature, freshness: f.freshness});
} else {
// If not 'passAllFeatures', we are done
f.feature._matching_layer_id = layer.layerDef.id;
newFeatures.push(f);
break;
}
}
}
}
return newFeatures;
})
}
}

162
Logic/FeatureSource/FilteringFeatureSource.ts
View file

@ -1,162 +0,0 @@
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import Loc from "../../Models/Loc";
import Hash from "../Web/Hash";
import {TagsFilter} from "../Tags/TagsFilter";
import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
export default class FilteringFeatureSource implements FeatureSource {
public features: UIEventSource<{ feature: any; freshness: Date }[]> =
new UIEventSource<{ feature: any; freshness: Date }[]>([]);
public readonly name = "FilteringFeatureSource";
constructor(
layers: UIEventSource<{
isDisplayed: UIEventSource<boolean>;
layerDef: LayerConfig;
appliedFilters: UIEventSource<TagsFilter>;
}[]>,
location: UIEventSource<Loc>,
selectedElement: UIEventSource<any>,
upstream: FeatureSource
) {
const self = this;
function update() {
const layerDict = {};
if (layers.data.length == 0) {
console.warn("No layers defined!");
return;
}
for (const layer of layers.data) {
const prev = layerDict[layer.layerDef.id]
if (prev !== undefined) {
// We have seen this layer before!
// We prefer the one which has a name
if (layer.layerDef.name === undefined) {
// This one is hidden, so we skip it
console.log("Ignoring layer selection from ", layer)
continue;
}
}
layerDict[layer.layerDef.id] = layer;
}
const features: { feature: any; freshness: Date }[] =
upstream.features.data;
const missingLayers = new Set<string>();
const newFeatures = features.filter((f) => {
const layerId = f.feature._matching_layer_id;
if (
selectedElement.data?.id === f.feature.id ||
f.feature.id === Hash.hash.data) {
// This is the selected object - it gets a free pass even if zoom is not sufficient or it is filtered away
return true;
}
if (layerId === undefined) {
return false;
}
const layer: {
isDisplayed: UIEventSource<boolean>;
layerDef: LayerConfig;
appliedFilters: UIEventSource<TagsFilter>;
} = layerDict[layerId];
if (layer === undefined) {
missingLayers.add(layerId);
return false;
}
const isShown = layer.layerDef.isShown;
const tags = f.feature.properties;
if (isShown.IsKnown(tags)) {
const result = layer.layerDef.isShown.GetRenderValue(
f.feature.properties
).txt;
if (result !== "yes") {
return false;
}
}
const tagsFilter = layer.appliedFilters.data;
if (tagsFilter) {
if (!tagsFilter.matchesProperties(f.feature.properties)) {
// Hidden by the filter on the layer itself - we want to hide it no matter wat
return false;
}
}
if (!FilteringFeatureSource.showLayer(layer, location)) {
// The layer itself is either disabled or hidden due to zoom constraints
// We should return true, but it might still match some other layer
return false;
}
return true;
});
self.features.setData(newFeatures);
if (missingLayers.size > 0) {
console.error(
"Some layers were not found: ",
Array.from(missingLayers)
);
}
}
upstream.features.addCallback(() => {
update();
});
location
.map((l) => {
// We want something that is stable for the shown layers
const displayedLayerIndexes = [];
for (let i = 0; i < layers.data.length; i++) {
const layer = layers.data[i];
if (l.zoom < layer.layerDef.minzoom) {
continue;
}
if (!layer.isDisplayed.data) {
continue;
}
displayedLayerIndexes.push(i);
}
return displayedLayerIndexes.join(",");
})
.addCallback(() => {
update();
});
layers.addCallback(update);
const registered = new Set<UIEventSource<boolean>>();
layers.addCallbackAndRun((layers) => {
for (const layer of layers) {
if (registered.has(layer.isDisplayed)) {
continue;
}
registered.add(layer.isDisplayed);
layer.isDisplayed.addCallback(() => update());
layer.appliedFilters.addCallback(() => update());
}
});
update();
}
private static showLayer(
layer: {
isDisplayed: UIEventSource<boolean>;
layerDef: LayerConfig;
},
location: UIEventSource<Loc>
) {
return (
layer.isDisplayed.data &&
layer.layerDef.minzoomVisible <= location.data.zoom
);
}
}

207
Logic/FeatureSource/GeoJsonSource.ts
View file

@ -1,207 +0,0 @@
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import Loc from "../../Models/Loc";
import State from "../../State";
import {Utils} from "../../Utils";
import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
/**
* Fetches a geojson file somewhere and passes it along
*/
export default class GeoJsonSource implements FeatureSource {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name;
public readonly isOsmCache: boolean
private onFail: ((errorMsg: any, url: string) => void) = undefined;
private readonly layerId: string;
private readonly seenids: Set<string> = new Set<string>()
private constructor(locationControl: UIEventSource<Loc>,
flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig },
onFail?: ((errorMsg: any) => void)) {
this.layerId = flayer.layerDef.id;
let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id);
this.name = "GeoJsonSource of " + url;
const zoomLevel = flayer.layerDef.source.geojsonZoomLevel;
this.isOsmCache = flayer.layerDef.source.isOsmCacheLayer;
this.features = new UIEventSource<{ feature: any; freshness: Date }[]>([])
if (zoomLevel === undefined) {
// This is a classic, static geojson layer
if (onFail === undefined) {
onFail = _ => {
}
}
this.onFail = onFail;
this.LoadJSONFrom(url)
} else {
this.ConfigureDynamicLayer(url, zoomLevel, locationControl, flayer)
}
}
/**
* Merges together the layers which have the same source
* @param flayers
* @param locationControl
* @constructor
*/
public static ConstructMultiSource(flayers: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }[], locationControl: UIEventSource<Loc>): GeoJsonSource[] {
const flayersPerSource = new Map<string, { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }[]>();
for (const flayer of flayers) {
const url = flayer.layerDef.source.geojsonSource?.replace(/{layer}/g, flayer.layerDef.id)
if (url === undefined) {
continue;
}
if (!flayersPerSource.has(url)) {
flayersPerSource.set(url, [])
}
flayersPerSource.get(url).push(flayer)
}
const sources: GeoJsonSource[] = []
flayersPerSource.forEach((flayers, key) => {
if (flayers.length == 1) {
sources.push(new GeoJsonSource(locationControl, flayers[0]));
return;
}
const zoomlevels = Utils.Dedup(flayers.map(flayer => "" + (flayer.layerDef.source.geojsonZoomLevel ?? "")))
if (zoomlevels.length > 1) {
throw "Multiple zoomlevels defined for same geojson source " + key
}
let isShown = new UIEventSource<boolean>(true, "IsShown for multiple layers: or of multiple values");
for (const flayer of flayers) {
flayer.isDisplayed.addCallbackAndRun(() => {
let value = false;
for (const flayer of flayers) {
value = flayer.isDisplayed.data || value;
}
isShown.setData(value);
});
}
const source = new GeoJsonSource(locationControl, {
isDisplayed: isShown,
layerDef: flayers[0].layerDef // We only care about the source info here
})
sources.push(source)
})
return sources;
}
private ConfigureDynamicLayer(url: string, zoomLevel: number, locationControl: UIEventSource<Loc>, flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }) {
// This is a dynamic template with a fixed zoom level
url = url.replace("{z}", "" + zoomLevel)
const loadedTiles = new Set<string>();
const self = this;
this.onFail = (msg, url) => {
console.warn(`Could not load geojson layer from`, url, "due to", msg)
loadedTiles.add(url); // We add the url to the 'loadedTiles' in order to not reload it in the future
}
const neededTiles = locationControl.map(
location => {
if (!flayer.isDisplayed.data) {
// No need to download! - the layer is disabled
return undefined;
}
if (location.zoom < flayer.layerDef.minzoom) {
// No need to download! - the layer is disabled
return undefined;
}
// Yup, this is cheating to just get the bounds here
const bounds = State.state.leafletMap.data?.getBounds()
if(bounds === undefined){
// We'll retry later
return undefined
}
const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const needed = Utils.MapRange(tileRange, (x, y) => {
return url.replace("{x}", "" + x).replace("{y}", "" + y);
})
return new Set<string>(needed);
}
, [flayer.isDisplayed, State.state.leafletMap]);
neededTiles.stabilized(250).addCallback((needed: Set<string>) => {
if (needed === undefined) {
return;
}
needed.forEach(neededTile => {
if (loadedTiles.has(neededTile)) {
return;
}
loadedTiles.add(neededTile)
self.LoadJSONFrom(neededTile)
})
})
}
private LoadJSONFrom(url: string) {
const eventSource = this.features;
const self = this;
Utils.downloadJson(url)
.then(json => {
if (json.elements === [] && json.remarks.indexOf("runtime error") > 0) {
self.onFail("Runtime error (timeout)", url)
return;
}
const time = new Date();
const newFeatures: { feature: any, freshness: Date } [] = []
let i = 0;
let skipped = 0;
for (const feature of json.features) {
const props = feature.presets
for (const key in props) {
if(typeof props[key] !== "string"){
props[key] = ""+props[key]
}
}
if (props.id === undefined) {
props.id = url + "/" + i;
feature.id = url + "/" + i;
i++;
}
if (self.seenids.has(props.id)) {
skipped++;
continue;
}
self.seenids.add(props.id)
let freshness: Date = time;
if (feature.properties["_last_edit:timestamp"] !== undefined) {
freshness = new Date(props["_last_edit:timestamp"])
}
newFeatures.push({feature: feature, freshness: freshness})
}
console.debug("Downloaded " + newFeatures.length + " new features and " + skipped + " already seen features from " + url);
if (newFeatures.length == 0) {
return;
}
eventSource.setData(eventSource.data.concat(newFeatures))
}).catch(msg => self.onFail(msg, url))
}
}

41
Logic/FeatureSource/LocalStorageSaver.ts
View file

@ -1,41 +0,0 @@
/***
* Saves all the features that are passed in to localstorage, so they can be retrieved on the next run
*
* Technically, more an Actor then a featuresource, but it fits more neatly this ay
*/
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig";
export default class LocalStorageSaver implements FeatureSource {
public static readonly storageKey: string = "cached-features";
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name = "LocalStorageSaver";
constructor(source: FeatureSource, layout: UIEventSource<LayoutConfig>) {
this.features = source.features;
this.features.addCallbackAndRunD(features => {
const now = new Date().getTime()
features = features.filter(f => layout.data.cacheTimeout > Math.abs(now - f.freshness.getTime()) / 1000)
if (features.length == 0) {
return;
}
try {
const key = LocalStorageSaver.storageKey + layout.data.id
localStorage.setItem(key, JSON.stringify(features));
console.log("Saved ", features.length, "elements to", key)
} catch (e) {
console.warn("Could not save the features to local storage:", e)
}
})
}
}

52
Logic/FeatureSource/MetaTaggingFeatureSource.ts
View file

@ -1,52 +0,0 @@
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import State from "../../State";
import Hash from "../Web/Hash";
import MetaTagging from "../MetaTagging";
export default class MetaTaggingFeatureSource implements FeatureSource {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>(undefined);
public readonly name;
/***
* Constructs a new metatagger which'll calculate various tags
* @param allFeaturesSource: A source where all the currently known features can be found - used to calculate overlaps etc
* @param source: the source of features that should get their metatag and which should be exported again
* @param updateTrigger
*/
constructor(allFeaturesSource: UIEventSource<{ feature: any; freshness: Date }[]>, source: FeatureSource, updateTrigger?: UIEventSource<any>) {
const self = this;
this.name = "MetaTagging of " + source.name
if (allFeaturesSource === undefined) {
throw ("UIEVentSource is undefined")
}
function update() {
const featuresFreshness = source.features.data
if (featuresFreshness === undefined) {
return;
}
featuresFreshness.forEach(featureFresh => {
const feature = featureFresh.feature;
if (Hash.hash.data === feature.properties.id) {
State.state.selectedElement.setData(feature);
}
})
MetaTagging.addMetatags(featuresFreshness,
allFeaturesSource,
State.state.knownRelations.data, State.state.layoutToUse.data.layers);
self.features.setData(featuresFreshness);
}
source.features.addCallbackAndRun(_ => update());
updateTrigger?.addCallback(_ => {
console.debug("Updating because of external call")
update();
})
}
}

87
Logic/FeatureSource/PerLayerFeatureSourceSplitter.ts Normal file
View file

@ -0,0 +1,87 @@
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import FilteredLayer from "../../Models/FilteredLayer";
import OverpassFeatureSource from "../Actors/OverpassFeatureSource";
import SimpleFeatureSource from "./SimpleFeatureSource";
/**
* In some rare cases, some elements are shown on multiple layers (when 'passthrough' is enabled)
* If this is the case, multiple objects with a different _matching_layer_id are generated.
* In any case, this featureSource marks the objects with _matching_layer_id
*/
export default class PerLayerFeatureSourceSplitter {
constructor(layers: UIEventSource<FilteredLayer[]>,
handleLayerData: (source: FeatureSource) => void,
upstream: OverpassFeatureSource) {
const knownLayers = new Map<string, FeatureSource>()
function update() {
const features = upstream.features.data;
if (features === undefined) {
return;
}
if(layers.data === undefined){
return;
}
// We try to figure out (for each feature) in which feature store it should be saved.
// Note that this splitter is only run when it is invoked by the overpass feature source, so we can't be sure in which layer it should go
const featuresPerLayer = new Map<string, { feature, freshness } []>();
function addTo(layer: FilteredLayer, feature: { feature, freshness }) {
const id = layer.layerDef.id
const list = featuresPerLayer.get(id)
if (list !== undefined) {
list.push(feature)
} else {
featuresPerLayer.set(id, [feature])
}
}
for (const f of features) {
for (const layer of layers.data) {
if (layer.layerDef.source.osmTags.matchesProperties(f.feature.properties)) {
// We have found our matching layer!
addTo(layer, f)
if (!layer.layerDef.passAllFeatures) {
// If not 'passAllFeatures', we are done for this feature
break;
}
}
}
}
// At this point, we have our features per layer as a list
// We assign them to the correct featureSources
for (const layer of layers.data) {
const id = layer.layerDef.id;
const features = featuresPerLayer.get(id)
if (features === undefined) {
// No such features for this layer
continue;
}
let featureSource = knownLayers.get(id)
if (featureSource === undefined) {
// Not yet initialized - now is a good time
featureSource = new SimpleFeatureSource(layer)
knownLayers.set(id, featureSource)
handleLayerData(featureSource)
}
featureSource.features.setData(features)
}
upstream.features.addCallbackAndRunD(_ => update())
layers.addCallbackAndRunD(_ => update())
}
layers.addCallbackAndRunD(_ => update())
upstream.features.addCallbackAndRunD(_ => update())
}
}

47
Logic/FeatureSource/FeatureSourceMerger.ts → Logic/FeatureSource/Sources/FeatureSourceMerger.ts
View file

@ -1,27 +1,44 @@
import FeatureSource from "./FeatureSource"; import FeatureSource, {FeatureSourceForLayer} from "./FeatureSource";
import {UIEventSource} from "../UIEventSource"; import {UIEventSource} from "../UIEventSource";
import FilteredLayer from "../../Models/FilteredLayer";
/** /**
* Merges features from different featureSources * Merges features from different featureSources for a single layer
* Uses the freshest feature available in the case multiple sources offer data with the same identifier * Uses the freshest feature available in the case multiple sources offer data with the same identifier
*/ */
export default class FeatureSourceMerger implements FeatureSource { export default class FeatureSourceMerger implements FeatureSourceForLayer {
public features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]); public features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]);
public readonly name; public readonly name;
private readonly _sources: FeatureSource[]; public readonly layer: FilteredLayer
private readonly _sources: UIEventSource<FeatureSource[]>;
constructor(sources: FeatureSource[]) { constructor(layer: FilteredLayer ,sources: UIEventSource<FeatureSource[]>) {
this._sources = sources; this._sources = sources;
this.name = "SourceMerger of (" + sources.map(s => s.name).join(", ") + ")" this.layer = layer;
this.name = "SourceMerger"
const self = this; const self = this;
for (let i = 0; i < sources.length; i++) {
let source = sources[i]; const handledSources = new Set<FeatureSource>();
source.features.addCallback(() => {
self.Update(); sources.addCallbackAndRunD(sources => {
}); let newSourceRegistered = false;
} for (let i = 0; i < sources.length; i++) {
this.Update(); let source = sources[i];
if (handledSources.has(source)) {
continue
}
handledSources.add(source)
newSourceRegistered = true
source.features.addCallback(() => {
self.Update();
});
if (newSourceRegistered) {
self.Update();
}
}
})
} }
private Update() { private Update() {
@ -34,7 +51,7 @@ export default class FeatureSourceMerger implements FeatureSource {
all.set(oldValue.feature.id + oldValue.feature._matching_layer_id, oldValue) all.set(oldValue.feature.id + oldValue.feature._matching_layer_id, oldValue)
} }
for (const source of this._sources) { for (const source of this._sources.data) {
if (source?.features?.data === undefined) { if (source?.features?.data === undefined) {
continue; continue;
} }
@ -64,7 +81,7 @@ export default class FeatureSourceMerger implements FeatureSource {
} }
const newList = []; const newList = [];
all.forEach((value, key) => { all.forEach((value, _) => {
newList.push(value) newList.push(value)
}) })
this.features.setData(newList); this.features.setData(newList);

101
Logic/FeatureSource/Sources/FilteringFeatureSource.ts Normal file
View file

@ -0,0 +1,101 @@
import {FeatureSourceForLayer} from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import Hash from "../Web/Hash";
import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
import FilteredLayer from "../../Models/FilteredLayer";
export default class FilteringFeatureSource implements FeatureSourceForLayer {
public features: UIEventSource<{ feature: any; freshness: Date }[]> =
new UIEventSource<{ feature: any; freshness: Date }[]>([]);
public readonly name = "FilteringFeatureSource";
public readonly layer: FilteredLayer;
constructor(
state: {
locationControl: UIEventSource<{ zoom: number }>,
selectedElement: UIEventSource<any>,
},
upstream: FeatureSourceForLayer
) {
const self = this;
this.layer = upstream.layer;
const layer = upstream.layer;
function update() {
const features: { feature: any; freshness: Date }[] = upstream.features.data;
const newFeatures = features.filter((f) => {
if (
state.selectedElement.data?.id === f.feature.id ||
f.feature.id === Hash.hash.data) {
// This is the selected object - it gets a free pass even if zoom is not sufficient or it is filtered away
return true;
}
const isShown = layer.layerDef.isShown;
const tags = f.feature.properties;
if (isShown.IsKnown(tags)) {
const result = layer.layerDef.isShown.GetRenderValue(
f.feature.properties
).txt;
if (result !== "yes") {
return false;
}
}
const tagsFilter = layer.appliedFilters.data;
if (tagsFilter) {
if (!tagsFilter.matchesProperties(f.feature.properties)) {
// Hidden by the filter on the layer itself - we want to hide it no matter wat
return false;
}
}
if (!FilteringFeatureSource.showLayer(layer, state.locationControl.data)) {
// The layer itself is either disabled or hidden due to zoom constraints
// We should return true, but it might still match some other layer
return false;
}
return true;
});
self.features.setData(newFeatures);
}
upstream.features.addCallback(() => {
update();
});
let isShown = state.locationControl.map((l) => FilteringFeatureSource.showLayer(layer, l),
[layer.isDisplayed])
isShown.addCallback(isShown => {
if (isShown) {
update();
} else {
self.features.setData([])
}
});
layer.appliedFilters.addCallback(_ => {
if(!isShown.data){
// Currently not shown.
// Note that a change in 'isSHown' will trigger an update as well, so we don't have to watch it another time
return;
}
update()
})
update();
}
private static showLayer(
layer: {
isDisplayed: UIEventSource<boolean>;
layerDef: LayerConfig;
},
location: { zoom: number }) {
return layer.isDisplayed.data &&
layer.layerDef.minzoomVisible <= location.zoom;
}
}

95
Logic/FeatureSource/Sources/GeoJsonSource.ts Normal file
View file

@ -0,0 +1,95 @@
import {FeatureSourceForLayer} from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import {Utils} from "../../Utils";
import FilteredLayer from "../../Models/FilteredLayer";
import {control} from "leaflet";
/**
* Fetches a geojson file somewhere and passes it along
*/
export default class GeoJsonSource implements FeatureSourceForLayer {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name;
public readonly isOsmCache: boolean
private onFail: ((errorMsg: any, url: string) => void) = undefined;
private readonly seenids: Set<string> = new Set<string>()
public readonly layer: FilteredLayer;
public constructor(flayer: FilteredLayer,
zxy?: [number, number, number]) {
if (flayer.layerDef.source.geojsonZoomLevel !== undefined && zxy === undefined) {
throw "Dynamic layers are not supported. Use 'DynamicGeoJsonTileSource instead"
}
this.layer = flayer;
let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id);
if (zxy !== undefined) {
url = url
.replace('{z}', "" + zxy[0])
.replace('{x}', "" + zxy[1])
.replace('{y}', "" + zxy[2])
}
this.name = "GeoJsonSource of " + url;
this.isOsmCache = flayer.layerDef.source.isOsmCacheLayer;
this.features = new UIEventSource<{ feature: any; freshness: Date }[]>([])
this.LoadJSONFrom(url)
}
private LoadJSONFrom(url: string) {
const eventSource = this.features;
const self = this;
Utils.downloadJson(url)
.then(json => {
if (json.elements === [] && json.remarks.indexOf("runtime error") > 0) {
self.onFail("Runtime error (timeout)", url)
return;
}
const time = new Date();
const newFeatures: { feature: any, freshness: Date } [] = []
let i = 0;
let skipped = 0;
for (const feature of json.features) {
const props = feature.properties
for (const key in props) {
if (typeof props[key] !== "string") {
props[key] = "" + props[key]
}
}
if (props.id === undefined) {
props.id = url + "/" + i;
feature.id = url + "/" + i;
i++;
}
if (self.seenids.has(props.id)) {
skipped++;
continue;
}
self.seenids.add(props.id)
let freshness: Date = time;
if (feature.properties["_last_edit:timestamp"] !== undefined) {
freshness = new Date(props["_last_edit:timestamp"])
}
newFeatures.push({feature: feature, freshness: freshness})
}
console.debug("Downloaded " + newFeatures.length + " new features and " + skipped + " already seen features from " + url);
if (newFeatures.length == 0) {
return;
}
eventSource.setData(eventSource.data.concat(newFeatures))
}).catch(msg => console.error("Could not load geojon layer", url, "due to", msg))
}
}

4
Logic/FeatureSource/LocalStorageSource.ts → Logic/FeatureSource/Sources/LocalStorageSource.ts
View file

@ -1,6 +1,6 @@
import FeatureSource from "./FeatureSource"; import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource"; import {UIEventSource} from "../UIEventSource";
import LocalStorageSaver from "./LocalStorageSaver"; import LocalStorageSaverActor from "./LocalStorageSaverActor";
import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig"; import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig";
export default class LocalStorageSource implements FeatureSource { export default class LocalStorageSource implements FeatureSource {
@ -9,7 +9,7 @@ export default class LocalStorageSource implements FeatureSource {
constructor(layout: UIEventSource<LayoutConfig>) { constructor(layout: UIEventSource<LayoutConfig>) {
this.features = new UIEventSource<{ feature: any; freshness: Date }[]>([]) this.features = new UIEventSource<{ feature: any; freshness: Date }[]>([])
const key = LocalStorageSaver.storageKey + layout.data.id const key = LocalStorageSaverActor.storageKey + layout.data.id
layout.addCallbackAndRun(_ => { layout.addCallbackAndRun(_ => {
try { try {
const fromStorage = localStorage.getItem(key); const fromStorage = localStorage.getItem(key);

27
Logic/FeatureSource/OsmApiFeatureSource.ts → Logic/FeatureSource/Sources/OsmApiFeatureSource.ts
View file

@ -4,7 +4,6 @@ import {OsmObject} from "../Osm/OsmObject";
import {Utils} from "../../Utils"; import {Utils} from "../../Utils";
import Loc from "../../Models/Loc"; import Loc from "../../Models/Loc";
import FilteredLayer from "../../Models/FilteredLayer"; import FilteredLayer from "../../Models/FilteredLayer";
import Constants from "../../Models/Constants";
export default class OsmApiFeatureSource implements FeatureSource { export default class OsmApiFeatureSource implements FeatureSource {
@ -15,19 +14,23 @@ export default class OsmApiFeatureSource implements FeatureSource {
leafletMap: UIEventSource<any>; leafletMap: UIEventSource<any>;
locationControl: UIEventSource<Loc>, filteredLayers: UIEventSource<FilteredLayer[]>}; locationControl: UIEventSource<Loc>, filteredLayers: UIEventSource<FilteredLayer[]>};
constructor(minZoom = undefined, state: {locationControl: UIEventSource<Loc>, filteredLayers: UIEventSource<FilteredLayer[]>, leafletMap: UIEventSource<any>}) { constructor(state: {locationControl: UIEventSource<Loc>, filteredLayers: UIEventSource<FilteredLayer[]>, leafletMap: UIEventSource<any>,
overpassMaxZoom: UIEventSource<number>}) {
this._state = state; this._state = state;
if(minZoom !== undefined){ const self = this;
function update(){
const minZoom = state.overpassMaxZoom.data;
const location = state.locationControl.data
if(minZoom === undefined || location === undefined){
return;
}
if(minZoom < 14){ if(minZoom < 14){
throw "MinZoom should be at least 14 or higher, OSM-api won't work otherwise" throw "MinZoom should be at least 14 or higher, OSM-api won't work otherwise"
} }
const self = this; if(location.zoom > minZoom){
state.locationControl.addCallbackAndRunD(location => { return;
if(location.zoom > minZoom){ }
return; self.loadArea()
}
self.loadArea()
})
} }
} }
@ -59,10 +62,6 @@ export default class OsmApiFeatureSource implements FeatureSource {
if (disabledLayers.length > 0) { if (disabledLayers.length > 0) {
return false; return false;
} }
const loc = this._state.locationControl.data;
if (loc.zoom < Constants.useOsmApiAt) {
return false;
}
if (this._state.leafletMap.data === undefined) { if (this._state.leafletMap.data === undefined) {
return false; // Not yet inited return false; // Not yet inited
} }

18
Logic/FeatureSource/RememberingSource.ts → Logic/FeatureSource/Sources/RememberingSource.ts
View file

@ -1,12 +1,14 @@
/**
* Every previously added point is remembered, but new points are added import FeatureSource, {FeatureSourceForLayer} from "./FeatureSource";
*/
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource"; import {UIEventSource} from "../UIEventSource";
import FilteredLayer from "../../Models/FilteredLayer";
/**
* Every previously added point is remembered, but new points are added.
* Data coming from upstream will always overwrite a previous value
*/
export default class RememberingSource implements FeatureSource { export default class RememberingSource implements FeatureSource {
public readonly features: UIEventSource<{ feature: any, freshness: Date }[]>;
public readonly features: UIEventSource<{ feature: any, freshness: Date }[]>;
public readonly name; public readonly name;
constructor(source: FeatureSource) { constructor(source: FeatureSource) {
@ -20,9 +22,9 @@ export default class RememberingSource implements FeatureSource {
} }
// Then new ids // Then new ids
const ids = new Set<string>(features.map(f => f.feature.properties.id + f.feature.geometry.type + f.feature._matching_layer_id)); const ids = new Set<string>(features.map(f => f.feature.properties.id + f.feature.geometry.type));
// the old data // the old data
const oldData = oldFeatures.filter(old => !ids.has(old.feature.properties.id + old.feature.geometry.type + old.feature._matching_layer_id)) const oldData = oldFeatures.filter(old => !ids.has(old.feature.properties.id + old.feature.geometry.type))
return [...features, ...oldData]; return [...features, ...oldData];
}) })
} }

16
Logic/FeatureSource/Sources/SimpleFeatureSource.ts Normal file
View file

@ -0,0 +1,16 @@
import {FeatureSourceForLayer} from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import FilteredLayer from "../../Models/FilteredLayer";
export default class SimpleFeatureSource implements FeatureSourceForLayer {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]);
public readonly name: string = "SimpleFeatureSource";
public readonly layer: FilteredLayer;
constructor(layer: FilteredLayer) {
this.name = "SimpleFeatureSource("+layer.layerDef.id+")"
this.layer = layer
}
}

40
Logic/FeatureSource/WayHandlingApplyingFeatureSource.ts → Logic/FeatureSource/Sources/WayHandlingApplyingFeatureSource.ts
View file

@ -1,4 +1,4 @@
import FeatureSource from "./FeatureSource"; import {FeatureSourceForLayer} from "./FeatureSource";
import {UIEventSource} from "../UIEventSource"; import {UIEventSource} from "../UIEventSource";
import {GeoOperations} from "../GeoOperations"; import {GeoOperations} from "../GeoOperations";
import LayerConfig from "../../Models/ThemeConfig/LayerConfig"; import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
@ -6,39 +6,31 @@ import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
/** /**
* This is the part of the pipeline which introduces extra points at the center of an area (but only if this is demanded by the wayhandling) * This is the part of the pipeline which introduces extra points at the center of an area (but only if this is demanded by the wayhandling)
*/ */
export default class WayHandlingApplyingFeatureSource implements FeatureSource { export default class WayHandlingApplyingFeatureSource implements FeatureSourceForLayer {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>; public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name; public readonly name;
public readonly layer;
constructor(layers: UIEventSource<{ constructor(upstream: FeatureSourceForLayer) {
layerDef: LayerConfig
}[]>,
upstream: FeatureSource) {
this.name = "Wayhandling of " + upstream.name; this.name = "Wayhandling of " + upstream.name;
this.layer = upstream.layer
const layer = upstream.layer.layerDef;
if (layer.wayHandling === LayerConfig.WAYHANDLING_DEFAULT) {
// We don't have to do anything fancy
// lets just wire up the upstream
this.features = upstream.features;
return;
}
this.features = upstream.features.map( this.features = upstream.features.map(
features => { features => {
if (features === undefined) { if (features === undefined) {
return; return;
} }
const layerDict = {};
let allDefaultWayHandling = true;
for (const layer of layers.data) {
layerDict[layer.layerDef.id] = layer;
if (layer.layerDef.wayHandling !== LayerConfig.WAYHANDLING_DEFAULT) {
allDefaultWayHandling = false;
}
}
const newFeatures: { feature: any, freshness: Date }[] = []; const newFeatures: { feature: any, freshness: Date }[] = [];
for (const f of features) { for (const f of features) {
const feat = f.feature; const feat = f.feature;
const layerId = feat._matching_layer_id;
const layer: LayerConfig = layerDict[layerId].layerDef;
if (layer === undefined) {
console.error("No layer found with id " + layerId);
continue;
}
if (layer.wayHandling === LayerConfig.WAYHANDLING_DEFAULT) { if (layer.wayHandling === LayerConfig.WAYHANDLING_DEFAULT) {
newFeatures.push(f); newFeatures.push(f);
@ -47,19 +39,17 @@ export default class WayHandlingApplyingFeatureSource implements FeatureSource {
if (feat.geometry.type === "Point") { if (feat.geometry.type === "Point") {
newFeatures.push(f); newFeatures.push(f);
// it is a point, nothing to do here // feature is a point, nothing to do here
continue; continue;
} }
// Create the copy // Create the copy
const centerPoint = GeoOperations.centerpoint(feat); const centerPoint = GeoOperations.centerpoint(feat);
centerPoint["_matching_layer_id"] = feat._matching_layer_id;
newFeatures.push({feature: centerPoint, freshness: f.freshness}); newFeatures.push({feature: centerPoint, freshness: f.freshness});
if (layer.wayHandling === LayerConfig.WAYHANDLING_CENTER_AND_WAY) { if (layer.wayHandling === LayerConfig.WAYHANDLING_CENTER_AND_WAY) {
newFeatures.push(f); newFeatures.push(f);
} }
} }
return newFeatures; return newFeatures;
} }

0
Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts Normal file
View file

72
Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts Normal file
View file

@ -0,0 +1,72 @@
/***
* A tiled source which dynamically loads the required tiles
*/
import State from "../../../State";
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer} from "../FeatureSource";
import {Utils} from "../../../Utils";
import {UIEventSource} from "../../UIEventSource";
import Loc from "../../../Models/Loc";
export default class DynamicTileSource {
private readonly _loadedTiles = new Set<number>();
public readonly existingTiles: Map<number, Map<number, FeatureSourceForLayer>> = new Map<number, Map<number, FeatureSourceForLayer>>()
constructor(
layer: FilteredLayer,
zoomlevel: number,
constructTile: (xy: [number, number]) => FeatureSourceForLayer,
state: {
locationControl: UIEventSource<Loc>
leafletMap: any
}
) {
state = State.state
const self = this;
const neededTiles = state.locationControl.map(
location => {
if (!layer.isDisplayed.data) {
// No need to download! - the layer is disabled
return undefined;
}
if (location.zoom < layer.layerDef.minzoom) {
// No need to download! - the layer is disabled
return undefined;
}
// Yup, this is cheating to just get the bounds here
const bounds = state.leafletMap.data?.getBounds()
if (bounds === undefined) {
// We'll retry later
return undefined
}
const tileRange = Utils.TileRangeBetween(zoomlevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const needed = Utils.MapRange(tileRange, (x, y) => Utils.tile_index(zoomlevel, x, y)).filter(i => !self._loadedTiles.has(i))
if(needed.length === 0){
return undefined
}
return needed
}
, [layer.isDisplayed, state.leafletMap]).stabilized(250);
neededTiles.addCallbackAndRunD(neededIndexes => {
for (const neededIndex of neededIndexes) {
self._loadedTiles.add(neededIndex)
const xy = Utils.tile_from_index(zoomlevel, neededIndex)
const src = constructTile(xy)
let xmap = self.existingTiles.get(xy[0])
if(xmap === undefined){
xmap = new Map<number, FeatureSourceForLayer>()
self.existingTiles.set(xy[0], xmap)
}
xmap.set(xy[1], src)
}
})
}
}

3
Logic/FeatureSource/TiledFeatureSource/README.md Normal file
View file

@ -0,0 +1,3 @@
Data in MapComplete can come from multiple sources.
In order to keep thins snappy, they are distributed over a tiled database

0
Logic/FeatureSource/TiledFeatureSource/SingleLayerSource.ts Normal file
View file

0
Logic/FeatureSource/TiledFeatureSource/TileHierarchy.ts Normal file
View file

0
Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource.ts Normal file
View file

40
Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts Normal file
View file

@ -0,0 +1,40 @@
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import Loc from "../../../Models/Loc";
import GeoJsonSource from "../GeoJsonSource";
import DynamicTileSource from "./DynamicTileSource";
export default class DynamicGeoJsonTileSource extends DynamicTileSource {
constructor(layer: FilteredLayer,
registerLayer: (layer: FeatureSourceForLayer) => void,
state: {
locationControl: UIEventSource<Loc>
leafletMap: any
}) {
const source = layer.layerDef.source
if (source.geojsonZoomLevel === undefined) {
throw "Invalid layer: geojsonZoomLevel expected"
}
if (source.geojsonSource === undefined) {
throw "Invalid layer: geojsonSource expected"
}
super(
layer,
source.geojsonZoomLevel,
(xy) => {
const xyz: [number, number, number] = [xy[0], xy[1], source.geojsonZoomLevel]
const src = new GeoJsonSource(
layer,
xyz
)
registerLayer(src)
return src
},
state
);
}
}

0
Logic/Osm/ExtractRelations.ts → Logic/Osm/RelationsTracker.ts
View file