Huge refactoring of the feature pipeline, WIP

Logic/FeatureSource/Sources/FeatureSourceMerger.ts

@@ -0,0 +1,91 @@
+import FeatureSource, {FeatureSourceForLayer} from "./FeatureSource";
+import {UIEventSource} from "../UIEventSource";
+import FilteredLayer from "../../Models/FilteredLayer";
+
+/**
+ * Merges features from different featureSources for a single layer
+ * Uses the freshest feature available in the case multiple sources offer data with the same identifier
+ */
+export default class FeatureSourceMerger implements FeatureSourceForLayer {
+
+    public features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]);
+    public readonly name;
+    public readonly layer: FilteredLayer
+    private readonly _sources: UIEventSource<FeatureSource[]>;
+
+    constructor(layer: FilteredLayer ,sources: UIEventSource<FeatureSource[]>) {
+        this._sources = sources;
+        this.layer = layer;
+        this.name = "SourceMerger"
+        const self = this;
+
+        const handledSources = new Set<FeatureSource>();
+
+        sources.addCallbackAndRunD(sources => {
+            let newSourceRegistered = false;
+            for (let i = 0; i < sources.length; i++) {
+                let source = sources[i];
+                if (handledSources.has(source)) {
+                    continue
+                }
+                handledSources.add(source)
+                newSourceRegistered = true
+                source.features.addCallback(() => {
+                    self.Update();
+                });
+                if (newSourceRegistered) {
+                    self.Update();
+                }
+            }
+        })
+
+    }
+
+    private Update() {
+
+        let somethingChanged = false;
+        const all: Map<string, { feature: any, freshness: Date }> = new Map<string, { feature: any; freshness: Date }>();
+        // We seed the dictionary with the previously loaded features
+        const oldValues = this.features.data ?? [];
+        for (const oldValue of oldValues) {
+            all.set(oldValue.feature.id + oldValue.feature._matching_layer_id, oldValue)
+        }
+
+        for (const source of this._sources.data) {
+            if (source?.features?.data === undefined) {
+                continue;
+            }
+            for (const f of source.features.data) {
+                const id = f.feature.properties.id + f.feature._matching_layer_id;
+                if (!all.has(id)) {
+                    // This is a new feature
+                    somethingChanged = true;
+                    all.set(id, f);
+                    continue;
+                }
+
+                // This value has been seen already, either in a previous run or by a previous datasource
+                // Let's figure out if something changed
+                const oldV = all.get(id);
+                if (oldV.freshness < f.freshness) {
+                    // Jup, this feature is fresher
+                    all.set(id, f);
+                    somethingChanged = true;
+                }
+            }
+        }
+
+        if (!somethingChanged) {
+            // We don't bother triggering an update
+            return;
+        }
+
+        const newList = [];
+        all.forEach((value, _) => {
+            newList.push(value)
+        })
+        this.features.setData(newList);
+    }
+
+
+}

Logic/FeatureSource/Sources/FilteringFeatureSource.ts

@@ -0,0 +1,101 @@
+import {FeatureSourceForLayer} from "./FeatureSource";
+import {UIEventSource} from "../UIEventSource";
+import Hash from "../Web/Hash";
+import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
+import FilteredLayer from "../../Models/FilteredLayer";
+
+export default class FilteringFeatureSource implements FeatureSourceForLayer {
+    public features: UIEventSource<{ feature: any; freshness: Date }[]> =
+        new UIEventSource<{ feature: any; freshness: Date }[]>([]);
+    public readonly name = "FilteringFeatureSource";
+    public readonly layer: FilteredLayer;
+
+    constructor(
+        state: {
+            locationControl: UIEventSource<{ zoom: number }>,
+            selectedElement: UIEventSource<any>,
+        },
+        upstream: FeatureSourceForLayer
+    ) {
+        const self = this;
+
+        this.layer = upstream.layer;
+        const layer = upstream.layer;
+
+        function update() {
+            const features: { feature: any; freshness: Date }[] = upstream.features.data;
+            const newFeatures = features.filter((f) => {
+                if (
+                    state.selectedElement.data?.id === f.feature.id ||
+                    f.feature.id === Hash.hash.data) {
+                    // This is the selected object - it gets a free pass even if zoom is not sufficient or it is filtered away
+                    return true;
+                }
+
+                const isShown = layer.layerDef.isShown;
+                const tags = f.feature.properties;
+                if (isShown.IsKnown(tags)) {
+                    const result = layer.layerDef.isShown.GetRenderValue(
+                        f.feature.properties
+                    ).txt;
+                    if (result !== "yes") {
+                        return false;
+                    }
+                }
+
+                const tagsFilter = layer.appliedFilters.data;
+                if (tagsFilter) {
+                    if (!tagsFilter.matchesProperties(f.feature.properties)) {
+                        // Hidden by the filter on the layer itself - we want to hide it no matter wat
+                        return false;
+                    }
+                }
+                if (!FilteringFeatureSource.showLayer(layer, state.locationControl.data)) {
+                    // The layer itself is either disabled or hidden due to zoom constraints
+                    // We should return true, but it might still match some other layer
+                    return false;
+                }
+                return true;
+            });
+
+            self.features.setData(newFeatures);
+        }
+
+        upstream.features.addCallback(() => {
+            update();
+        });
+
+        let isShown = state.locationControl.map((l) => FilteringFeatureSource.showLayer(layer, l),
+            [layer.isDisplayed])
+            
+        isShown.addCallback(isShown => {
+            if (isShown) {
+                update();
+            } else {
+                self.features.setData([])
+            }
+        });
+
+        layer.appliedFilters.addCallback(_ => {
+            if(!isShown.data){
+                // Currently not shown.
+                // Note that a change in 'isSHown' will trigger an update as well, so we don't have to watch it another time
+                return;
+            }
+            update()
+        })
+
+        update();
+    }
+
+    private static showLayer(
+        layer: {
+            isDisplayed: UIEventSource<boolean>;
+            layerDef: LayerConfig;
+        },
+        location: { zoom: number }) {
+        return layer.isDisplayed.data &&
+            layer.layerDef.minzoomVisible <= location.zoom;
+
+    }
+}

Logic/FeatureSource/Sources/GeoJsonSource.ts

@@ -0,0 +1,95 @@
+import {FeatureSourceForLayer} from "./FeatureSource";
+import {UIEventSource} from "../UIEventSource";
+import {Utils} from "../../Utils";
+import FilteredLayer from "../../Models/FilteredLayer";
+import {control} from "leaflet";
+
+
+/**
+ * Fetches a geojson file somewhere and passes it along
+ */
+export default class GeoJsonSource implements FeatureSourceForLayer {
+
+    public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
+    public readonly name;
+    public readonly isOsmCache: boolean
+    private onFail: ((errorMsg: any, url: string) => void) = undefined;
+    private readonly seenids: Set<string> = new Set<string>()
+    public readonly layer: FilteredLayer;
+
+
+    public constructor(flayer: FilteredLayer,
+                       zxy?: [number, number, number]) {
+
+        if (flayer.layerDef.source.geojsonZoomLevel !== undefined && zxy === undefined) {
+            throw "Dynamic layers are not supported. Use 'DynamicGeoJsonTileSource instead"
+        }
+
+        this.layer = flayer;
+        let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id);
+        if (zxy !== undefined) {
+            url = url
+                .replace('{z}', "" + zxy[0])
+                .replace('{x}', "" + zxy[1])
+                .replace('{y}', "" + zxy[2])
+        }
+
+        this.name = "GeoJsonSource of " + url;
+
+        this.isOsmCache = flayer.layerDef.source.isOsmCacheLayer;
+        this.features = new UIEventSource<{ feature: any; freshness: Date }[]>([])
+        this.LoadJSONFrom(url)
+    }
+
+
+    private LoadJSONFrom(url: string) {
+        const eventSource = this.features;
+        const self = this;
+        Utils.downloadJson(url)
+            .then(json => {
+                if (json.elements === [] && json.remarks.indexOf("runtime error") > 0) {
+                    self.onFail("Runtime error (timeout)", url)
+                    return;
+                }
+                const time = new Date();
+                const newFeatures: { feature: any, freshness: Date } [] = []
+                let i = 0;
+                let skipped = 0;
+                for (const feature of json.features) {
+                    const props = feature.properties
+                    for (const key in props) {
+                        if (typeof props[key] !== "string") {
+                            props[key] = "" + props[key]
+                        }
+                    }
+
+                    if (props.id === undefined) {
+                        props.id = url + "/" + i;
+                        feature.id = url + "/" + i;
+                        i++;
+                    }
+                    if (self.seenids.has(props.id)) {
+                        skipped++;
+                        continue;
+                    }
+                    self.seenids.add(props.id)
+
+                    let freshness: Date = time;
+                    if (feature.properties["_last_edit:timestamp"] !== undefined) {
+                        freshness = new Date(props["_last_edit:timestamp"])
+                    }
+
+                    newFeatures.push({feature: feature, freshness: freshness})
+                }
+                console.debug("Downloaded " + newFeatures.length + " new features and " + skipped + " already seen features from " + url);
+
+                if (newFeatures.length == 0) {
+                    return;
+                }
+
+                eventSource.setData(eventSource.data.concat(newFeatures))
+
+            }).catch(msg => console.error("Could not load geojon layer", url, "due to", msg))
+    }
+
+}

Logic/FeatureSource/Sources/LocalStorageSource.ts

@@ -0,0 +1,35 @@
+import FeatureSource from "./FeatureSource";
+import {UIEventSource} from "../UIEventSource";
+import LocalStorageSaverActor from "./LocalStorageSaverActor";
+import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig";
+
+export default class LocalStorageSource implements FeatureSource {
+    public features: UIEventSource<{ feature: any; freshness: Date }[]>;
+    public readonly name = "LocalStorageSource";
+
+    constructor(layout: UIEventSource<LayoutConfig>) {
+        this.features = new UIEventSource<{ feature: any; freshness: Date }[]>([])
+        const key = LocalStorageSaverActor.storageKey + layout.data.id
+        layout.addCallbackAndRun(_ => {
+            try {
+                const fromStorage = localStorage.getItem(key);
+                if (fromStorage == null) {
+                    return;
+                }
+                const loaded: { feature: any; freshness: Date | string }[] =
+                    JSON.parse(fromStorage);
+
+                const parsed: { feature: any; freshness: Date }[] = loaded.map(ff => ({
+                    feature: ff.feature,
+                    freshness: typeof ff.freshness == "string" ? new Date(ff.freshness) : ff.freshness
+                }))
+
+                this.features.setData(parsed);
+                console.log("Loaded ", loaded.length, " features from localstorage as cache")
+            } catch (e) {
+                console.log("Could not load features from localStorage:", e)
+                localStorage.removeItem(key)
+            }
+        })
+    }
+}

Logic/FeatureSource/Sources/OsmApiFeatureSource.ts

@@ -0,0 +1,110 @@
+import FeatureSource from "./FeatureSource";
+import {UIEventSource} from "../UIEventSource";
+import {OsmObject} from "../Osm/OsmObject";
+import {Utils} from "../../Utils";
+import Loc from "../../Models/Loc";
+import FilteredLayer from "../../Models/FilteredLayer";
+
+
+export default class OsmApiFeatureSource implements FeatureSource {
+    public readonly features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]);
+    public readonly name: string = "OsmApiFeatureSource";
+    private readonly loadedTiles: Set<string> = new Set<string>();
+    private readonly _state: {
+        leafletMap: UIEventSource<any>;
+        locationControl: UIEventSource<Loc>, filteredLayers: UIEventSource<FilteredLayer[]>};
+
+    constructor(state: {locationControl: UIEventSource<Loc>, filteredLayers: UIEventSource<FilteredLayer[]>, leafletMap: UIEventSource<any>,
+    overpassMaxZoom: UIEventSource<number>}) {
+        this._state = state;
+        const self = this;
+        function update(){
+            const minZoom = state.overpassMaxZoom.data;
+            const location = state.locationControl.data
+            if(minZoom === undefined || location === undefined){
+                return;
+            }
+            if(minZoom < 14){
+                throw "MinZoom should be at least 14 or higher, OSM-api won't work otherwise"
+            }
+            if(location.zoom > minZoom){
+                return;
+            }
+            self.loadArea()
+        }
+    }
+
+
+    public load(id: string) {
+        if (id.indexOf("-") >= 0) {
+            // Newly added point - not yet in OSM
+            return;
+        }
+        console.debug("Downloading", id, "from the OSM-API")
+        OsmObject.DownloadObject(id).addCallbackAndRunD(element => {
+            try {
+                const geojson = element.asGeoJson();
+                geojson.id = geojson.properties.id;
+                this.features.setData([{feature: geojson, freshness: element.timestamp}])
+            } catch (e) {
+                console.error(e)
+            }
+        })
+    }
+
+    /**
+     * Loads the current inview-area
+     */
+    public loadArea(z: number = 14): boolean {
+        const layers = this._state.filteredLayers.data;
+
+        const disabledLayers = layers.filter(layer => layer.layerDef.source.overpassScript !== undefined || layer.layerDef.source.geojsonSource !== undefined)
+        if (disabledLayers.length > 0) {
+            return false;
+        }
+        if (this._state.leafletMap.data === undefined) {
+            return false; // Not yet inited
+        }
+        const bounds = this._state.leafletMap.data.getBounds()
+        const tileRange = Utils.TileRangeBetween(z, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
+        const self = this;
+        Utils.MapRange(tileRange, (x, y) => {
+            const key = x + "/" + y;
+            if (self.loadedTiles.has(key)) {
+                return;
+            }
+
+            self.loadedTiles.add(key);
+
+            const bounds = Utils.tile_bounds(z, x, y);
+            console.log("Loading OSM data tile", z, x, y, " with bounds", bounds)
+            OsmObject.LoadArea(bounds, objects => {
+                const keptGeoJson: { feature: any, freshness: Date }[] = []
+                // Which layer does the object match?
+                for (const object of objects) {
+
+                    for (const flayer of layers) {
+                        const layer = flayer.layerDef;
+                        const tags = object.tags
+                        const doesMatch = layer.source.osmTags.matchesProperties(tags);
+                        if (doesMatch) {
+                            const geoJson = object.asGeoJson();
+                            geoJson._matching_layer_id = layer.id
+                            keptGeoJson.push({feature: geoJson, freshness: object.timestamp})
+                            break;
+                        }
+
+                    }
+
+                }
+
+                self.features.setData(keptGeoJson)
+            });
+
+        });
+
+        return true;
+
+    }
+
+}

Logic/FeatureSource/Sources/RememberingSource.ts

@@ -0,0 +1,32 @@
+
+import FeatureSource, {FeatureSourceForLayer} from "./FeatureSource";
+import {UIEventSource} from "../UIEventSource";
+import FilteredLayer from "../../Models/FilteredLayer";
+/**
+ * Every previously added point is remembered, but new points are added.
+ * Data coming from upstream will always overwrite a previous value
+ */
+export default class RememberingSource implements FeatureSource {
+
+    public readonly features: UIEventSource<{ feature: any, freshness: Date }[]>;
+    public readonly name;
+
+    constructor(source: FeatureSource) {
+        const self = this;
+        this.name = "RememberingSource of " + source.name;
+        const empty = [];
+        this.features = source.features.map(features => {
+            const oldFeatures = self.features?.data ?? empty;
+            if (features === undefined) {
+                return oldFeatures;
+            }
+
+            // Then new ids
+            const ids = new Set<string>(features.map(f => f.feature.properties.id + f.feature.geometry.type));
+            // the old data
+            const oldData = oldFeatures.filter(old => !ids.has(old.feature.properties.id + old.feature.geometry.type))
+            return [...features, ...oldData];
+        })
+    }
+
+}

Logic/FeatureSource/Sources/SimpleFeatureSource.ts

@@ -0,0 +1,16 @@
+import {FeatureSourceForLayer} from "./FeatureSource";
+import {UIEventSource} from "../UIEventSource";
+import FilteredLayer from "../../Models/FilteredLayer";
+
+export default class SimpleFeatureSource implements FeatureSourceForLayer {
+    public readonly features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]);
+    public readonly name: string = "SimpleFeatureSource";
+    public readonly layer: FilteredLayer;
+
+    constructor(layer: FilteredLayer) {
+        this.name = "SimpleFeatureSource("+layer.layerDef.id+")"
+        this.layer = layer
+    }
+
+
+}

Logic/FeatureSource/Sources/WayHandlingApplyingFeatureSource.ts

@@ -0,0 +1,60 @@
+import {FeatureSourceForLayer} from "./FeatureSource";
+import {UIEventSource} from "../UIEventSource";
+import {GeoOperations} from "../GeoOperations";
+import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
+
+/**
+ * This is the part of the pipeline which introduces extra points at the center of an area (but only if this is demanded by the wayhandling)
+ */
+export default class WayHandlingApplyingFeatureSource implements FeatureSourceForLayer {
+    public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
+    public readonly name;
+    public readonly layer;
+
+    constructor(upstream: FeatureSourceForLayer) {
+        this.name = "Wayhandling of " + upstream.name;
+        this.layer = upstream.layer
+        const layer = upstream.layer.layerDef;
+        
+        if (layer.wayHandling === LayerConfig.WAYHANDLING_DEFAULT) {
+            // We don't have to do anything fancy
+            // lets just wire up the upstream
+            this.features = upstream.features;
+            return;
+        }
+
+        this.features = upstream.features.map(
+            features => {
+                if (features === undefined) {
+                    return;
+                }
+                const newFeatures: { feature: any, freshness: Date }[] = [];
+                for (const f of features) {
+                    const feat = f.feature;
+
+                    if (layer.wayHandling === LayerConfig.WAYHANDLING_DEFAULT) {
+                        newFeatures.push(f);
+                        continue;
+                    }
+
+                    if (feat.geometry.type === "Point") {
+                        newFeatures.push(f);
+                        // feature is a point, nothing to do here
+                        continue;
+                    }
+
+                    // Create the copy
+                    const centerPoint = GeoOperations.centerpoint(feat);
+
+                    newFeatures.push({feature: centerPoint, freshness: f.freshness});
+                    if (layer.wayHandling === LayerConfig.WAYHANDLING_CENTER_AND_WAY) {
+                        newFeatures.push(f);
+                    }
+                }
+                return newFeatures;
+            }
+        );
+
+    }
+
+}