More refactoring, move minimap behind facade

Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts

@@ -0,0 +1,63 @@
+import FilteredLayer from "../../../Models/FilteredLayer";
+import {FeatureSourceForLayer} from "../FeatureSource";
+import {UIEventSource} from "../../UIEventSource";
+import Loc from "../../../Models/Loc";
+import DynamicTileSource from "./DynamicTileSource";
+import {Utils} from "../../../Utils";
+import GeoJsonSource from "../Sources/GeoJsonSource";
+
+export default class DynamicGeoJsonTileSource extends DynamicTileSource {
+    constructor(layer: FilteredLayer,
+                registerLayer: (layer: FeatureSourceForLayer) => void,
+                state: {
+                    locationControl: UIEventSource<Loc>
+                    leafletMap: any
+                }) {
+        const source = layer.layerDef.source
+        if (source.geojsonZoomLevel === undefined) {
+            throw "Invalid layer: geojsonZoomLevel expected"
+        }
+        if (source.geojsonSource === undefined) {
+            throw "Invalid layer: geojsonSource expected"
+        }
+        
+        const whitelistUrl = source.geojsonSource.replace("{z}_{x}_{y}.geojson", "overview.json")
+            .replace("{layer}",layer.layerDef.id)
+        
+        let whitelist = undefined
+        Utils.downloadJson(whitelistUrl).then(
+            json => {
+                const data = new Map<number, Set<number>>();
+                for (const x in json) {
+                    data.set(Number(x), new Set(json[x]))
+                }
+                whitelist = data
+            }
+        ).catch(err => {
+            console.warn("No whitelist found for ", layer.layerDef.id, err)
+        })
+
+        super(
+            layer,
+            source.geojsonZoomLevel,
+            (zxy) => {
+                if(whitelist !== undefined){
+                    const isWhiteListed = whitelist.get(zxy[1])?.has(zxy[2])
+                    if(!isWhiteListed){
+                        return undefined;
+                    }
+                }
+                
+                const src = new GeoJsonSource(
+                    layer,
+                    zxy
+                )
+                registerLayer(src)
+                return src
+            },
+            state
+        );
+
+    }
+
+}

Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts

@@ -1,22 +1,24 @@
-/***
- * A tiled source which dynamically loads the required tiles
- */
+
 import State from "../../../State";
 import FilteredLayer from "../../../Models/FilteredLayer";
-import {FeatureSourceForLayer} from "../FeatureSource";
+import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
 import {Utils} from "../../../Utils";
 import {UIEventSource} from "../../UIEventSource";
 import Loc from "../../../Models/Loc";
+import TileHierarchy from "./TileHierarchy";
 
-export default class DynamicTileSource {
+/***
+ * A tiled source which dynamically loads the required tiles at a fixed zoom level
+ */
+export default class DynamicTileSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
     private readonly _loadedTiles = new Set<number>();
-    
-    public readonly existingTiles: Map<number, Map<number, FeatureSourceForLayer>> = new Map<number, Map<number, FeatureSourceForLayer>>()
+
+    public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled>;
 
     constructor(
         layer: FilteredLayer,
         zoomlevel: number,
-        constructTile: (xy: [number, number]) => FeatureSourceForLayer,
+        constructTile: (zxy: [number, number, number]) => (FeatureSourceForLayer & Tiled),
         state: {
             locationControl: UIEventSource<Loc>
             leafletMap: any
@@ -24,6 +26,8 @@ export default class DynamicTileSource {
     ) {
         state = State.state
         const self = this;
+
+        this.loadedTiles = new Map<number,FeatureSourceForLayer & Tiled>()
         const neededTiles = state.locationControl.map(
             location => {
                 if (!layer.isDisplayed.data) {
@@ -45,28 +49,30 @@ export default class DynamicTileSource {
                 const tileRange = Utils.TileRangeBetween(zoomlevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
 
                 const needed = Utils.MapRange(tileRange, (x, y) => Utils.tile_index(zoomlevel, x, y)).filter(i => !self._loadedTiles.has(i))
-                if(needed.length === 0){
+                if (needed.length === 0) {
                     return undefined
                 }
                 return needed
             }
             , [layer.isDisplayed, state.leafletMap]).stabilized(250);
-        
+
         neededTiles.addCallbackAndRunD(neededIndexes => {
+            console.log("Tiled geojson source ",layer.layerDef.id," needs", neededIndexes)
+            if (neededIndexes === undefined) {
+                return;
+            }
             for (const neededIndex of neededIndexes) {
                 self._loadedTiles.add(neededIndex)
-                const xy = Utils.tile_from_index(zoomlevel, neededIndex)
-                const src = constructTile(xy)
-                let xmap = self.existingTiles.get(xy[0])
-                if(xmap === undefined){
-                   xmap =  new Map<number, FeatureSourceForLayer>()
-                   self.existingTiles.set(xy[0], xmap) 
+                const src = constructTile( Utils.tile_from_index(neededIndex))
+                if(src !== undefined){
+                    self.loadedTiles.set(neededIndex, src)
                 }
-            xmap.set(xy[1], src)
             }
         })
 
 
     }
 
-}
+
+}
+

Logic/FeatureSource/TiledFeatureSource/README.md

@@ -1,3 +1,27 @@
 Data in MapComplete can come from multiple sources.
 
-In order to keep thins snappy, they are distributed over a tiled database
+Currently, they are:
+
+- The Overpass-API
+- The OSM-API
+- One or more GeoJSON files. This can be a single file or a set of tiled geojson files
+- LocalStorage, containing features from a previous visit
+- Changes made by the user introducing new features
+
+When the data enters from Overpass or from the OSM-API, they are first distributed per layer:
+
+OVERPASS | ---PerLayerFeatureSource---> FeatureSourceForLayer[]
+OSM      |
+
+The GeoJSon files (not tiled) are then added to this list
+
+A single FeatureSourcePerLayer is then further handled by splitting it into a tile hierarchy.
+
+
+
+In order to keep thins snappy, they are distributed over a tiled database per layer.
+
+
+## Notes
+
+`cached-featuresbookcases` is the old key used `cahced-features{themeid}` and should be cleaned up

Logic/FeatureSource/TiledFeatureSource/TileHierarchy.ts

@@ -0,0 +1,25 @@
+import FeatureSource, {Tiled} from "../FeatureSource";
+import {BBox} from "../../GeoOperations";
+
+export default interface TileHierarchy<T extends FeatureSource & Tiled> {
+
+    /**
+     * A mapping from 'tile_index' to the actual tile featrues
+     */
+    loadedTiles: Map<number, T>
+
+}
+
+export class TileHierarchyTools {
+
+    public static getTiles<T extends FeatureSource & Tiled>(hierarchy: TileHierarchy<T>, bbox: BBox): T[] {
+        const result = []
+        hierarchy.loadedTiles.forEach((tile) => {
+            if (tile.bbox.overlapsWith(bbox)) {
+                result.push(tile)
+            }
+        })
+        return result;
+    }
+
+}

Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts

@@ -1,10 +1,10 @@
-import TileHierarchy from "./TiledFeatureSource/TileHierarchy";
-import FeatureSource, {FeatureSourceForLayer, Tiled} from "./FeatureSource";
-import {UIEventSource} from "../UIEventSource";
-import FilteredLayer from "../../Models/FilteredLayer";
-import FeatureSourceMerger from "./Sources/FeatureSourceMerger";
-import {BBox} from "../GeoOperations";
-import {Utils} from "../../Utils";
+import TileHierarchy from "./TileHierarchy";
+import {UIEventSource} from "../../UIEventSource";
+import FeatureSource, {FeatureSourceForLayer, Tiled} from "../FeatureSource";
+import FilteredLayer from "../../../Models/FilteredLayer";
+import {Utils} from "../../../Utils";
+import {BBox} from "../../GeoOperations";
+import FeatureSourceMerger from "../Sources/FeatureSourceMerger";
 
 export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer & Tiled> {
     public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
@@ -24,8 +24,9 @@ export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer
      * @param src
      * @param index
      */
-    public registerTile(src: FeatureSource, index: number) {
+    public registerTile(src: FeatureSource  & Tiled) {
 
+        const index = src.tileIndex
         if (this.sources.has(index)) {
             const sources = this.sources.get(index)
             sources.data.push(src)

Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource.ts

@@ -0,0 +1,191 @@
+import FeatureSource, {FeatureSourceForLayer, IndexedFeatureSource, Tiled} from "../FeatureSource";
+import {UIEventSource} from "../../UIEventSource";
+import {Utils} from "../../../Utils";
+import {BBox} from "../../GeoOperations";
+import FilteredLayer from "../../../Models/FilteredLayer";
+import TileHierarchy from "./TileHierarchy";
+import {feature} from "@turf/turf";
+
+/**
+ * Contains all features in a tiled fashion.
+ * The data will be automatically broken down into subtiles when there are too much features in a single tile or if the zoomlevel is too high
+ */
+export default class TiledFeatureSource implements Tiled, IndexedFeatureSource, FeatureSourceForLayer, TileHierarchy<IndexedFeatureSource & FeatureSourceForLayer & Tiled> {
+    public readonly z: number;
+    public readonly x: number;
+    public readonly y: number;
+    public readonly parent: TiledFeatureSource;
+    public readonly root: TiledFeatureSource
+    public readonly layer: FilteredLayer;
+    /* An index of all known tiles. allTiles[z][x][y].get('layerid') will yield the corresponding tile.
+    * Only defined on the root element!
+     */
+    public readonly loadedTiles: Map<number, TiledFeatureSource & FeatureSourceForLayer> = undefined;
+
+    public readonly maxFeatureCount: number;
+    public readonly name;
+    public readonly features: UIEventSource<{ feature: any, freshness: Date }[]>
+    public readonly containedIds: UIEventSource<Set<string>>
+
+    public readonly bbox: BBox;
+    private upper_left: TiledFeatureSource
+    private upper_right: TiledFeatureSource
+    private lower_left: TiledFeatureSource
+    private lower_right: TiledFeatureSource
+    private readonly maxzoom: number;
+    private readonly options: TiledFeatureSourceOptions
+    public readonly tileIndex: number;
+
+    private constructor(z: number, x: number, y: number, parent: TiledFeatureSource, options?: TiledFeatureSourceOptions) {
+        this.z = z;
+        this.x = x;
+        this.y = y;
+        this.bbox = BBox.fromTile(z, x, y)
+        this.tileIndex = Utils.tile_index(z, x, y)
+        this.name = `TiledFeatureSource(${z},${x},${y})`
+        this.parent = parent;
+        this.layer = options.layer
+        options = options ?? {}
+        this.maxFeatureCount = options?.maxFeatureCount ?? 500;
+        this.maxzoom = options.maxZoomLevel ?? 18
+        this.options = options;
+        if (parent === undefined) {
+            throw "Parent is not allowed to be undefined. Use null instead"
+        }
+        if (parent === null && z !== 0 && x !== 0 && y !== 0) {
+            throw "Invalid root tile: z, x and y should all be null"
+        }
+        if (parent === null) {
+            this.root = this;
+            this.loadedTiles = new Map()
+        } else {
+            this.root = this.parent.root;
+            this.loadedTiles = this.root.loadedTiles;
+            const i = Utils.tile_index(z, x, y)
+            this.root.loadedTiles.set(i, this)
+        }
+        this.features = new UIEventSource<any[]>([])
+        this.containedIds = this.features.map(features => {
+            if (features === undefined) {
+                return undefined;
+            }
+            return new Set(features.map(f => f.feature.properties.id))
+        })
+
+        // We register this tile, but only when there is some data in it
+        if (this.options.registerTile !== undefined) {
+            this.features.addCallbackAndRunD(features => {
+                if (features.length === 0) {
+                    return;
+                }
+                this.options.registerTile(this)
+                return true;
+            })
+        }
+
+
+    }
+
+    public static createHierarchy(features: FeatureSource, options?: TiledFeatureSourceOptions): TiledFeatureSource {
+        const root = new TiledFeatureSource(0, 0, 0, null, options)
+        features.features?.addCallbackAndRunD(feats => root.addFeatures(feats))
+        return root;
+    }
+
+    private isSplitNeeded(featureCount: number){
+        if(this.upper_left !== undefined){
+            // This tile has been split previously, so we keep on splitting
+            return true;
+        }
+        if(this.z >= this.maxzoom){
+            // We are not allowed to split any further
+            return false
+        }
+        if(this.options.minZoomLevel !== undefined && this.z < this.options.minZoomLevel){
+            // We must have at least this zoom level before we are allowed to start splitting
+            return true
+        }
+        
+        // To much features - we split
+        return featureCount > this.maxFeatureCount
+        
+        
+    }
+    
+    /***
+     * Adds the list of features to this hierarchy.
+     * If there are too much features, the list will be broken down and distributed over the subtiles (only retaining features that don't fit a subtile on this level)
+     * @param features
+     * @private
+     */
+    private addFeatures(features: { feature: any, freshness: Date }[]) {
+        if (features === undefined || features.length === 0) {
+            return;
+        }
+        
+        if (!this.isSplitNeeded(features.length)) {
+            this.features.setData(features)
+            return;
+        }
+
+        if (this.upper_left === undefined) {
+            this.upper_left = new TiledFeatureSource(this.z + 1, this.x * 2, this.y * 2, this, this.options)
+            this.upper_right = new TiledFeatureSource(this.z + 1, this.x * 2 + 1, this.y * 2, this, this.options)
+            this.lower_left = new TiledFeatureSource(this.z + 1, this.x * 2, this.y * 2 + 1, this, this.options)
+            this.lower_right = new TiledFeatureSource(this.z + 1, this.x * 2 + 1, this.y * 2 + 1, this, this.options)
+        }
+
+        const ulf = []
+        const urf = []
+        const llf = []
+        const lrf = []
+        const overlapsboundary = []
+
+        for (const feature of features) {
+            const bbox = BBox.get(feature.feature)
+            if (this.options.minZoomLevel === undefined) {
+
+
+                if (bbox.isContainedIn(this.upper_left.bbox)) {
+                    ulf.push(feature)
+                } else if (bbox.isContainedIn(this.upper_right.bbox)) {
+                    urf.push(feature)
+                } else if (bbox.isContainedIn(this.lower_left.bbox)) {
+                    llf.push(feature)
+                } else if (bbox.isContainedIn(this.lower_right.bbox)) {
+                    lrf.push(feature)
+                } else {
+                    overlapsboundary.push(feature)
+                }
+            } else {
+                // We duplicate a feature on a boundary into every tile as we need to get to the minZoomLevel
+                if (bbox.overlapsWith(this.upper_left.bbox)) {
+                    ulf.push(feature)
+                }
+                if (bbox.overlapsWith(this.upper_right.bbox)) {
+                    urf.push(feature)
+                }
+                if (bbox.overlapsWith(this.lower_left.bbox)) {
+                    llf.push(feature)
+                }
+                if (bbox.overlapsWith(this.lower_right.bbox)) {
+                    lrf.push(feature)
+                }
+            }
+        }
+        this.upper_left.addFeatures(ulf)
+        this.upper_right.addFeatures(urf)
+        this.lower_left.addFeatures(llf)
+        this.lower_right.addFeatures(lrf)
+        this.features.setData(overlapsboundary)
+
+    }
+}
+
+export interface TiledFeatureSourceOptions {
+    readonly maxFeatureCount?: number,
+    readonly maxZoomLevel?: number,
+    readonly minZoomLevel?: number,
+    readonly registerTile?: (tile: TiledFeatureSource & Tiled) => void,
+    readonly layer?: FilteredLayer
+}

Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts

@@ -1,40 +1,102 @@
 import FilteredLayer from "../../../Models/FilteredLayer";
-import {FeatureSourceForLayer} from "../FeatureSource";
+import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
 import {UIEventSource} from "../../UIEventSource";
 import Loc from "../../../Models/Loc";
-import GeoJsonSource from "../GeoJsonSource";
-import DynamicTileSource from "./DynamicTileSource";
+import TileHierarchy from "./TileHierarchy";
+import {Utils} from "../../../Utils";
+import LocalStorageSaverActor from "../Actors/LocalStorageSaverActor";
+import {BBox} from "../../GeoOperations";
+
+export default class TiledFromLocalStorageSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
+    public loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
 
-export default class DynamicGeoJsonTileSource extends DynamicTileSource {
     constructor(layer: FilteredLayer,
-                registerLayer: (layer: FeatureSourceForLayer) => void,
+                handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void,
                 state: {
                     locationControl: UIEventSource<Loc>
                     leafletMap: any
                 }) {
-        const source = layer.layerDef.source
-        if (source.geojsonZoomLevel === undefined) {
-            throw "Invalid layer: geojsonZoomLevel expected"
-        }
-        if (source.geojsonSource === undefined) {
-            throw "Invalid layer: geojsonSource expected"
-        }
 
-        super(
-            layer,
-            source.geojsonZoomLevel,
-            (xy) => {
-                const xyz: [number, number, number] = [xy[0], xy[1], source.geojsonZoomLevel]
-                const src = new GeoJsonSource(
-                    layer,
-                    xyz
-                )
-                registerLayer(src)
-                return src
-            },
-            state
-        );
+        const prefix = LocalStorageSaverActor.storageKey + "-" + layer.layerDef.id + "-"
+        // @ts-ignore
+        const indexes: number[] = Object.keys(localStorage)
+            .filter(key => {
+                return key.startsWith(prefix) && !key.endsWith("-time");
+            })
+            .map(key => {
+                return Number(key.substring(prefix.length));
+            })
+
+        console.log("Avaible datasets in local storage:", indexes)
+
+        const zLevels = indexes.map(i => i % 100)
+        const indexesSet = new Set(indexes)
+        const maxZoom = Math.max(...zLevels)
+        const minZoom = Math.min(...zLevels)
+        const self = this;
+
+        const neededTiles = state.locationControl.map(
+            location => {
+                if (!layer.isDisplayed.data) {
+                    // No need to download! - the layer is disabled
+                    return undefined;
+                }
+
+                if (location.zoom < layer.layerDef.minzoom) {
+                    // No need to download! - the layer is disabled
+                    return undefined;
+                }
+
+                // Yup, this is cheating to just get the bounds here
+                const bounds = state.leafletMap.data?.getBounds()
+                if (bounds === undefined) {
+                    // We'll retry later
+                    return undefined
+                }
+
+                const needed = []
+                for (let z = minZoom; z <= maxZoom; z++) {
+
+                    const tileRange = Utils.TileRangeBetween(z, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
+                    const neededZ = Utils.MapRange(tileRange, (x, y) => Utils.tile_index(z, x, y))
+                        .filter(i => !self.loadedTiles.has(i) && indexesSet.has(i))
+                    needed.push(...neededZ)
+                }
+
+                if (needed.length === 0) {
+                    return undefined
+                }
+                return needed
+            }
+            , [layer.isDisplayed, state.leafletMap]).stabilized(50);
+
+        neededTiles.addCallbackAndRun(t => console.log("Tiles to load from localstorage:", t))
+
+        neededTiles.addCallbackAndRunD(neededIndexes => {
+            for (const neededIndex of neededIndexes) {
+                // We load the features from localStorage
+                try {
+                    const key = LocalStorageSaverActor.storageKey + "-" + layer.layerDef.id + "-" + neededIndex
+                    const data = localStorage.getItem(key)
+                    const features = JSON.parse(data)
+                    const src = {
+                        layer: layer,
+                        features: new UIEventSource<{ feature: any; freshness: Date }[]>(features),
+                        name: "FromLocalStorage(" + key + ")",
+                        tileIndex: neededIndex,
+                        bbox: BBox.fromTile(...Utils.tile_from_index(neededIndex))
+                    }
+                    handleFeatureSource(src, neededIndex)
+                    self.loadedTiles.set(neededIndex, src)
+                } catch (e) {
+                    console.error("Could not load data tile from local storage due to", e)
+                }
+            }
+
+
+        })
 
     }
 
+
 }