midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

More refactoring, move minimap behind facade

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-09-21 02:10:42 +02:00
parent c11ff652b8
commit d5c1ba4cd1
79 changed files with 1848 additions and 1118 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Logic/FeatureSource/Actors/LocalStorageSaverActor.ts
View file

@ -1,11 +1,11 @@
import {FeatureSourceForLayer} from "./FeatureSource";
import {Utils} from "../../Utils";
/***
* Saves all the features that are passed in to localstorage, so they can be retrieved on the next run
*
* Technically, more an Actor then a featuresource, but it fits more neatly this ay
*/
import {FeatureSourceForLayer} from "../FeatureSource";
import {Utils} from "../../../Utils";
export default class LocalStorageSaverActor {
public static readonly storageKey: string = "cached-features";
@ -21,7 +21,6 @@ export default class LocalStorageSaverActor {
try {
localStorage.setItem(key, JSON.stringify(features));
console.log("Saved ", features.length, "elements to", key)
localStorage.setItem(key + "-time", JSON.stringify(now))
} catch (e) {
console.warn("Could not save the features to local storage:", e)

6
Logic/FeatureSource/Actors/RegisteringAllFromFeatureSourceActor.ts
View file

@ -1,6 +1,6 @@
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import State from "../../State";
import FeatureSource from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import State from "../../../State";
export default class RegisteringAllFromFeatureSourceActor {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;

58
Logic/FeatureSource/ChangeApplicator.ts
View file

@ -1,10 +1,35 @@
import FeatureSource from "./FeatureSource";
import FeatureSource, {IndexedFeatureSource} from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import {Changes} from "../Osm/Changes";
import {ChangeDescription} from "../Osm/Actions/ChangeDescription";
import {Utils} from "../../Utils";
import {OsmNode, OsmRelation, OsmWay} from "../Osm/OsmObject";
/**
* A feature source containing exclusively new elements
*/
export class NewGeometryChangeApplicatorFeatureSource implements FeatureSource{
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name: string = "newFeatures";
constructor(changes: Changes) {
const seenChanges = new Set<ChangeDescription>();
changes.pendingChanges.addCallbackAndRunD(changes => {
for (const change of changes) {
if(seenChanges.has(change)){
continue
}
seenChanges.add(change)
if(change.id < 0){
// This is a new object!
}
}
})
}
}
/**
* Applies changes from 'Changes' onto a featureSource
@ -12,10 +37,18 @@ import {OsmNode, OsmRelation, OsmWay} from "../Osm/OsmObject";
export default class ChangeApplicator implements FeatureSource {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name: string;
private readonly source: IndexedFeatureSource;
private readonly changes: Changes;
private readonly mode?: {
generateNewGeometries: boolean
};
constructor(source: FeatureSource, changes: Changes, mode?: {
constructor(source: IndexedFeatureSource, changes: Changes, mode?: {
generateNewGeometries: boolean
}) {
this.source = source;
this.changes = changes;
this.mode = mode;
this.name = "ChangesApplied(" + source.name + ")"
this.features = source.features
@ -26,7 +59,7 @@ export default class ChangeApplicator implements FeatureSource {
if (runningUpdate) {
return; // No need to ping again
}
ChangeApplicator.ApplyChanges(features, changes.pendingChanges.data, mode)
self.ApplyChanges()
seenChanges.clear()
})
@ -34,19 +67,20 @@ export default class ChangeApplicator implements FeatureSource {
runningUpdate = true;
changes = changes.filter(ch => !seenChanges.has(ch))
changes.forEach(c => seenChanges.add(c))
ChangeApplicator.ApplyChanges(self.features.data, changes, mode)
self.ApplyChanges()
source.features.ping()
runningUpdate = false;
})
}
/**
* Returns true if the geometry is changed and the source should be pinged
*/
private static ApplyChanges(features: { feature: any; freshness: Date }[], cs: ChangeDescription[], mode: { generateNewGeometries: boolean }): boolean {
private ApplyChanges(): boolean {
const cs = this.changes.pendingChanges.data
const features = this.source.features.data
const loadedIds = this.source.containedIds
if (cs.length === 0 || features === undefined) {
return;
}
@ -56,12 +90,18 @@ export default class ChangeApplicator implements FeatureSource {
const changesPerId: Map<string, ChangeDescription[]> = new Map<string, ChangeDescription[]>()
for (const c of cs) {
const id = c.type + "/" + c.id
if (!loadedIds.has(id)) {
continue
}
if (!changesPerId.has(id)) {
changesPerId.set(id, [])
}
changesPerId.get(id).push(c)
}
if (changesPerId.size === 0) {
// The current feature source set doesn't contain any changed feature, so we can safely skip
return;
}
const now = new Date()
@ -77,7 +117,7 @@ export default class ChangeApplicator implements FeatureSource {
// First, create the new features - they have a negative ID
// We don't set the properties yet though
if (mode?.generateNewGeometries) {
if (this.mode?.generateNewGeometries) {
changesPerId.forEach(cs => {
cs
.forEach(change => {

250
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -1,95 +1,191 @@
import FilteringFeatureSource from "../FeatureSource/FilteringFeatureSource";
import FeatureSourceMerger from "../FeatureSource/FeatureSourceMerger";
import RememberingSource from "../FeatureSource/RememberingSource";
import WayHandlingApplyingFeatureSource from "../FeatureSource/WayHandlingApplyingFeatureSource";
import FeatureDuplicatorPerLayer from "../FeatureSource/FeatureDuplicatorPerLayer";
import FeatureSource from "../FeatureSource/FeatureSource";
import {UIEventSource} from "../UIEventSource";
import LocalStorageSaver from "./LocalStorageSaver";
import LocalStorageSource from "./LocalStorageSource";
import Loc from "../../Models/Loc";
import GeoJsonSource from "./GeoJsonSource";
import MetaTaggingFeatureSource from "./MetaTaggingFeatureSource";
import RegisteringFeatureSource from "./RegisteringFeatureSource";
import FilteredLayer from "../../Models/FilteredLayer";
import {Changes} from "../Osm/Changes";
import ChangeApplicator from "./ChangeApplicator";
import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig";
import FilteringFeatureSource from "./Sources/FilteringFeatureSource";
import PerLayerFeatureSourceSplitter from "./PerLayerFeatureSourceSplitter";
import FeatureSource, {FeatureSourceForLayer, FeatureSourceState, Tiled} from "./FeatureSource";
import TiledFeatureSource from "./TiledFeatureSource/TiledFeatureSource";
import {UIEventSource} from "../UIEventSource";
import {TileHierarchyTools} from "./TiledFeatureSource/TileHierarchy";
import FilteredLayer from "../../Models/FilteredLayer";
import MetaTagging from "../MetaTagging";
import RememberingSource from "./Sources/RememberingSource";
import OverpassFeatureSource from "../Actors/OverpassFeatureSource";
import {Changes} from "../Osm/Changes";
import GeoJsonSource from "./Sources/GeoJsonSource";
import Loc from "../../Models/Loc";
import WayHandlingApplyingFeatureSource from "./Sources/WayHandlingApplyingFeatureSource";
import RegisteringAllFromFeatureSourceActor from "./Actors/RegisteringAllFromFeatureSourceActor";
import {Utils} from "../../Utils";
import TiledFromLocalStorageSource from "./TiledFeatureSource/TiledFromLocalStorageSource";
import LocalStorageSaverActor from "./Actors/LocalStorageSaverActor";
import DynamicGeoJsonTileSource from "./TiledFeatureSource/DynamicGeoJsonTileSource";
import {BBox} from "../GeoOperations";
import {TileHierarchyMerger} from "./TiledFeatureSource/TileHierarchyMerger";
import RelationsTracker from "../Osm/RelationsTracker";
export default class FeaturePipeline implements FeatureSource {
public features: UIEventSource<{ feature: any; freshness: Date }[]>;
export default class FeaturePipeline implements FeatureSourceState {
public readonly name = "FeaturePipeline"
public readonly sufficientlyZoomed: UIEventSource<boolean>;
public readonly runningQuery: UIEventSource<boolean>;
public readonly timeout: UIEventSource<number>;
public readonly somethingLoaded: UIEventSource<boolean> = new UIEventSource<boolean>(false)
constructor(flayers: UIEventSource<FilteredLayer[]>,
changes: Changes,
updater: FeatureSource,
fromOsmApi: FeatureSource,
layout: UIEventSource<LayoutConfig>,
locationControl: UIEventSource<Loc>,
selectedElement: UIEventSource<any>) {
private readonly overpassUpdater: OverpassFeatureSource
private readonly relationTracker: RelationsTracker
private readonly perLayerHierarchy: Map<string, TileHierarchyMerger>;
constructor(
handleFeatureSource: (source: FeatureSourceForLayer) => void,
state: {
osmApiFeatureSource: FeatureSource,
filteredLayers: UIEventSource<FilteredLayer[]>,
locationControl: UIEventSource<Loc>,
selectedElement: UIEventSource<any>,
changes: Changes,
layoutToUse: UIEventSource<LayoutConfig>,
leafletMap: any,
readonly overpassUrl: UIEventSource<string>;
readonly overpassTimeout: UIEventSource<number>;
readonly overpassMaxZoom: UIEventSource<number>;
}) {
const allLoadedFeatures = new UIEventSource<{ feature: any; freshness: Date }[]>([])
const self = this
const updater = new OverpassFeatureSource(state);
this.overpassUpdater = updater;
this.sufficientlyZoomed = updater.sufficientlyZoomed
this.runningQuery = updater.runningQuery
this.timeout = updater.timeout
this.relationTracker = updater.relationsTracker
// Register everything in the state' 'AllElements'
new RegisteringAllFromFeatureSourceActor(updater)
// first we metatag, then we save to get the metatags into storage too
// Note that we need to register before we do metatagging (as it expects the event sources)
const perLayerHierarchy = new Map<string, TileHierarchyMerger>()
this.perLayerHierarchy = perLayerHierarchy
// AT last, the metaTagging also needs to be run _after_ the duplicatorPerLayer
const amendedOverpassSource =
new RememberingSource(
new LocalStorageSaver(
new MetaTaggingFeatureSource(allLoadedFeatures,
new FeatureDuplicatorPerLayer(flayers,
new RegisteringFeatureSource(
new ChangeApplicator(
updater, changes
))
)), layout));
const patchedHandleFeatureSource = function (src: FeatureSourceForLayer) {
// This will already contain the merged features for this tile. In other words, this will only be triggered once for every tile
const srcFiltered =
new FilteringFeatureSource(state,
new WayHandlingApplyingFeatureSource(
src,
)
)
handleFeatureSource(srcFiltered)
self.somethingLoaded.setData(true)
};
const geojsonSources: FeatureSource [] = GeoJsonSource
.ConstructMultiSource(flayers.data, locationControl)
.map(geojsonSource => {
let source = new RegisteringFeatureSource(
new FeatureDuplicatorPerLayer(flayers,
new ChangeApplicator(geojsonSource, changes)));
if (!geojsonSource.isOsmCache) {
source = new MetaTaggingFeatureSource(allLoadedFeatures, source, updater.features);
}
return source
});
function addToHierarchy(src: FeatureSource & Tiled, layerId: string) {
perLayerHierarchy.get(layerId).registerTile(src)
}
const amendedLocalStorageSource =
new RememberingSource(new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, new ChangeApplicator(new LocalStorageSource(layout), changes))
));
for (const filteredLayer of state.filteredLayers.data) {
const hierarchy = new TileHierarchyMerger(filteredLayer, (tile, _) => patchedHandleFeatureSource(tile))
const id = filteredLayer.layerDef.id
perLayerHierarchy.set(id, hierarchy)
const source = filteredLayer.layerDef.source
const amendedOsmApiSource = new RememberingSource(
new MetaTaggingFeatureSource(allLoadedFeatures,
new FeatureDuplicatorPerLayer(flayers,
new RegisteringFeatureSource(new ChangeApplicator(fromOsmApi, changes,
{
// We lump in the new points here
generateNewGeometries: true
if (source.geojsonSource === undefined) {
// This is an OSM layer
// We load the cached values and register them
// Getting data from upstream happens a bit lower
new TiledFromLocalStorageSource(filteredLayer,
(src) => {
new RegisteringAllFromFeatureSourceActor(src)
hierarchy.registerTile(src);
}, state)
continue
}
if (source.geojsonZoomLevel === undefined) {
// This is a 'load everything at once' geojson layer
// We split them up into tiles
const src = new GeoJsonSource(filteredLayer)
TiledFeatureSource.createHierarchy(src, {
layer: src.layer,
registerTile: (tile) => {
new RegisteringAllFromFeatureSourceActor(tile)
addToHierarchy(tile, id)
}
})
} else {
new DynamicGeoJsonTileSource(
filteredLayer,
src => TiledFeatureSource.createHierarchy(src, {
layer: src.layer,
registerTile: (tile) => {
new RegisteringAllFromFeatureSourceActor(tile)
addToHierarchy(tile, id)
}
)))));
}),
state
)
}
const merged =
new FeatureSourceMerger([
amendedOverpassSource,
amendedOsmApiSource,
amendedLocalStorageSource,
...geojsonSources
]);
}
merged.features.syncWith(allLoadedFeatures)
// Actually load data from the overpass source
new PerLayerFeatureSourceSplitter(state.filteredLayers,
(source) => TiledFeatureSource.createHierarchy(source, {
layer: source.layer,
registerTile: (tile) => {
// We save the tile data for the given layer to local storage
const [z, x, y] = Utils.tile_from_index(tile.tileIndex)
new LocalStorageSaverActor(tile, x, y, z)
addToHierarchy(tile, source.layer.layerDef.id);
}
}), new RememberingSource(updater))
// Whenever fresh data comes in, we need to update the metatagging
updater.features.addCallback(_ => {
self.updateAllMetaTagging()
})
this.features = new WayHandlingApplyingFeatureSource(flayers,
new FilteringFeatureSource(
flayers,
locationControl,
selectedElement,
merged
)).features;
}
private updateAllMetaTagging() {
console.log("Updating the meta tagging")
const self = this;
this.perLayerHierarchy.forEach(hierarchy => {
hierarchy.loadedTiles.forEach(src => {
MetaTagging.addMetatags(
src.features.data,
{
memberships: this.relationTracker,
getFeaturesWithin: (layerId, bbox: BBox) => self.GetFeaturesWithin(layerId, bbox)
},
src.layer.layerDef
)
})
})
}
public GetAllFeaturesWithin(bbox: BBox): any[][]{
const self = this
const tiles = []
Array.from(this.perLayerHierarchy.keys())
.forEach(key => tiles.push(...self.GetFeaturesWithin(key, bbox)))
return tiles;
}
public GetFeaturesWithin(layerId: string, bbox: BBox): any[][]{
const requestedHierarchy = this.perLayerHierarchy.get(layerId)
if (requestedHierarchy === undefined) {
return undefined;
}
return TileHierarchyTools.getTiles(requestedHierarchy, bbox)
.filter(featureSource => featureSource.features?.data !== undefined)
.map(featureSource => featureSource.features.data.map(fs => fs.feature))
}
public GetTilesPerLayerWithin(bbox: BBox, handleTile: (tile: FeatureSourceForLayer & Tiled) => void){
Array.from(this.perLayerHierarchy.values()).forEach(hierarchy => {
TileHierarchyTools.getTiles(hierarchy, bbox).forEach(handleTile)
})
}
public ForceRefresh() {
this.overpassUpdater.ForceRefresh()
}
}

58
Logic/FeatureSource/FeatureSource.ts
View file

@ -1,5 +1,7 @@
import {UIEventSource} from "../UIEventSource";
import {Utils} from "../../Utils";
import FilteredLayer from "../../Models/FilteredLayer";
import {BBox} from "../GeoOperations";
export default interface FeatureSource {
features: UIEventSource<{ feature: any, freshness: Date }[]>;
@ -9,38 +11,30 @@ export default interface FeatureSource {
name: string;
}
export class FeatureSourceUtils {
export interface Tiled {
tileIndex: number,
bbox: BBox
}
/**
* Exports given featurePipeline as a geojson FeatureLists (downloads as a json)
* @param featurePipeline The FeaturePipeline you want to export
* @param options The options object
* @param options.metadata True if you want to include the MapComplete metadata, false otherwise
*/
public static extractGeoJson(featurePipeline: FeatureSource, options: { metadata?: boolean } = {}) {
let defaults = {
metadata: false,
}
options = Utils.setDefaults(options, defaults);
/**
* A feature source which only contains features for the defined layer
*/
export interface FeatureSourceForLayer extends FeatureSource{
readonly layer: FilteredLayer
}
// Select all features, ignore the freshness and other data
let featureList: any[] = featurePipeline.features.data.map((feature) =>
JSON.parse(JSON.stringify((feature.feature)))); // Make a deep copy!
/**
* A feature source which is aware of the indexes it contains
*/
export interface IndexedFeatureSource extends FeatureSource {
readonly containedIds: UIEventSource<Set<string>>
}
if (!options.metadata) {
for (let i = 0; i < featureList.length; i++) {
let feature = featureList[i];
for (let property in feature.properties) {
if (property[0] == "_" && property !== "_lat" && property !== "_lon") {
delete featureList[i]["properties"][property];
}
}
}
}
return {type: "FeatureCollection", features: featureList}
}
}
/**
* A feature source which has some extra data about it's state
*/
export interface FeatureSourceState {
readonly sufficientlyZoomed: UIEventSource<boolean>;
readonly runningQuery: UIEventSource<boolean>;
readonly timeout: UIEventSource<number>;
}

26
Logic/FeatureSource/PerLayerFeatureSourceSplitter.ts
View file

@ -1,8 +1,7 @@
import FeatureSource from "./FeatureSource";
import FeatureSource, {FeatureSourceForLayer} from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import FilteredLayer from "../../Models/FilteredLayer";
import OverpassFeatureSource from "../Actors/OverpassFeatureSource";
import SimpleFeatureSource from "./SimpleFeatureSource";
import SimpleFeatureSource from "./Sources/SimpleFeatureSource";
/**
@ -13,17 +12,17 @@ import SimpleFeatureSource from "./SimpleFeatureSource";
export default class PerLayerFeatureSourceSplitter {
constructor(layers: UIEventSource<FilteredLayer[]>,
handleLayerData: (source: FeatureSource) => void,
upstream: OverpassFeatureSource) {
handleLayerData: (source: FeatureSourceForLayer) => void,
upstream: FeatureSource) {
const knownLayers = new Map<string, FeatureSource>()
const knownLayers = new Map<string, FeatureSourceForLayer>()
function update() {
const features = upstream.features.data;
if (features === undefined) {
return;
}
if(layers.data === undefined){
if (layers.data === undefined) {
return;
}
@ -69,19 +68,16 @@ export default class PerLayerFeatureSourceSplitter {
if (featureSource === undefined) {
// Not yet initialized - now is a good time
featureSource = new SimpleFeatureSource(layer)
featureSource.features.setData(features)
knownLayers.set(id, featureSource)
handleLayerData(featureSource)
} else {
featureSource.features.setData(features)
}
featureSource.features.setData(features)
}
upstream.features.addCallbackAndRunD(_ => update())
layers.addCallbackAndRunD(_ => update())
}
layers.addCallbackAndRunD(_ => update())
layers.addCallback(_ => update())
upstream.features.addCallbackAndRunD(_ => update())
}
}

20
Logic/FeatureSource/Sources/FeatureSourceMerger.ts
View file

@ -1,22 +1,28 @@
import FeatureSource, {FeatureSourceForLayer} from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import FilteredLayer from "../../Models/FilteredLayer";
/**
* Merges features from different featureSources for a single layer
* Uses the freshest feature available in the case multiple sources offer data with the same identifier
*/
export default class FeatureSourceMerger implements FeatureSourceForLayer {
import {UIEventSource} from "../../UIEventSource";
import FeatureSource, {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import FilteredLayer from "../../../Models/FilteredLayer";
import {BBox} from "../../GeoOperations";
import {Utils} from "../../../Utils";
export default class FeatureSourceMerger implements FeatureSourceForLayer, Tiled {
public features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]);
public readonly name;
public readonly layer: FilteredLayer
private readonly _sources: UIEventSource<FeatureSource[]>;
public readonly tileIndex: number;
public readonly bbox: BBox;
constructor(layer: FilteredLayer ,sources: UIEventSource<FeatureSource[]>) {
constructor(layer: FilteredLayer, tileIndex: number, bbox: BBox, sources: UIEventSource<FeatureSource[]>) {
this.tileIndex = tileIndex;
this.bbox = bbox;
this._sources = sources;
this.layer = layer;
this.name = "SourceMerger"
this.name = "FeatureSourceMerger("+layer.layerDef.id+", "+Utils.tile_from_index(tileIndex).join(",")+")"
const self = this;
const handledSources = new Set<FeatureSource>();

13
Logic/FeatureSource/Sources/FilteringFeatureSource.ts
View file

@ -1,13 +1,13 @@
import {FeatureSourceForLayer} from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import Hash from "../Web/Hash";
import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
import FilteredLayer from "../../Models/FilteredLayer";
import {UIEventSource} from "../../UIEventSource";
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig";
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer} from "../FeatureSource";
import Hash from "../../Web/Hash";
export default class FilteringFeatureSource implements FeatureSourceForLayer {
public features: UIEventSource<{ feature: any; freshness: Date }[]> =
new UIEventSource<{ feature: any; freshness: Date }[]>([]);
public readonly name = "FilteringFeatureSource";
public readonly name;
public readonly layer: FilteredLayer;
constructor(
@ -18,6 +18,7 @@ export default class FilteringFeatureSource implements FeatureSourceForLayer {
upstream: FeatureSourceForLayer
) {
const self = this;
this.name = "FilteringFeatureSource("+upstream.name+")"
this.layer = upstream.layer;
const layer = upstream.layer;

30
Logic/FeatureSource/Sources/GeoJsonSource.ts
View file

@ -1,14 +1,14 @@
import {FeatureSourceForLayer} from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import {Utils} from "../../Utils";
import FilteredLayer from "../../Models/FilteredLayer";
import {control} from "leaflet";
/**
* Fetches a geojson file somewhere and passes it along
*/
export default class GeoJsonSource implements FeatureSourceForLayer {
import {UIEventSource} from "../../UIEventSource";
import FilteredLayer from "../../../Models/FilteredLayer";
import {Utils} from "../../../Utils";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {BBox} from "../../GeoOperations";
export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name;
@ -17,6 +17,8 @@ export default class GeoJsonSource implements FeatureSourceForLayer {
private readonly seenids: Set<string> = new Set<string>()
public readonly layer: FilteredLayer;
public readonly tileIndex
public readonly bbox;
public constructor(flayer: FilteredLayer,
zxy?: [number, number, number]) {
@ -28,10 +30,16 @@ export default class GeoJsonSource implements FeatureSourceForLayer {
this.layer = flayer;
let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id);
if (zxy !== undefined) {
const [z, x, y] = zxy;
url = url
.replace('{z}', "" + zxy[0])
.replace('{x}', "" + zxy[1])
.replace('{y}', "" + zxy[2])
.replace('{z}', "" + z)
.replace('{x}', "" + x)
.replace('{y}', "" + y)
this.tileIndex = Utils.tile_index(z, x, y)
this.bbox = BBox.fromTile(z, x, y)
} else {
this.tileIndex = Utils.tile_index(0, 0, 0)
this.bbox = BBox.global;
}
this.name = "GeoJsonSource of " + url;

12
Logic/FeatureSource/Sources/OsmApiFeatureSource.ts
View file

@ -1,9 +1,9 @@
import FeatureSource from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import {OsmObject} from "../Osm/OsmObject";
import {Utils} from "../../Utils";
import Loc from "../../Models/Loc";
import FilteredLayer from "../../Models/FilteredLayer";
import FeatureSource from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import Loc from "../../../Models/Loc";
import FilteredLayer from "../../../Models/FilteredLayer";
import {Utils} from "../../../Utils";
import {OsmObject} from "../../Osm/OsmObject";
export default class OsmApiFeatureSource implements FeatureSource {

7
Logic/FeatureSource/Sources/RememberingSource.ts
View file

@ -1,11 +1,10 @@
import FeatureSource, {FeatureSourceForLayer} from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import FilteredLayer from "../../Models/FilteredLayer";
/**
* Every previously added point is remembered, but new points are added.
* Data coming from upstream will always overwrite a previous value
*/
import FeatureSource from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
export default class RememberingSource implements FeatureSource {
public readonly features: UIEventSource<{ feature: any, freshness: Date }[]>;

6
Logic/FeatureSource/Sources/SimpleFeatureSource.ts
View file

@ -1,6 +1,6 @@
import {FeatureSourceForLayer} from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import FilteredLayer from "../../Models/FilteredLayer";
import {UIEventSource} from "../../UIEventSource";
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer} from "../FeatureSource";
export default class SimpleFeatureSource implements FeatureSourceForLayer {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]);

17
Logic/FeatureSource/Sources/StaticFeatureSource.ts
View file

@ -8,12 +8,19 @@ export default class StaticFeatureSource implements FeatureSource {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name: string = "StaticFeatureSource"
constructor(features: any[]) {
constructor(features: any[] | UIEventSource<any[]>, useFeaturesDirectly = false) {
const now = new Date();
this.features = new UIEventSource(features.map(f => ({
feature: f,
freshness: now
})))
if(useFeaturesDirectly){
// @ts-ignore
this.features = features
}else if (features instanceof UIEventSource) {
this.features = features.map(features => features.map(f => ({feature: f, freshness: now})))
} else {
this.features = new UIEventSource(features.map(f => ({
feature: f,
freshness: now
})))
}
}

12
Logic/FeatureSource/Sources/WayHandlingApplyingFeatureSource.ts
View file

@ -1,18 +1,18 @@
import {FeatureSourceForLayer} from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import {GeoOperations} from "../GeoOperations";
import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
/**
* This is the part of the pipeline which introduces extra points at the center of an area (but only if this is demanded by the wayhandling)
*/
import {UIEventSource} from "../../UIEventSource";
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig";
import {GeoOperations} from "../../GeoOperations";
import {FeatureSourceForLayer} from "../FeatureSource";
export default class WayHandlingApplyingFeatureSource implements FeatureSourceForLayer {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name;
public readonly layer;
constructor(upstream: FeatureSourceForLayer) {
this.name = "Wayhandling of " + upstream.name;
this.name = "Wayhandling(" + upstream.name+")";
this.layer = upstream.layer
const layer = upstream.layer.layerDef;

63
Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts
View file

@ -0,0 +1,63 @@
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import Loc from "../../../Models/Loc";
import DynamicTileSource from "./DynamicTileSource";
import {Utils} from "../../../Utils";
import GeoJsonSource from "../Sources/GeoJsonSource";
export default class DynamicGeoJsonTileSource extends DynamicTileSource {
constructor(layer: FilteredLayer,
registerLayer: (layer: FeatureSourceForLayer) => void,
state: {
locationControl: UIEventSource<Loc>
leafletMap: any
}) {
const source = layer.layerDef.source
if (source.geojsonZoomLevel === undefined) {
throw "Invalid layer: geojsonZoomLevel expected"
}
if (source.geojsonSource === undefined) {
throw "Invalid layer: geojsonSource expected"
}
const whitelistUrl = source.geojsonSource.replace("{z}_{x}_{y}.geojson", "overview.json")
.replace("{layer}",layer.layerDef.id)
let whitelist = undefined
Utils.downloadJson(whitelistUrl).then(
json => {
const data = new Map<number, Set<number>>();
for (const x in json) {
data.set(Number(x), new Set(json[x]))
}
whitelist = data
}
).catch(err => {
console.warn("No whitelist found for ", layer.layerDef.id, err)
})
super(
layer,
source.geojsonZoomLevel,
(zxy) => {
if(whitelist !== undefined){
const isWhiteListed = whitelist.get(zxy[1])?.has(zxy[2])
if(!isWhiteListed){
return undefined;
}
}
const src = new GeoJsonSource(
layer,
zxy
)
registerLayer(src)
return src
},
state
);
}
}

42
Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts
View file

@ -1,22 +1,24 @@
/***
* A tiled source which dynamically loads the required tiles
*/
import State from "../../../State";
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer} from "../FeatureSource";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {Utils} from "../../../Utils";
import {UIEventSource} from "../../UIEventSource";
import Loc from "../../../Models/Loc";
import TileHierarchy from "./TileHierarchy";
export default class DynamicTileSource {
/***
* A tiled source which dynamically loads the required tiles at a fixed zoom level
*/
export default class DynamicTileSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
private readonly _loadedTiles = new Set<number>();
public readonly existingTiles: Map<number, Map<number, FeatureSourceForLayer>> = new Map<number, Map<number, FeatureSourceForLayer>>()
public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled>;
constructor(
layer: FilteredLayer,
zoomlevel: number,
constructTile: (xy: [number, number]) => FeatureSourceForLayer,
constructTile: (zxy: [number, number, number]) => (FeatureSourceForLayer & Tiled),
state: {
locationControl: UIEventSource<Loc>
leafletMap: any
@ -24,6 +26,8 @@ export default class DynamicTileSource {
) {
state = State.state
const self = this;
this.loadedTiles = new Map<number,FeatureSourceForLayer & Tiled>()
const neededTiles = state.locationControl.map(
location => {
if (!layer.isDisplayed.data) {
@ -45,28 +49,30 @@ export default class DynamicTileSource {
const tileRange = Utils.TileRangeBetween(zoomlevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const needed = Utils.MapRange(tileRange, (x, y) => Utils.tile_index(zoomlevel, x, y)).filter(i => !self._loadedTiles.has(i))
if(needed.length === 0){
if (needed.length === 0) {
return undefined
}
return needed
}
, [layer.isDisplayed, state.leafletMap]).stabilized(250);
neededTiles.addCallbackAndRunD(neededIndexes => {
console.log("Tiled geojson source ",layer.layerDef.id," needs", neededIndexes)
if (neededIndexes === undefined) {
return;
}
for (const neededIndex of neededIndexes) {
self._loadedTiles.add(neededIndex)
const xy = Utils.tile_from_index(zoomlevel, neededIndex)
const src = constructTile(xy)
let xmap = self.existingTiles.get(xy[0])
if(xmap === undefined){
xmap = new Map<number, FeatureSourceForLayer>()
self.existingTiles.set(xy[0], xmap)
const src = constructTile( Utils.tile_from_index(neededIndex))
if(src !== undefined){
self.loadedTiles.set(neededIndex, src)
}
xmap.set(xy[1], src)
}
})
}
}
}

26
Logic/FeatureSource/TiledFeatureSource/README.md
View file

@ -1,3 +1,27 @@
Data in MapComplete can come from multiple sources.
In order to keep thins snappy, they are distributed over a tiled database
Currently, they are:
- The Overpass-API
- The OSM-API
- One or more GeoJSON files. This can be a single file or a set of tiled geojson files
- LocalStorage, containing features from a previous visit
- Changes made by the user introducing new features
When the data enters from Overpass or from the OSM-API, they are first distributed per layer:
OVERPASS | ---PerLayerFeatureSource---> FeatureSourceForLayer[]
OSM |
The GeoJSon files (not tiled) are then added to this list
A single FeatureSourcePerLayer is then further handled by splitting it into a tile hierarchy.
In order to keep thins snappy, they are distributed over a tiled database per layer.
## Notes
`cached-featuresbookcases` is the old key used `cahced-features{themeid}` and should be cleaned up

25
Logic/FeatureSource/TiledFeatureSource/TileHierarchy.ts
View file

@ -0,0 +1,25 @@
import FeatureSource, {Tiled} from "../FeatureSource";
import {BBox} from "../../GeoOperations";
export default interface TileHierarchy<T extends FeatureSource & Tiled> {
/**
* A mapping from 'tile_index' to the actual tile featrues
*/
loadedTiles: Map<number, T>
}
export class TileHierarchyTools {
public static getTiles<T extends FeatureSource & Tiled>(hierarchy: TileHierarchy<T>, bbox: BBox): T[] {
const result = []
hierarchy.loadedTiles.forEach((tile) => {
if (tile.bbox.overlapsWith(bbox)) {
result.push(tile)
}
})
return result;
}
}

17
Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts
View file

@ -1,10 +1,10 @@
import TileHierarchy from "./TiledFeatureSource/TileHierarchy";
import FeatureSource, {FeatureSourceForLayer, Tiled} from "./FeatureSource";
import {UIEventSource} from "../UIEventSource";
import FilteredLayer from "../../Models/FilteredLayer";
import FeatureSourceMerger from "./Sources/FeatureSourceMerger";
import {BBox} from "../GeoOperations";
import {Utils} from "../../Utils";
import TileHierarchy from "./TileHierarchy";
import {UIEventSource} from "../../UIEventSource";
import FeatureSource, {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import FilteredLayer from "../../../Models/FilteredLayer";
import {Utils} from "../../../Utils";
import {BBox} from "../../GeoOperations";
import FeatureSourceMerger from "../Sources/FeatureSourceMerger";
export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
@ -24,8 +24,9 @@ export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer
* @param src
* @param index
*/
public registerTile(src: FeatureSource, index: number) {
public registerTile(src: FeatureSource & Tiled) {
const index = src.tileIndex
if (this.sources.has(index)) {
const sources = this.sources.get(index)
sources.data.push(src)

191
Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource.ts
View file

@ -0,0 +1,191 @@
import FeatureSource, {FeatureSourceForLayer, IndexedFeatureSource, Tiled} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import {Utils} from "../../../Utils";
import {BBox} from "../../GeoOperations";
import FilteredLayer from "../../../Models/FilteredLayer";
import TileHierarchy from "./TileHierarchy";
import {feature} from "@turf/turf";
/**
* Contains all features in a tiled fashion.
* The data will be automatically broken down into subtiles when there are too much features in a single tile or if the zoomlevel is too high
*/
export default class TiledFeatureSource implements Tiled, IndexedFeatureSource, FeatureSourceForLayer, TileHierarchy<IndexedFeatureSource & FeatureSourceForLayer & Tiled> {
public readonly z: number;
public readonly x: number;
public readonly y: number;
public readonly parent: TiledFeatureSource;
public readonly root: TiledFeatureSource
public readonly layer: FilteredLayer;
/* An index of all known tiles. allTiles[z][x][y].get('layerid') will yield the corresponding tile.
* Only defined on the root element!
*/
public readonly loadedTiles: Map<number, TiledFeatureSource & FeatureSourceForLayer> = undefined;
public readonly maxFeatureCount: number;
public readonly name;
public readonly features: UIEventSource<{ feature: any, freshness: Date }[]>
public readonly containedIds: UIEventSource<Set<string>>
public readonly bbox: BBox;
private upper_left: TiledFeatureSource
private upper_right: TiledFeatureSource
private lower_left: TiledFeatureSource
private lower_right: TiledFeatureSource
private readonly maxzoom: number;
private readonly options: TiledFeatureSourceOptions
public readonly tileIndex: number;
private constructor(z: number, x: number, y: number, parent: TiledFeatureSource, options?: TiledFeatureSourceOptions) {
this.z = z;
this.x = x;
this.y = y;
this.bbox = BBox.fromTile(z, x, y)
this.tileIndex = Utils.tile_index(z, x, y)
this.name = `TiledFeatureSource(${z},${x},${y})`
this.parent = parent;
this.layer = options.layer
options = options ?? {}
this.maxFeatureCount = options?.maxFeatureCount ?? 500;
this.maxzoom = options.maxZoomLevel ?? 18
this.options = options;
if (parent === undefined) {
throw "Parent is not allowed to be undefined. Use null instead"
}
if (parent === null && z !== 0 && x !== 0 && y !== 0) {
throw "Invalid root tile: z, x and y should all be null"
}
if (parent === null) {
this.root = this;
this.loadedTiles = new Map()
} else {
this.root = this.parent.root;
this.loadedTiles = this.root.loadedTiles;
const i = Utils.tile_index(z, x, y)
this.root.loadedTiles.set(i, this)
}
this.features = new UIEventSource<any[]>([])
this.containedIds = this.features.map(features => {
if (features === undefined) {
return undefined;
}
return new Set(features.map(f => f.feature.properties.id))
})
// We register this tile, but only when there is some data in it
if (this.options.registerTile !== undefined) {
this.features.addCallbackAndRunD(features => {
if (features.length === 0) {
return;
}
this.options.registerTile(this)
return true;
})
}
}
public static createHierarchy(features: FeatureSource, options?: TiledFeatureSourceOptions): TiledFeatureSource {
const root = new TiledFeatureSource(0, 0, 0, null, options)
features.features?.addCallbackAndRunD(feats => root.addFeatures(feats))
return root;
}
private isSplitNeeded(featureCount: number){
if(this.upper_left !== undefined){
// This tile has been split previously, so we keep on splitting
return true;
}
if(this.z >= this.maxzoom){
// We are not allowed to split any further
return false
}
if(this.options.minZoomLevel !== undefined && this.z < this.options.minZoomLevel){
// We must have at least this zoom level before we are allowed to start splitting
return true
}
// To much features - we split
return featureCount > this.maxFeatureCount
}
/***
* Adds the list of features to this hierarchy.
* If there are too much features, the list will be broken down and distributed over the subtiles (only retaining features that don't fit a subtile on this level)
* @param features
* @private
*/
private addFeatures(features: { feature: any, freshness: Date }[]) {
if (features === undefined || features.length === 0) {
return;
}
if (!this.isSplitNeeded(features.length)) {
this.features.setData(features)
return;
}
if (this.upper_left === undefined) {
this.upper_left = new TiledFeatureSource(this.z + 1, this.x * 2, this.y * 2, this, this.options)
this.upper_right = new TiledFeatureSource(this.z + 1, this.x * 2 + 1, this.y * 2, this, this.options)
this.lower_left = new TiledFeatureSource(this.z + 1, this.x * 2, this.y * 2 + 1, this, this.options)
this.lower_right = new TiledFeatureSource(this.z + 1, this.x * 2 + 1, this.y * 2 + 1, this, this.options)
}
const ulf = []
const urf = []
const llf = []
const lrf = []
const overlapsboundary = []
for (const feature of features) {
const bbox = BBox.get(feature.feature)
if (this.options.minZoomLevel === undefined) {
if (bbox.isContainedIn(this.upper_left.bbox)) {
ulf.push(feature)
} else if (bbox.isContainedIn(this.upper_right.bbox)) {
urf.push(feature)
} else if (bbox.isContainedIn(this.lower_left.bbox)) {
llf.push(feature)
} else if (bbox.isContainedIn(this.lower_right.bbox)) {
lrf.push(feature)
} else {
overlapsboundary.push(feature)
}
} else {
// We duplicate a feature on a boundary into every tile as we need to get to the minZoomLevel
if (bbox.overlapsWith(this.upper_left.bbox)) {
ulf.push(feature)
}
if (bbox.overlapsWith(this.upper_right.bbox)) {
urf.push(feature)
}
if (bbox.overlapsWith(this.lower_left.bbox)) {
llf.push(feature)
}
if (bbox.overlapsWith(this.lower_right.bbox)) {
lrf.push(feature)
}
}
}
this.upper_left.addFeatures(ulf)
this.upper_right.addFeatures(urf)
this.lower_left.addFeatures(llf)
this.lower_right.addFeatures(lrf)
this.features.setData(overlapsboundary)
}
}
export interface TiledFeatureSourceOptions {
readonly maxFeatureCount?: number,
readonly maxZoomLevel?: number,
readonly minZoomLevel?: number,
readonly registerTile?: (tile: TiledFeatureSource & Tiled) => void,
readonly layer?: FilteredLayer
}

114
Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts
View file

@ -1,40 +1,102 @@
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer} from "../FeatureSource";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import Loc from "../../../Models/Loc";
import GeoJsonSource from "../GeoJsonSource";
import DynamicTileSource from "./DynamicTileSource";
import TileHierarchy from "./TileHierarchy";
import {Utils} from "../../../Utils";
import LocalStorageSaverActor from "../Actors/LocalStorageSaverActor";
import {BBox} from "../../GeoOperations";
export default class TiledFromLocalStorageSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
export default class DynamicGeoJsonTileSource extends DynamicTileSource {
constructor(layer: FilteredLayer,
registerLayer: (layer: FeatureSourceForLayer) => void,
handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void,
state: {
locationControl: UIEventSource<Loc>
leafletMap: any
}) {
const source = layer.layerDef.source
if (source.geojsonZoomLevel === undefined) {
throw "Invalid layer: geojsonZoomLevel expected"
}
if (source.geojsonSource === undefined) {
throw "Invalid layer: geojsonSource expected"
}
super(
layer,
source.geojsonZoomLevel,
(xy) => {
const xyz: [number, number, number] = [xy[0], xy[1], source.geojsonZoomLevel]
const src = new GeoJsonSource(
layer,
xyz
)
registerLayer(src)
return src
},
state
);
const prefix = LocalStorageSaverActor.storageKey + "-" + layer.layerDef.id + "-"
// @ts-ignore
const indexes: number[] = Object.keys(localStorage)
.filter(key => {
return key.startsWith(prefix) && !key.endsWith("-time");
})
.map(key => {
return Number(key.substring(prefix.length));
})
console.log("Avaible datasets in local storage:", indexes)
const zLevels = indexes.map(i => i % 100)
const indexesSet = new Set(indexes)
const maxZoom = Math.max(...zLevels)
const minZoom = Math.min(...zLevels)
const self = this;
const neededTiles = state.locationControl.map(
location => {
if (!layer.isDisplayed.data) {
// No need to download! - the layer is disabled
return undefined;
}
if (location.zoom < layer.layerDef.minzoom) {
// No need to download! - the layer is disabled
return undefined;
}
// Yup, this is cheating to just get the bounds here
const bounds = state.leafletMap.data?.getBounds()
if (bounds === undefined) {
// We'll retry later
return undefined
}
const needed = []
for (let z = minZoom; z <= maxZoom; z++) {
const tileRange = Utils.TileRangeBetween(z, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const neededZ = Utils.MapRange(tileRange, (x, y) => Utils.tile_index(z, x, y))
.filter(i => !self.loadedTiles.has(i) && indexesSet.has(i))
needed.push(...neededZ)
}
if (needed.length === 0) {
return undefined
}
return needed
}
, [layer.isDisplayed, state.leafletMap]).stabilized(50);
neededTiles.addCallbackAndRun(t => console.log("Tiles to load from localstorage:", t))
neededTiles.addCallbackAndRunD(neededIndexes => {
for (const neededIndex of neededIndexes) {
// We load the features from localStorage
try {
const key = LocalStorageSaverActor.storageKey + "-" + layer.layerDef.id + "-" + neededIndex
const data = localStorage.getItem(key)
const features = JSON.parse(data)
const src = {
layer: layer,
features: new UIEventSource<{ feature: any; freshness: Date }[]>(features),
name: "FromLocalStorage(" + key + ")",
tileIndex: neededIndex,
bbox: BBox.fromTile(...Utils.tile_from_index(neededIndex))
}
handleFeatureSource(src, neededIndex)
self.loadedTiles.set(neededIndex, src)
} catch (e) {
console.error("Could not load data tile from local storage due to", e)
}
}
})
}
}