contrapunctus/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

More refactoring of the featurepipeline, introduction of fetching data from the OSM-API directly per tile, personal theme refactoring

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-09-28 17:30:48 +02:00
parent 0a9e7c0b36
commit 41a2a79fe9
48 changed files with 746 additions and 590 deletions
Download patch file Download diff file Expand all files Collapse all files

92
Logic/Actors/OverpassFeatureSource.ts
View file

@ -9,9 +9,10 @@ import {TagsFilter} from "../Tags/TagsFilter";
import SimpleMetaTagger from "../SimpleMetaTagger";
import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig";
import RelationsTracker from "../Osm/RelationsTracker";
import {BBox} from "../BBox";
export default class OverpassFeatureSource implements FeatureSource, FeatureSourceState {
export default class OverpassFeatureSource implements FeatureSource {
public readonly name = "OverpassFeatureSource"
@ -21,7 +22,6 @@ export default class OverpassFeatureSource implements FeatureSource, FeatureSour
public readonly features: UIEventSource<{ feature: any, freshness: Date }[]> = new UIEventSource<any[]>(undefined);
public readonly sufficientlyZoomed: UIEventSource<boolean>;
public readonly runningQuery: UIEventSource<boolean> = new UIEventSource<boolean>(false);
public readonly timeout: UIEventSource<number> = new UIEventSource<number>(0);
@ -40,10 +40,12 @@ export default class OverpassFeatureSource implements FeatureSource, FeatureSour
private readonly state: {
readonly locationControl: UIEventSource<Loc>,
readonly layoutToUse: UIEventSource<LayoutConfig>,
readonly leafletMap: any,
readonly overpassUrl: UIEventSource<string>;
readonly overpassTimeout: UIEventSource<number>;
readonly currentBounds :UIEventSource<BBox>
}
private readonly _isActive: UIEventSource<boolean>;
private _onUpdated?: (bbox: BBox, dataFreshness: Date) => void;
/**
* The most important layer should go first, as that one gets first pick for the questions
*/
@ -51,33 +53,24 @@ export default class OverpassFeatureSource implements FeatureSource, FeatureSour
state: {
readonly locationControl: UIEventSource<Loc>,
readonly layoutToUse: UIEventSource<LayoutConfig>,
readonly leafletMap: any,
readonly overpassUrl: UIEventSource<string>;
readonly overpassTimeout: UIEventSource<number>;
readonly overpassMaxZoom: UIEventSource<number>
}) {
readonly overpassMaxZoom: UIEventSource<number>,
readonly currentBounds :UIEventSource<BBox>
},
options?: {
isActive?: UIEventSource<boolean>,
onUpdated?: (bbox: BBox, freshness: Date) => void,
relationTracker: RelationsTracker}) {
this.state = state
this.relationsTracker = new RelationsTracker()
this._isActive = options.isActive;
this._onUpdated =options. onUpdated;
this.relationsTracker = options.relationTracker
const location = state.locationControl
const self = this;
this.sufficientlyZoomed = location.map(location => {
if (location?.zoom === undefined) {
return false;
}
let minzoom = Math.min(...state.layoutToUse.data.layers.map(layer => layer.minzoom ?? 18));
if (location.zoom < minzoom) {
return false;
}
const maxZoom = state.overpassMaxZoom.data
if (maxZoom !== undefined && location.zoom > maxZoom) {
return false;
}
return true;
}, [state.layoutToUse]
);
for (let i = 0; i < 25; i++) {
// This update removes all data on all layers -> erase the map on lower levels too
this._previousBounds.set(i, []);
@ -89,16 +82,11 @@ export default class OverpassFeatureSource implements FeatureSource, FeatureSour
location.addCallback(() => {
self.update()
});
state.leafletMap.addCallbackAndRunD(_ => {
self.update();
state.currentBounds.addCallback(_ => {
self.update()
})
}
public ForceRefresh() {
for (let i = 0; i < 25; i++) {
this._previousBounds.set(i, []);
}
this.update();
}
private GetFilter(): Overpass {
@ -152,24 +140,34 @@ export default class OverpassFeatureSource implements FeatureSource, FeatureSour
}
private update() {
this.updateAsync().then(_ => {
if(!this._isActive.data){
return;
}
const self = this
this.updateAsync().then(bboxAndDate => {
if(bboxAndDate === undefined || self._onUpdated === undefined){
return;
}
const [bbox, date] = bboxAndDate
self._onUpdated(bbox, date);
})
}
private async updateAsync(): Promise<void> {
private async updateAsync(): Promise<[BBox, Date]> {
if (this.runningQuery.data) {
console.log("Still running a query, not updating");
return;
return undefined;
}
if (this.timeout.data > 0) {
console.log("Still in timeout - not updating")
return;
return undefined;
}
const bounds = this.state.leafletMap.data?.getBounds()?.pad(this.state.layoutToUse.data.widenFactor);
const bounds = this.state.currentBounds.data?.pad(this.state.layoutToUse.data.widenFactor)?.expandToTileBounds(14);
if (bounds === undefined) {
return;
return undefined;
}
const n = Math.min(90, bounds.getNorth());
@ -178,13 +176,12 @@ export default class OverpassFeatureSource implements FeatureSource, FeatureSour
const w = Math.max(-180, bounds.getWest());
const queryBounds = {north: n, east: e, south: s, west: w};
const z = Math.floor(this.state.locationControl.data.zoom ?? 0);
const self = this;
const overpass = this.GetFilter();
if (overpass === undefined) {
return;
return undefined;
}
this.runningQuery.setData(true);
@ -195,15 +192,14 @@ export default class OverpassFeatureSource implements FeatureSource, FeatureSour
try {
[data, date] = await overpass.queryGeoJson(queryBounds)
console.log("Querying overpass is done", data)
} catch (e) {
console.error(`QUERY FAILED (retrying in ${5 * self.retries.data} sec) due to`, e);
self.retries.data++;
self.retries.ping();
console.error(`QUERY FAILED (retrying in ${5 * self.retries.data} sec) due to`, e);
self.timeout.setData(self.retries.data * 5);
self.runningQuery.setData(false);
while (self.timeout.data > 0) {
await Utils.waitFor(1000)
self.timeout.data--
@ -212,16 +208,20 @@ export default class OverpassFeatureSource implements FeatureSource, FeatureSour
}
} while (data === undefined);
const z = Math.floor(this.state.locationControl.data.zoom ?? 0);
self._previousBounds.get(z).push(queryBounds);
self.retries.setData(0);
try {
data.features.forEach(feature => SimpleMetaTagger.objectMetaInfo.applyMetaTagsOnFeature(feature, date));
self.features.setData(data.features.map(f => ({feature: f, freshness: date})));
return [bounds, date];
} catch (e) {
console.error("Got the overpass response, but could not process it: ", e, e.stack)
}finally {
self.runningQuery.setData(false);
}
self.runningQuery.setData(false);
}
@ -231,7 +231,7 @@ export default class OverpassFeatureSource implements FeatureSource, FeatureSour
return false;
}
const b = this.state.leafletMap.data.getBounds();
const b = this.state.currentBounds.data;
return b.getSouth() >= bounds.south &&
b.getNorth() <= bounds.north &&
b.getEast() <= bounds.east &&

158
Logic/BBox.ts Normal file
View file

@ -0,0 +1,158 @@
import * as turf from "@turf/turf";
import {TileRange, Tiles} from "../Models/TileRange";
export class BBox {
readonly maxLat: number;
readonly maxLon: number;
readonly minLat: number;
readonly minLon: number;
static global: BBox = new BBox([[-180, -90], [180, 90]]);
constructor(coordinates) {
this.maxLat = -90;
this.maxLon = -180;
this.minLat = 90;
this.minLon = 180;
for (const coordinate of coordinates) {
this.maxLon = Math.max(this.maxLon, coordinate[0]);
this.maxLat = Math.max(this.maxLat, coordinate[1]);
this.minLon = Math.min(this.minLon, coordinate[0]);
this.minLat = Math.min(this.minLat, coordinate[1]);
}
this.check();
}
static fromLeafletBounds(bounds) {
return new BBox([[bounds.getWest(), bounds.getNorth()], [bounds.getEast(), bounds.getSouth()]])
}
static get(feature): BBox {
if (feature.bbox?.overlapsWith === undefined) {
const turfBbox: number[] = turf.bbox(feature)
feature.bbox = new BBox([[turfBbox[0], turfBbox[1]], [turfBbox[2], turfBbox[3]]]);
}
return feature.bbox;
}
/**
* Constructs a tilerange which fully contains this bbox (thus might be a bit larger)
* @param zoomlevel
*/
public containingTileRange(zoomlevel): TileRange{
return Tiles.TileRangeBetween(zoomlevel, this.minLat, this.minLon, this.maxLat, this.maxLon)
}
public overlapsWith(other: BBox) {
if (this.maxLon < other.minLon) {
return false;
}
if (this.maxLat < other.minLat) {
return false;
}
if (this.minLon > other.maxLon) {
return false;
}
return this.minLat <= other.maxLat;
}
public isContainedIn(other: BBox) {
if (this.maxLon > other.maxLon) {
return false;
}
if (this.maxLat > other.maxLat) {
return false;
}
if (this.minLon < other.minLon) {
return false;
}
if (this.minLat < other.minLat) {
return false
}
return true;
}
private check() {
if (isNaN(this.maxLon) || isNaN(this.maxLat) || isNaN(this.minLon) || isNaN(this.minLat)) {
console.log(this);
throw "BBOX has NAN";
}
}
static fromTile(z: number, x: number, y: number): BBox {
return new BBox(Tiles.tile_bounds_lon_lat(z, x, y))
}
static fromTileIndex(i: number): BBox {
if (i === 0) {
return BBox.global
}
return BBox.fromTile(...Tiles.tile_from_index(i))
}
getEast() {
return this.maxLon
}
getNorth() {
return this.maxLat
}
getWest() {
return this.minLon
}
getSouth() {
return this.minLat
}
pad(factor: number): BBox {
const latDiff = this.maxLat - this.minLat
const lat = (this.maxLat + this.minLat) / 2
const lonDiff = this.maxLon - this.minLon
const lon = (this.maxLon + this.minLon) / 2
return new BBox([[
lon - lonDiff * factor,
lat - latDiff * factor
], [lon + lonDiff * factor,
lat + latDiff * factor]])
}
toLeaflet() {
return [[this.minLat, this.minLon], [this.maxLat, this.maxLon]]
}
asGeoJson(properties: any): any {
return {
type: "Feature",
properties: properties,
geometry: {
type: "Polygon",
coordinates: [[
[this.minLon, this.minLat],
[this.maxLon, this.minLat],
[this.maxLon, this.maxLat],
[this.minLon, this.maxLat],
[this.minLon, this.minLat],
]]
}
}
}
/**
* Expands the BBOx so that it contains complete tiles for the given zoomlevel
* @param zoomlevel
*/
expandToTileBounds(zoomlevel: number) : BBox{
const ul = Tiles.embedded_tile(this.minLat, this.minLon, zoomlevel)
const lr = Tiles.embedded_tile(this.maxLat, this.maxLon, zoomlevel)
const boundsul = Tiles.tile_bounds_lon_lat(ul.z, ul.x, ul.y)
const boundslr = Tiles.tile_bounds_lon_lat(lr.z, lr.x, lr.y)
return new BBox([].concat(boundsul, boundslr))
}
}

2
Logic/ContributorCount.ts
View file

@ -2,7 +2,7 @@
import {UIEventSource} from "./UIEventSource";
import FeaturePipeline from "./FeatureSource/FeaturePipeline";
import Loc from "../Models/Loc";
import {BBox} from "./GeoOperations";
import {BBox} from "./BBox";
export default class ContributorCount {

3
Logic/ExtraFunction.ts
View file

@ -1,4 +1,4 @@
import {BBox, GeoOperations} from "./GeoOperations";
import {GeoOperations} from "./GeoOperations";
import Combine from "../UI/Base/Combine";
import RelationsTracker from "./Osm/RelationsTracker";
import State from "../State";
@ -7,6 +7,7 @@ import List from "../UI/Base/List";
import Title from "../UI/Base/Title";
import {UIEventSourceTools} from "./UIEventSource";
import AspectedRouting from "./Osm/aspectedRouting";
import {BBox} from "./BBox";
export interface ExtraFuncParams {
/**

146
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -17,18 +17,23 @@ import RegisteringAllFromFeatureSourceActor from "./Actors/RegisteringAllFromFea
import TiledFromLocalStorageSource from "./TiledFeatureSource/TiledFromLocalStorageSource";
import SaveTileToLocalStorageActor from "./Actors/SaveTileToLocalStorageActor";
import DynamicGeoJsonTileSource from "./TiledFeatureSource/DynamicGeoJsonTileSource";
import {BBox} from "../GeoOperations";
import {TileHierarchyMerger} from "./TiledFeatureSource/TileHierarchyMerger";
import RelationsTracker from "../Osm/RelationsTracker";
import {NewGeometryFromChangesFeatureSource} from "./Sources/NewGeometryFromChangesFeatureSource";
import ChangeGeometryApplicator from "./Sources/ChangeGeometryApplicator";
import {BBox} from "../BBox";
import OsmFeatureSource from "./TiledFeatureSource/OsmFeatureSource";
import {OsmConnection} from "../Osm/OsmConnection";
import {Tiles} from "../../Models/TileRange";
export default class FeaturePipeline implements FeatureSourceState {
export default class FeaturePipeline {
public readonly sufficientlyZoomed: UIEventSource<boolean>;
public readonly runningQuery: UIEventSource<boolean>;
public readonly timeout: UIEventSource<number>;
public readonly somethingLoaded: UIEventSource<boolean> = new UIEventSource<boolean>(false)
public readonly newDataLoadedSignal: UIEventSource<FeatureSource> = new UIEventSource<FeatureSource>(undefined)
@ -39,27 +44,59 @@ export default class FeaturePipeline implements FeatureSourceState {
constructor(
handleFeatureSource: (source: FeatureSourceForLayer & Tiled) => void,
state: {
filteredLayers: UIEventSource<FilteredLayer[]>,
locationControl: UIEventSource<Loc>,
selectedElement: UIEventSource<any>,
changes: Changes,
layoutToUse: UIEventSource<LayoutConfig>,
leafletMap: any,
readonly filteredLayers: UIEventSource<FilteredLayer[]>,
readonly locationControl: UIEventSource<Loc>,
readonly selectedElement: UIEventSource<any>,
readonly changes: Changes,
readonly layoutToUse: UIEventSource<LayoutConfig>,
readonly leafletMap: any,
readonly overpassUrl: UIEventSource<string>;
readonly overpassTimeout: UIEventSource<number>;
readonly overpassMaxZoom: UIEventSource<number>;
readonly osmConnection: OsmConnection
readonly currentBounds: UIEventSource<BBox>
}) {
const self = this
const updater = new OverpassFeatureSource(state);
/**
* Maps tileid onto last download moment
*/
const tileFreshnesses = new Map<number, Date>()
const osmSourceZoomLevel = 14
const useOsmApi = state.locationControl.map(l => l.zoom > (state.overpassMaxZoom.data ?? 12))
this.relationTracker = new RelationsTracker()
const updater = new OverpassFeatureSource(state,
{
relationTracker: this.relationTracker,
isActive: useOsmApi.map(b => !b),
onUpdated: (bbox, freshness) => {
// This callback contains metadata of the overpass call
const range = bbox.containingTileRange(osmSourceZoomLevel)
Tiles.MapRange(range, (x, y) => {
tileFreshnesses.set(Tiles.tile_index(osmSourceZoomLevel, x, y), freshness)
})
}
});
this.overpassUpdater = updater;
this.sufficientlyZoomed = updater.sufficientlyZoomed
this.runningQuery = updater.runningQuery
this.sufficientlyZoomed = state.locationControl.map(location => {
if (location?.zoom === undefined) {
return false;
}
let minzoom = Math.min(...state.layoutToUse.data.layers.map(layer => layer.minzoom ?? 18));
return location.zoom >= minzoom;
}
);
this.timeout = updater.timeout
this.relationTracker = updater.relationsTracker
// Register everything in the state' 'AllElements'
new RegisteringAllFromFeatureSourceActor(updater)
const perLayerHierarchy = new Map<string, TileHierarchyMerger>()
this.perLayerHierarchy = perLayerHierarchy
@ -72,7 +109,7 @@ export default class FeaturePipeline implements FeatureSourceState {
new ChangeGeometryApplicator(src, state.changes)
)
)
handleFeatureSource(srcFiltered)
self.somethingLoaded.setData(true)
};
@ -81,6 +118,7 @@ export default class FeaturePipeline implements FeatureSourceState {
perLayerHierarchy.get(layerId).registerTile(src)
}
for (const filteredLayer of state.filteredLayers.data) {
const hierarchy = new TileHierarchyMerger(filteredLayer, (tile, _) => patchedHandleFeatureSource(tile))
const id = filteredLayer.layerDef.id
@ -91,12 +129,25 @@ export default class FeaturePipeline implements FeatureSourceState {
// This is an OSM layer
// We load the cached values and register them
// Getting data from upstream happens a bit lower
new TiledFromLocalStorageSource(filteredLayer,
const localStorage = new TiledFromLocalStorageSource(filteredLayer,
(src) => {
new RegisteringAllFromFeatureSourceActor(src)
hierarchy.registerTile(src);
src.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(src))
}, state)
localStorage.tileFreshness.forEach((value, key) => {
if (tileFreshnesses.has(key)) {
const previous = tileFreshnesses.get(key)
if (value < previous) {
tileFreshnesses.set(key, value)
}
} else {
tileFreshnesses.set(key, value)
}
})
continue
}
@ -106,7 +157,7 @@ export default class FeaturePipeline implements FeatureSourceState {
const src = new GeoJsonSource(filteredLayer)
TiledFeatureSource.createHierarchy(src, {
layer: src.layer,
minZoomLevel:14,
minZoomLevel: 14,
dontEnforceMinZoom: true,
registerTile: (tile) => {
new RegisteringAllFromFeatureSourceActor(tile)
@ -118,16 +169,54 @@ export default class FeaturePipeline implements FeatureSourceState {
new DynamicGeoJsonTileSource(
filteredLayer,
tile => {
new RegisteringAllFromFeatureSourceActor(tile)
addToHierarchy(tile, id)
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
},
new RegisteringAllFromFeatureSourceActor(tile)
addToHierarchy(tile, id)
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
},
state
)
}
}
console.log("Tilefreshnesses are", tileFreshnesses)
const oldestAllowedDate = new Date(new Date().getTime() - (60 * 60 * 24 * 30 * 1000));
const neededTilesFromOsm = state.currentBounds.map(bbox => {
if (bbox === undefined) {
return
}
const range = bbox.containingTileRange(osmSourceZoomLevel)
const tileIndexes = []
if (range.total > 100) {
// Too much tiles!
return []
}
Tiles.MapRange(range, (x, y) => {
const i = Tiles.tile_index(osmSourceZoomLevel, x, y);
if (tileFreshnesses.get(i) > oldestAllowedDate) {
console.debug("Skipping tile", osmSourceZoomLevel, x, y, "as a decently fresh one is available")
// The cached tiles contain decently fresh data
return;
}
tileIndexes.push(i)
})
return tileIndexes
})
const osmFeatureSource = new OsmFeatureSource({
isActive: useOsmApi,
neededTiles: neededTilesFromOsm,
handleTile: tile => {
new RegisteringAllFromFeatureSourceActor(tile)
new SaveTileToLocalStorageActor(tile, tile.tileIndex)
addToHierarchy(tile, tile.layer.layerDef.id),
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
},
state: state
})
// Actually load data from the overpass source
new PerLayerFeatureSourceSplitter(state.filteredLayers,
(source) => TiledFeatureSource.createHierarchy(source, {
@ -169,9 +258,15 @@ export default class FeaturePipeline implements FeatureSourceState {
self.updateAllMetaTagging()
})
this.runningQuery = updater.runningQuery.map(
overpass => overpass || osmFeatureSource.isRunning.data, [osmFeatureSource.isRunning]
)
}
private applyMetaTags(src: FeatureSourceForLayer){
private applyMetaTags(src: FeatureSourceForLayer) {
const self = this
console.debug("Applying metatagging onto ", src.name)
window.setTimeout(
@ -192,7 +287,7 @@ export default class FeaturePipeline implements FeatureSourceState {
},
15
)
}
private updateAllMetaTagging() {
@ -231,7 +326,4 @@ export default class FeaturePipeline implements FeatureSourceState {
})
}
public ForceRefresh() {
this.overpassUpdater.ForceRefresh()
}
}

2
Logic/FeatureSource/FeatureSource.ts
View file

@ -1,7 +1,7 @@
import {UIEventSource} from "../UIEventSource";
import {Utils} from "../../Utils";
import FilteredLayer from "../../Models/FilteredLayer";
import {BBox} from "../GeoOperations";
import {BBox} from "../BBox";
export default interface FeatureSource {
features: UIEventSource<{ feature: any, freshness: Date }[]>;

3
Logic/FeatureSource/PerLayerFeatureSourceSplitter.ts
View file

@ -15,6 +15,7 @@ export default class PerLayerFeatureSourceSplitter {
handleLayerData: (source: FeatureSourceForLayer & Tiled) => void,
upstream: FeatureSource,
options?:{
tileIndex?: number,
handleLeftovers?: (featuresWithoutLayer: any[]) => void
}) {
@ -71,7 +72,7 @@ export default class PerLayerFeatureSourceSplitter {
let featureSource = knownLayers.get(id)
if (featureSource === undefined) {
// Not yet initialized - now is a good time
featureSource = new SimpleFeatureSource(layer)
featureSource = new SimpleFeatureSource(layer, options?.tileIndex)
featureSource.features.setData(features)
knownLayers.set(id, featureSource)
handleLayerData(featureSource)

2
Logic/FeatureSource/Sources/FeatureSourceMerger.ts
View file

@ -5,9 +5,9 @@
import {UIEventSource} from "../../UIEventSource";
import FeatureSource, {FeatureSourceForLayer, IndexedFeatureSource, Tiled} from "../FeatureSource";
import FilteredLayer from "../../../Models/FilteredLayer";
import {BBox} from "../../GeoOperations";
import {Utils} from "../../../Utils";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
export default class FeatureSourceMerger implements FeatureSourceForLayer, Tiled, IndexedFeatureSource {

2
Logic/FeatureSource/Sources/FilteringFeatureSource.ts
View file

@ -3,7 +3,7 @@ import LayerConfig from "../../../Models/ThemeConfig/LayerConfig";
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import Hash from "../../Web/Hash";
import {BBox} from "../../GeoOperations";
import {BBox} from "../../BBox";
export default class FilteringFeatureSource implements FeatureSourceForLayer, Tiled {
public features: UIEventSource<{ feature: any; freshness: Date }[]> =

2
Logic/FeatureSource/Sources/GeoJsonSource.ts
View file

@ -5,8 +5,8 @@ import {UIEventSource} from "../../UIEventSource";
import FilteredLayer from "../../../Models/FilteredLayer";
import {Utils} from "../../../Utils";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {BBox} from "../../GeoOperations";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {

2
Logic/FeatureSource/Sources/RememberingSource.ts
View file

@ -4,7 +4,7 @@
*/
import FeatureSource, {Tiled} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import {BBox} from "../../GeoOperations";
import {BBox} from "../../BBox";
export default class RememberingSource implements FeatureSource , Tiled{

8
Logic/FeatureSource/Sources/SimpleFeatureSource.ts
View file

@ -1,20 +1,22 @@
import {UIEventSource} from "../../UIEventSource";
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {BBox} from "../../GeoOperations";
import {Utils} from "../../../Utils";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
export default class SimpleFeatureSource implements FeatureSourceForLayer, Tiled {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{ feature: any; freshness: Date }[]>([]);
public readonly name: string = "SimpleFeatureSource";
public readonly layer: FilteredLayer;
public readonly bbox: BBox = BBox.global;
public readonly tileIndex: number = Tiles.tile_index(0, 0, 0);
public readonly tileIndex: number;
constructor(layer: FilteredLayer) {
constructor(layer: FilteredLayer, tileIndex: number) {
this.name = "SimpleFeatureSource(" + layer.layerDef.id + ")"
this.layer = layer
this.tileIndex = tileIndex ?? 0;
this.bbox = BBox.fromTileIndex(this.tileIndex)
}
}

112
Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts Normal file
View file

@ -0,0 +1,112 @@
import {Utils} from "../../../Utils";
import * as OsmToGeoJson from "osmtogeojson";
import StaticFeatureSource from "../Sources/StaticFeatureSource";
import PerLayerFeatureSourceSplitter from "../PerLayerFeatureSourceSplitter";
import {UIEventSource} from "../../UIEventSource";
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
import {OsmConnection} from "../../Osm/OsmConnection";
export default class OsmFeatureSource {
private readonly _backend: string;
public readonly isRunning: UIEventSource<boolean> = new UIEventSource<boolean>(false)
private readonly filteredLayers: UIEventSource<FilteredLayer[]>;
private readonly handleTile: (fs: (FeatureSourceForLayer & Tiled)) => void;
private isActive: UIEventSource<boolean>;
private options: {
handleTile: (tile: FeatureSourceForLayer & Tiled) => void;
isActive: UIEventSource<boolean>,
neededTiles: UIEventSource<number[]>,
state: {
readonly osmConnection: OsmConnection;
};
};
private readonly downloadedTiles = new Set<number>()
constructor(options: {
handleTile: (tile: FeatureSourceForLayer & Tiled) => void;
isActive: UIEventSource<boolean>,
neededTiles: UIEventSource<number[]>,
state: {
readonly filteredLayers: UIEventSource<FilteredLayer[]>;
readonly osmConnection: OsmConnection;
};
}) {
this.options = options;
this._backend = options.state.osmConnection._oauth_config.url;
this.filteredLayers = options.state.filteredLayers.map(layers => layers.filter(layer => layer.layerDef.source.geojsonSource === undefined))
this.handleTile = options.handleTile
this.isActive = options.isActive
const self = this
options.neededTiles.addCallbackAndRunD(neededTiles => {
if (options.isActive?.data === false) {
return;
}
self.isRunning.setData(true)
try {
for (const neededTile of neededTiles) {
if (self.downloadedTiles.has(neededTile)) {
return;
}
self.downloadedTiles.add(neededTile)
Promise.resolve(self.LoadTile(...Tiles.tile_from_index(neededTile)).then(_ => {
}))
}
} catch (e) {
console.error(e)
}
self.isRunning.setData(false)
})
}
private async LoadTile(z, x, y): Promise<void> {
if (z > 18) {
throw "This is an absurd high zoom level"
}
const bbox = BBox.fromTile(z, x, y)
const url = `${this._backend}/api/0.6/map?bbox=${bbox.minLon},${bbox.minLat},${bbox.maxLon},${bbox.maxLat}`
try {
console.log("Attempting to get tile", z, x, y, "from the osm api")
const osmXml = await Utils.download(url, {"accept": "application/xml"})
try {
const parsed = new DOMParser().parseFromString(osmXml, "text/xml");
console.log("Got tile", z, x, y, "from the osm api")
const geojson = OsmToGeoJson.default(parsed,
// @ts-ignore
{
flatProperties: true
});
console.log("Tile geojson:", z, x, y, "is", geojson)
new PerLayerFeatureSourceSplitter(this.filteredLayers,
this.handleTile,
new StaticFeatureSource(geojson.features, false),
{
tileIndex: Tiles.tile_index(z, x, y)
}
);
} catch (e) {
console.error("Weird error: ", e)
}
} catch (e) {
console.error("Could not download tile", z, x, y, "due to", e, "; retrying with smaller bounds")
if (e === "rate limited") {
return;
}
await this.LoadTile(z + 1, x * 2, y * 2)
await this.LoadTile(z + 1, 1 + x * 2, y * 2)
await this.LoadTile(z + 1, x * 2, 1 + y * 2)
await this.LoadTile(z + 1, 1 + x * 2, 1 + y * 2)
return;
}
}
}

2
Logic/FeatureSource/TiledFeatureSource/TileHierarchy.ts
View file

@ -1,5 +1,5 @@
import FeatureSource, {Tiled} from "../FeatureSource";
import {BBox} from "../../GeoOperations";
import {BBox} from "../../BBox";
export default interface TileHierarchy<T extends FeatureSource & Tiled> {

2
Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts
View file

@ -3,9 +3,9 @@ import {UIEventSource} from "../../UIEventSource";
import FeatureSource, {FeatureSourceForLayer, IndexedFeatureSource, Tiled} from "../FeatureSource";
import FilteredLayer from "../../../Models/FilteredLayer";
import {Utils} from "../../../Utils";
import {BBox} from "../../GeoOperations";
import FeatureSourceMerger from "../Sources/FeatureSourceMerger";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();

18
Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource.ts
View file

@ -1,10 +1,10 @@
import FeatureSource, {FeatureSourceForLayer, IndexedFeatureSource, Tiled} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import {Utils} from "../../../Utils";
import {BBox} from "../../GeoOperations";
import FilteredLayer from "../../../Models/FilteredLayer";
import TileHierarchy from "./TileHierarchy";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
/**
* Contains all features in a tiled fashion.
@ -109,7 +109,6 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
// To much features - we split
return featureCount > this.maxFeatureCount
}
/***
@ -143,7 +142,20 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
for (const feature of features) {
const bbox = BBox.get(feature.feature)
if (this.options.dontEnforceMinZoom || this.options.minZoomLevel === undefined) {
if (this.options.dontEnforceMinZoom) {
if (bbox.overlapsWith(this.upper_left.bbox)) {
ulf.push(feature)
} else if (bbox.overlapsWith(this.upper_right.bbox)) {
urf.push(feature)
} else if (bbox.overlapsWith(this.lower_left.bbox)) {
llf.push(feature)
} else if (bbox.overlapsWith(this.lower_right.bbox)) {
lrf.push(feature)
} else {
overlapsboundary.push(feature)
}
}else if (this.options.minZoomLevel === undefined) {
if (bbox.isContainedIn(this.upper_left.bbox)) {
ulf.push(feature)
} else if (bbox.isContainedIn(this.upper_right.bbox)) {

16
Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts
View file

@ -5,12 +5,13 @@ import Loc from "../../../Models/Loc";
import TileHierarchy from "./TileHierarchy";
import {Utils} from "../../../Utils";
import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor";
import {BBox} from "../../GeoOperations";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
export default class TiledFromLocalStorageSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
public tileFreshness : Map<number, Date> = new Map<number, Date>()
constructor(layer: FilteredLayer,
handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void,
state: {
@ -29,7 +30,14 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
return Number(key.substring(prefix.length));
})
console.log("Layer", layer.layerDef.id, "has following tiles in available in localstorage", indexes.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", indexes.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
for (const index of indexes) {
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" +index+"-time";
const data = Number(localStorage.getItem(prefix))
const freshness = new Date()
freshness.setTime(data)
this.tileFreshness.set(index, freshness)
}
const zLevels = indexes.map(i => i % 100)
const indexesSet = new Set(indexes)
@ -72,7 +80,7 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
}
, [layer.isDisplayed, state.leafletMap]).stabilized(50);
neededTiles.addCallbackAndRun(t => console.log("Tiles to load from localstorage:", t))
neededTiles.addCallbackAndRun(t => console.debug("Tiles to load from localstorage:", t))
neededTiles.addCallbackAndRunD(neededIndexes => {
for (const neededIndex of neededIndexes) {

135
Logic/GeoOperations.ts
View file

@ -1,6 +1,5 @@
import * as turf from '@turf/turf'
import {Utils} from "../Utils";
import {Tiles} from "../Models/TileRange";
import {BBox} from "./BBox";
export class GeoOperations {
@ -379,135 +378,3 @@ export class GeoOperations {
}
export class BBox {
readonly maxLat: number;
readonly maxLon: number;
readonly minLat: number;
readonly minLon: number;
static global: BBox = new BBox([[-180, -90], [180, 90]]);
constructor(coordinates) {
this.maxLat = -90;
this.maxLon = -180;
this.minLat = 90;
this.minLon = 180;
for (const coordinate of coordinates) {
this.maxLon = Math.max(this.maxLon, coordinate[0]);
this.maxLat = Math.max(this.maxLat, coordinate[1]);
this.minLon = Math.min(this.minLon, coordinate[0]);
this.minLat = Math.min(this.minLat, coordinate[1]);
}
this.check();
}
static fromLeafletBounds(bounds) {
return new BBox([[bounds.getWest(), bounds.getNorth()], [bounds.getEast(), bounds.getSouth()]])
}
static get(feature): BBox {
if (feature.bbox?.overlapsWith === undefined) {
const turfBbox: number[] = turf.bbox(feature)
feature.bbox = new BBox([[turfBbox[0], turfBbox[1]], [turfBbox[2], turfBbox[3]]]);
}
return feature.bbox;
}
public overlapsWith(other: BBox) {
if (this.maxLon < other.minLon) {
return false;
}
if (this.maxLat < other.minLat) {
return false;
}
if (this.minLon > other.maxLon) {
return false;
}
return this.minLat <= other.maxLat;
}
public isContainedIn(other: BBox) {
if (this.maxLon > other.maxLon) {
return false;
}
if (this.maxLat > other.maxLat) {
return false;
}
if (this.minLon < other.minLon) {
return false;
}
if (this.minLat < other.minLat) {
return false
}
return true;
}
private check() {
if (isNaN(this.maxLon) || isNaN(this.maxLat) || isNaN(this.minLon) || isNaN(this.minLat)) {
console.log(this);
throw "BBOX has NAN";
}
}
static fromTile(z: number, x: number, y: number): BBox {
return new BBox(Tiles.tile_bounds_lon_lat(z, x, y))
}
static fromTileIndex(i: number): BBox {
return BBox.fromTile(...Tiles.tile_from_index(i))
}
getEast() {
return this.maxLon
}
getNorth() {
return this.maxLat
}
getWest() {
return this.minLon
}
getSouth() {
return this.minLat
}
pad(factor: number) : BBox {
const latDiff = this.maxLat - this.minLat
const lat = (this.maxLat + this.minLat) / 2
const lonDiff = this.maxLon - this.minLon
const lon = (this.maxLon + this.minLon) / 2
return new BBox([[
lon - lonDiff * factor,
lat - latDiff * factor
], [lon + lonDiff * factor,
lat + latDiff * factor]])
}
toLeaflet() {
return [[this.minLat, this.minLon], [this.maxLat, this.maxLon]]
}
asGeoJson(properties: any) : any{
return {
type:"Feature",
properties: properties,
geometry:{
type:"Polygon",
coordinates:[[
[this.minLon, this.minLat],
[this.maxLon, this.minLat],
[this.maxLon, this.maxLat],
[this.minLon, this.maxLat],
[this.minLon, this.minLat],
]]
}
}
}
}

32
Logic/Osm/OsmConnection.ts
View file

@ -62,26 +62,26 @@ export class OsmConnection {
};
private isChecking = false;
constructor(dryRun: boolean,
fakeUser: boolean,
constructor(options:{dryRun?: false | boolean,
fakeUser?: false | boolean,
allElements: ElementStorage,
changes: Changes,
oauth_token: UIEventSource<string>,
oauth_token?: UIEventSource<string>,
// Used to keep multiple changesets open and to write to the correct changeset
layoutName: string,
singlePage: boolean = true,
osmConfiguration: "osm" | "osm-test" = 'osm'
singlePage?: boolean,
osmConfiguration?: "osm" | "osm-test" }
) {
this.fakeUser = fakeUser;
this._singlePage = singlePage;
this._oauth_config = OsmConnection.oauth_configs[osmConfiguration] ?? OsmConnection.oauth_configs.osm;
this.fakeUser = options.fakeUser ?? false;
this._singlePage = options.singlePage ?? true;
this._oauth_config = OsmConnection.oauth_configs[options.osmConfiguration ?? 'osm'] ?? OsmConnection.oauth_configs.osm;
console.debug("Using backend", this._oauth_config.url)
OsmObject.SetBackendUrl(this._oauth_config.url + "/")
this._iframeMode = Utils.runningFromConsole ? false : window !== window.top;
this.userDetails = new UIEventSource<UserDetails>(new UserDetails(this._oauth_config.url), "userDetails");
this.userDetails.data.dryRun = dryRun || fakeUser;
if (fakeUser) {
this.userDetails.data.dryRun = (options.dryRun ?? false) || (options.fakeUser ?? false) ;
if (options.fakeUser) {
const ud = this.userDetails.data;
ud.csCount = 5678
ud.loggedIn = true;
@ -98,23 +98,23 @@ export class OsmConnection {
}
});
this.isLoggedIn.addCallbackAndRunD(li => console.log("User is logged in!", li))
this._dryRun = dryRun;
this._dryRun = options.dryRun;
this.updateAuthObject();
this.preferencesHandler = new OsmPreferences(this.auth, this);
this.changesetHandler = new ChangesetHandler(layoutName, dryRun, this, allElements, changes, this.auth);
if (oauth_token.data !== undefined) {
console.log(oauth_token.data)
this.changesetHandler = new ChangesetHandler(options.layoutName, options.dryRun, this, options.allElements, options.changes, this.auth);
if (options.oauth_token?.data !== undefined) {
console.log(options.oauth_token.data)
const self = this;
this.auth.bootstrapToken(oauth_token.data,
this.auth.bootstrapToken(options.oauth_token.data,
(x) => {
console.log("Called back: ", x)
self.AttemptLogin();
}, this.auth);
oauth_token.setData(undefined);
options. oauth_token.setData(undefined);
}
if (this.auth.authenticated()) {

2
Logic/Osm/OsmObject.ts
View file

@ -1,7 +1,7 @@
import {Utils} from "../../Utils";
import * as polygon_features from "../../assets/polygon-features.json";
import {UIEventSource} from "../UIEventSource";
import {BBox} from "../GeoOperations";
import {BBox} from "../BBox";
export abstract class OsmObject {

15
Logic/Osm/Overpass.ts
View file

@ -16,8 +16,8 @@ export class Overpass {
private readonly _extraScripts: string[];
private _includeMeta: boolean;
private _relationTracker: RelationsTracker;
constructor(filter: TagsFilter, extraScripts: string[],
interpreterUrl: UIEventSource<string>,
timeout: UIEventSource<number>,
@ -41,10 +41,13 @@ export class Overpass {
}
const self = this;
const json = await Utils.downloadJson(query)
if (json.elements === [] && ((json.remarks ?? json.remark).indexOf("runtime error") >= 0)) {
console.log("Timeout or other runtime error");
throw("Runtime error (timeout)")
console.log("Got json!", json)
if (json.elements.length === 0 && json.remark !== undefined) {
console.warn("Timeout or other runtime error while querying overpass", json.remark);
throw `Runtime error (timeout or similar)${json.remark}`
}
if(json.elements.length === 0){
console.warn("No features for" ,json)
}
self._relationTracker.RegisterRelations(json)