Osmwithspace/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

More refactoring of the featurepipeline, introduction of fetching data from the OSM-API directly per tile, personal theme refactoring

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-09-28 17:30:48 +02:00
parent 0a9e7c0b36
commit 41a2a79fe9
48 changed files with 746 additions and 590 deletions
Download patch file Download diff file Expand all files Collapse all files

112
Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts Normal file
View file

@ -0,0 +1,112 @@
import {Utils} from "../../../Utils";
import * as OsmToGeoJson from "osmtogeojson";
import StaticFeatureSource from "../Sources/StaticFeatureSource";
import PerLayerFeatureSourceSplitter from "../PerLayerFeatureSourceSplitter";
import {UIEventSource} from "../../UIEventSource";
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
import {OsmConnection} from "../../Osm/OsmConnection";
export default class OsmFeatureSource {
private readonly _backend: string;
public readonly isRunning: UIEventSource<boolean> = new UIEventSource<boolean>(false)
private readonly filteredLayers: UIEventSource<FilteredLayer[]>;
private readonly handleTile: (fs: (FeatureSourceForLayer & Tiled)) => void;
private isActive: UIEventSource<boolean>;
private options: {
handleTile: (tile: FeatureSourceForLayer & Tiled) => void;
isActive: UIEventSource<boolean>,
neededTiles: UIEventSource<number[]>,
state: {
readonly osmConnection: OsmConnection;
};
};
private readonly downloadedTiles = new Set<number>()
constructor(options: {
handleTile: (tile: FeatureSourceForLayer & Tiled) => void;
isActive: UIEventSource<boolean>,
neededTiles: UIEventSource<number[]>,
state: {
readonly filteredLayers: UIEventSource<FilteredLayer[]>;
readonly osmConnection: OsmConnection;
};
}) {
this.options = options;
this._backend = options.state.osmConnection._oauth_config.url;
this.filteredLayers = options.state.filteredLayers.map(layers => layers.filter(layer => layer.layerDef.source.geojsonSource === undefined))
this.handleTile = options.handleTile
this.isActive = options.isActive
const self = this
options.neededTiles.addCallbackAndRunD(neededTiles => {
if (options.isActive?.data === false) {
return;
}
self.isRunning.setData(true)
try {
for (const neededTile of neededTiles) {
if (self.downloadedTiles.has(neededTile)) {
return;
}
self.downloadedTiles.add(neededTile)
Promise.resolve(self.LoadTile(...Tiles.tile_from_index(neededTile)).then(_ => {
}))
}
} catch (e) {
console.error(e)
}
self.isRunning.setData(false)
})
}
private async LoadTile(z, x, y): Promise<void> {
if (z > 18) {
throw "This is an absurd high zoom level"
}
const bbox = BBox.fromTile(z, x, y)
const url = `${this._backend}/api/0.6/map?bbox=${bbox.minLon},${bbox.minLat},${bbox.maxLon},${bbox.maxLat}`
try {
console.log("Attempting to get tile", z, x, y, "from the osm api")
const osmXml = await Utils.download(url, {"accept": "application/xml"})
try {
const parsed = new DOMParser().parseFromString(osmXml, "text/xml");
console.log("Got tile", z, x, y, "from the osm api")
const geojson = OsmToGeoJson.default(parsed,
// @ts-ignore
{
flatProperties: true
});
console.log("Tile geojson:", z, x, y, "is", geojson)
new PerLayerFeatureSourceSplitter(this.filteredLayers,
this.handleTile,
new StaticFeatureSource(geojson.features, false),
{
tileIndex: Tiles.tile_index(z, x, y)
}
);
} catch (e) {
console.error("Weird error: ", e)
}
} catch (e) {
console.error("Could not download tile", z, x, y, "due to", e, "; retrying with smaller bounds")
if (e === "rate limited") {
return;
}
await this.LoadTile(z + 1, x * 2, y * 2)
await this.LoadTile(z + 1, 1 + x * 2, y * 2)
await this.LoadTile(z + 1, x * 2, 1 + y * 2)
await this.LoadTile(z + 1, 1 + x * 2, 1 + y * 2)
return;
}
}
}

2
Logic/FeatureSource/TiledFeatureSource/TileHierarchy.ts
View file

@ -1,5 +1,5 @@
import FeatureSource, {Tiled} from "../FeatureSource";
import {BBox} from "../../GeoOperations";
import {BBox} from "../../BBox";
export default interface TileHierarchy<T extends FeatureSource & Tiled> {

2
Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts
View file

@ -3,9 +3,9 @@ import {UIEventSource} from "../../UIEventSource";
import FeatureSource, {FeatureSourceForLayer, IndexedFeatureSource, Tiled} from "../FeatureSource";
import FilteredLayer from "../../../Models/FilteredLayer";
import {Utils} from "../../../Utils";
import {BBox} from "../../GeoOperations";
import FeatureSourceMerger from "../Sources/FeatureSourceMerger";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();

18
Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource.ts
View file

@ -1,10 +1,10 @@
import FeatureSource, {FeatureSourceForLayer, IndexedFeatureSource, Tiled} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import {Utils} from "../../../Utils";
import {BBox} from "../../GeoOperations";
import FilteredLayer from "../../../Models/FilteredLayer";
import TileHierarchy from "./TileHierarchy";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
/**
* Contains all features in a tiled fashion.
@ -109,7 +109,6 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
// To much features - we split
return featureCount > this.maxFeatureCount
}
/***
@ -143,7 +142,20 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
for (const feature of features) {
const bbox = BBox.get(feature.feature)
if (this.options.dontEnforceMinZoom || this.options.minZoomLevel === undefined) {
if (this.options.dontEnforceMinZoom) {
if (bbox.overlapsWith(this.upper_left.bbox)) {
ulf.push(feature)
} else if (bbox.overlapsWith(this.upper_right.bbox)) {
urf.push(feature)
} else if (bbox.overlapsWith(this.lower_left.bbox)) {
llf.push(feature)
} else if (bbox.overlapsWith(this.lower_right.bbox)) {
lrf.push(feature)
} else {
overlapsboundary.push(feature)
}
}else if (this.options.minZoomLevel === undefined) {
if (bbox.isContainedIn(this.upper_left.bbox)) {
ulf.push(feature)
} else if (bbox.isContainedIn(this.upper_right.bbox)) {

16
Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts
View file

@ -5,12 +5,13 @@ import Loc from "../../../Models/Loc";
import TileHierarchy from "./TileHierarchy";
import {Utils} from "../../../Utils";
import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor";
import {BBox} from "../../GeoOperations";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
export default class TiledFromLocalStorageSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
public tileFreshness : Map<number, Date> = new Map<number, Date>()
constructor(layer: FilteredLayer,
handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void,
state: {
@ -29,7 +30,14 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
return Number(key.substring(prefix.length));
})
console.log("Layer", layer.layerDef.id, "has following tiles in available in localstorage", indexes.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", indexes.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
for (const index of indexes) {
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" +index+"-time";
const data = Number(localStorage.getItem(prefix))
const freshness = new Date()
freshness.setTime(data)
this.tileFreshness.set(index, freshness)
}
const zLevels = indexes.map(i => i % 100)
const indexesSet = new Set(indexes)
@ -72,7 +80,7 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
}
, [layer.isDisplayed, state.leafletMap]).stabilized(50);
neededTiles.addCallbackAndRun(t => console.log("Tiles to load from localstorage:", t))
neededTiles.addCallbackAndRun(t => console.debug("Tiles to load from localstorage:", t))
neededTiles.addCallbackAndRunD(neededIndexes => {
for (const neededIndex of neededIndexes) {