midgard/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Full code cleanup

Browse source
This commit is contained in:
Pieter Vander Vennet 2021-11-07 16:34:51 +01:00
parent 8e6ee8c87f
commit bd21212eba
246 changed files with 19418 additions and 11729 deletions
Download patch file Download diff file Expand all files Collapse all files

9
Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts
View file

@ -9,9 +9,8 @@ import {Tiles} from "../../../Models/TileRange";
* A tiled source which dynamically loads the required tiles at a fixed zoom level
*/
export default class DynamicTileSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
private readonly _loadedTiles = new Set<number>();
public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled>;
private readonly _loadedTiles = new Set<number>();
constructor(
layer: FilteredLayer,
@ -24,7 +23,7 @@ export default class DynamicTileSource implements TileHierarchy<FeatureSourceFor
) {
const self = this;
this.loadedTiles = new Map<number,FeatureSourceForLayer & Tiled>()
this.loadedTiles = new Map<number, FeatureSourceForLayer & Tiled>()
const neededTiles = state.locationControl.map(
location => {
if (!layer.isDisplayed.data) {
@ -54,14 +53,14 @@ export default class DynamicTileSource implements TileHierarchy<FeatureSourceFor
, [layer.isDisplayed, state.leafletMap]).stabilized(250);
neededTiles.addCallbackAndRunD(neededIndexes => {
console.log("Tiled geojson source ",layer.layerDef.id," needs", neededIndexes)
console.log("Tiled geojson source ", layer.layerDef.id, " needs", neededIndexes)
if (neededIndexes === undefined) {
return;
}
for (const neededIndex of neededIndexes) {
self._loadedTiles.add(neededIndex)
const src = constructTile(Tiles.tile_from_index(neededIndex))
if(src !== undefined){
if (src !== undefined) {
self.loadedTiles.set(neededIndex, src)
}
}

2
Logic/FeatureSource/TiledFeatureSource/FullNodeDatabaseSource.ts
View file

@ -19,7 +19,7 @@ export default class FullNodeDatabaseSource implements TileHierarchy<FeatureSour
throw "Layer is undefined"
}
}
public handleOsmJson(osmJson: any, tileId: number) {
const allObjects = OsmObject.ParseObjects(osmJson.elements)

18
Logic/FeatureSource/TiledFeatureSource/OsmFeatureSource.ts
View file

@ -13,9 +13,10 @@ import {Or} from "../../Tags/Or";
import {TagsFilter} from "../../Tags/TagsFilter";
export default class OsmFeatureSource {
private readonly _backend: string;
public readonly isRunning: UIEventSource<boolean> = new UIEventSource<boolean>(false)
public readonly downloadedTiles = new Set<number>()
public rawDataHandlers: ((osmJson: any, tileId: number) => void)[] = []
private readonly _backend: string;
private readonly filteredLayers: UIEventSource<FilteredLayer[]>;
private readonly handleTile: (fs: (FeatureSourceForLayer & Tiled)) => void;
private isActive: UIEventSource<boolean>;
@ -28,10 +29,7 @@ export default class OsmFeatureSource {
},
markTileVisited?: (tileId: number) => void
};
public readonly downloadedTiles = new Set<number>()
private readonly allowedTags: TagsFilter;
public rawDataHandlers: ((osmJson: any, tileId: number) => void)[] = []
constructor(options: {
handleTile: (tile: FeatureSourceForLayer & Tiled) => void;
@ -54,13 +52,13 @@ export default class OsmFeatureSource {
if (options.isActive?.data === false) {
return;
}
neededTiles = neededTiles.filter(tile => !self.downloadedTiles.has(tile))
if(neededTiles.length == 0){
if (neededTiles.length == 0) {
return;
}
self.isRunning.setData(true)
try {
@ -73,7 +71,7 @@ export default class OsmFeatureSource {
}
} catch (e) {
console.error(e)
}finally {
} finally {
console.log("Done")
self.isRunning.setData(false)
}
@ -111,7 +109,7 @@ export default class OsmFeatureSource {
geojson.features = geojson.features.filter(feature => this.allowedTags.matchesProperties(feature.properties))
geojson.features.forEach(f => f.properties["_backend"] = this._backend)
const index = Tiles.tile_index(z, x, y);
new PerLayerFeatureSourceSplitter(this.filteredLayers,
this.handleTile,

5
Logic/FeatureSource/TiledFeatureSource/README.md
View file

@ -11,17 +11,14 @@ Currently, they are:
When the data enters from Overpass or from the OSM-API, they are first distributed per layer:
OVERPASS | ---PerLayerFeatureSource---> FeatureSourceForLayer[]
OSM |
OSM |
The GeoJSon files (not tiled) are then added to this list
A single FeatureSourcePerLayer is then further handled by splitting it into a tile hierarchy.
In order to keep thins snappy, they are distributed over a tiled database per layer.
## Notes
`cached-featuresbookcases` is the old key used `cahced-features{themeid}` and should be cleaned up

5
Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts
View file

@ -8,9 +8,8 @@ import {BBox} from "../../BBox";
export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
private readonly sources: Map<number, UIEventSource<FeatureSource[]>> = new Map<number, UIEventSource<FeatureSource[]>>();
public readonly layer: FilteredLayer;
private readonly sources: Map<number, UIEventSource<FeatureSource[]>> = new Map<number, UIEventSource<FeatureSource[]>>();
private _handleTile: (src: FeatureSourceForLayer & IndexedFeatureSource, index: number) => void;
constructor(layer: FilteredLayer, handleTile: (src: FeatureSourceForLayer & IndexedFeatureSource & Tiled, index: number) => void) {
@ -24,7 +23,7 @@ export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer
* @param src
* @param index
*/
public registerTile(src: FeatureSource & Tiled) {
public registerTile(src: FeatureSource & Tiled) {
const index = src.tileIndex
if (this.sources.has(index)) {

21
Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource.ts
View file

@ -1,6 +1,5 @@
import FeatureSource, {FeatureSourceForLayer, IndexedFeatureSource, Tiled} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import {Utils} from "../../../Utils";
import FilteredLayer from "../../../Models/FilteredLayer";
import TileHierarchy from "./TileHierarchy";
import {Tiles} from "../../../Models/TileRange";
@ -28,13 +27,13 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
public readonly containedIds: UIEventSource<Set<string>>
public readonly bbox: BBox;
public readonly tileIndex: number;
private upper_left: TiledFeatureSource
private upper_right: TiledFeatureSource
private lower_left: TiledFeatureSource
private lower_right: TiledFeatureSource
private readonly maxzoom: number;
private readonly options: TiledFeatureSourceOptions
public readonly tileIndex: number;
private constructor(z: number, x: number, y: number, parent: TiledFeatureSource, options?: TiledFeatureSourceOptions) {
this.z = z;
@ -92,25 +91,25 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
return root;
}
private isSplitNeeded(featureCount: number){
if(this.upper_left !== undefined){
private isSplitNeeded(featureCount: number) {
if (this.upper_left !== undefined) {
// This tile has been split previously, so we keep on splitting
return true;
}
if(this.z >= this.maxzoom){
if (this.z >= this.maxzoom) {
// We are not allowed to split any further
return false
}
if(this.options.minZoomLevel !== undefined && this.z < this.options.minZoomLevel){
if (this.options.minZoomLevel !== undefined && this.z < this.options.minZoomLevel) {
// We must have at least this zoom level before we are allowed to start splitting
return true
}
// To much features - we split
return featureCount > this.maxFeatureCount
}
/***
* Adds the list of features to this hierarchy.
* If there are too much features, the list will be broken down and distributed over the subtiles (only retaining features that don't fit a subtile on this level)
@ -121,7 +120,7 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
if (features === undefined || features.length === 0) {
return;
}
if (!this.isSplitNeeded(features.length)) {
this.features.setData(features)
return;
@ -155,7 +154,7 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
} else {
overlapsboundary.push(feature)
}
}else if (this.options.minZoomLevel === undefined) {
} else if (this.options.minZoomLevel === undefined) {
if (bbox.isContainedIn(this.upper_left.bbox)) {
ulf.push(feature)
} else if (bbox.isContainedIn(this.upper_right.bbox)) {

75
Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts
View file

@ -13,44 +13,6 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
private readonly handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void;
private readonly undefinedTiles: Set<number>;
public static GetFreshnesses(layerId: string): Map<number, Date> {
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layerId + "-"
const freshnesses = new Map<number, Date>()
for (const key of Object.keys(localStorage)) {
if (!(key.startsWith(prefix) && key.endsWith("-time"))) {
continue
}
const index = Number(key.substring(prefix.length, key.length - "-time".length))
const time = Number(localStorage.getItem(key))
const freshness = new Date()
freshness.setTime(time)
freshnesses.set(index, freshness)
}
return freshnesses
}
static cleanCacheForLayer(layer: LayerConfig) {
const now = new Date()
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.id + "-"
console.log("Cleaning tiles of ", prefix, "with max age",layer.maxAgeOfCache)
for (const key of Object.keys(localStorage)) {
if (!(key.startsWith(prefix) && key.endsWith("-time"))) {
continue
}
const index = Number(key.substring(prefix.length, key.length - "-time".length))
const time = Number(localStorage.getItem(key))
const timeDiff = (now.getTime() - time) / 1000
if(timeDiff >= layer.maxAgeOfCache){
const k = prefix+index;
localStorage.removeItem(k)
localStorage.removeItem(k+"-format")
localStorage.removeItem(k+"-time")
}
}
}
constructor(layer: FilteredLayer,
handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void,
state: {
@ -110,6 +72,43 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
}
public static GetFreshnesses(layerId: string): Map<number, Date> {
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layerId + "-"
const freshnesses = new Map<number, Date>()
for (const key of Object.keys(localStorage)) {
if (!(key.startsWith(prefix) && key.endsWith("-time"))) {
continue
}
const index = Number(key.substring(prefix.length, key.length - "-time".length))
const time = Number(localStorage.getItem(key))
const freshness = new Date()
freshness.setTime(time)
freshnesses.set(index, freshness)
}
return freshnesses
}
static cleanCacheForLayer(layer: LayerConfig) {
const now = new Date()
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.id + "-"
console.log("Cleaning tiles of ", prefix, "with max age", layer.maxAgeOfCache)
for (const key of Object.keys(localStorage)) {
if (!(key.startsWith(prefix) && key.endsWith("-time"))) {
continue
}
const index = Number(key.substring(prefix.length, key.length - "-time".length))
const time = Number(localStorage.getItem(key))
const timeDiff = (now.getTime() - time) / 1000
if (timeDiff >= layer.maxAgeOfCache) {
const k = prefix + index;
localStorage.removeItem(k)
localStorage.removeItem(k + "-format")
localStorage.removeItem(k + "-time")
}
}
}
private loadTile(neededIndex: number) {
try {
const key = SaveTileToLocalStorageActor.storageKey + "-" + this.layer.layerDef.id + "-" + neededIndex