Osmwithspace/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Download button: take advantage of MVT server, download button will now attempt to download everything

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-02-21 16:35:49 +01:00
parent bccda67e1c
commit e4eb8d6b52
21 changed files with 453 additions and 353 deletions
Download patch file Download diff file Expand all files Collapse all files

1
langs/en.json
View file

@ -213,6 +213,7 @@
"current_view_generic": "Export a PDF off the current view for {paper_size} in {orientation} orientation"
},
"title": "Download",
"toMuch": "There are to much features to download them all",
"uploadGpx": "Upload your track to OpenStreetMap"
},
"enableGeolocationForSafari": "Did you not get the popup to ask for geopermission?",

10
src/Logic/FeatureSource/FeatureSource.ts
View file

@ -5,6 +5,13 @@ import { Feature } from "geojson"
export interface FeatureSource<T extends Feature = Feature> {
features: Store<T[]>
}
export interface UpdatableFeatureSource<T extends Feature = Feature> extends FeatureSource<T> {
/**
* Forces an update and downloads the data, even if the feature source is supposed to be active
*/
updateAsync()
}
export interface WritableFeatureSource<T extends Feature = Feature> extends FeatureSource<T> {
features: UIEventSource<T[]>
}
@ -16,11 +23,10 @@ export interface FeatureSourceForLayer<T extends Feature = Feature> extends Feat
readonly layer: FilteredLayer
}
export interface FeatureSourceForTile <T extends Feature = Feature> extends FeatureSource<T> {
export interface FeatureSourceForTile<T extends Feature = Feature> extends FeatureSource<T> {
readonly x: number
readonly y: number
readonly z: number
}
/**
* A feature source which is aware of the indexes it contains

33
src/Logic/FeatureSource/Sources/FeatureSourceMerger.ts
View file

@ -1,18 +1,20 @@
import { Store, UIEventSource } from "../../UIEventSource"
import { FeatureSource, IndexedFeatureSource } from "../FeatureSource"
import { FeatureSource, IndexedFeatureSource, UpdatableFeatureSource } from "../FeatureSource"
import { Feature } from "geojson"
import { Utils } from "../../../Utils"
import DynamicTileSource from "../TiledFeatureSource/DynamicTileSource"
/**
* The featureSourceMerger receives complete geometries from various sources.
* If multiple sources contain the same object (as determined by 'id'), only one copy of them is retained
*/
export default class FeatureSourceMerger<Src extends FeatureSource = FeatureSource> implements IndexedFeatureSource {
export default class FeatureSourceMerger<Src extends FeatureSource = FeatureSource>
implements IndexedFeatureSource
{
public features: UIEventSource<Feature[]> = new UIEventSource([])
public readonly featuresById: Store<Map<string, Feature>>
protected readonly _featuresById: UIEventSource<Map<string, Feature>>
private readonly _sources: Src[] = []
protected readonly _sources: Src[]
/**
* Merges features from different featureSources.
* In case that multiple features have the same id, the latest `_version_number` will be used. Otherwise, we will take the last one
@ -27,22 +29,25 @@ export default class FeatureSourceMerger<Src extends FeatureSource = FeatureSour
self.addDataFromSources(sources)
})
}
this.addDataFromSources(sources)
this._sources = sources
this.addDataFromSources(sources)
}
public addSource(source: Src) {
if (!source) {
return
}
if (!source.features) {
console.error("No source found in", source)
}
this._sources.push(source)
source.features.addCallbackAndRun(() => {
this.addDataFromSources(this._sources)
})
}
protected addDataFromSources(sources: Src[]){
this.addData(sources.map(s => s.features.data))
protected addDataFromSources(sources: Src[]) {
this.addData(sources.map((s) => s.features.data))
}
protected addData(sources: Feature[][]) {
@ -93,3 +98,17 @@ export default class FeatureSourceMerger<Src extends FeatureSource = FeatureSour
this._featuresById.setData(all)
}
}
export class UpdatableFeatureSourceMerger<
Src extends UpdatableFeatureSource = UpdatableFeatureSource
>
extends FeatureSourceMerger<Src>
implements IndexedFeatureSource, UpdatableFeatureSource
{
constructor(...sources: Src[]) {
super(...sources)
}
async updateAsync() {
await Promise.all(this._sources.map((src) => src.updateAsync()))
}
}

56
src/Logic/FeatureSource/Sources/GeoJsonSource.ts
View file

@ -11,9 +11,14 @@ import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
import { Tiles } from "../../../Models/TileRange"
export default class GeoJsonSource implements FeatureSource {
public readonly features: Store<Feature[]>
private readonly _features: UIEventSource<Feature[]> = new UIEventSource<Feature[]>(undefined)
public readonly features: Store<Feature[]> = this._features
private readonly seenids: Set<string>
private readonly idKey?: string
private readonly url: string
private readonly layer: LayerConfig
private _isDownloaded = false
private currentlyRunning: Promise<any>
public constructor(
layer: LayerConfig,
@ -30,6 +35,7 @@ export default class GeoJsonSource implements FeatureSource {
this.idKey = layer.source.idKey
this.seenids = options?.featureIdBlacklist ?? new Set<string>()
let url = layer.source.geojsonSource.replace("{layer}", layer.id)
this.layer = layer
let zxy = options?.zxy
if (zxy !== undefined) {
let tile_bbox: BBox
@ -57,50 +63,48 @@ export default class GeoJsonSource implements FeatureSource {
.replace("{x_min}", "" + bounds.minLon)
.replace("{x_max}", "" + bounds.maxLon)
}
this.url = url
const eventsource = new UIEventSource<Feature[]>([])
if (options?.isActive !== undefined) {
options.isActive.addCallbackAndRunD(async (active) => {
if (!active) {
return
}
this.LoadJSONFrom(url, eventsource, layer)
.then((fs) => console.debug("Loaded", fs.length, "features from", url))
.catch((err) => console.warn("Could not load ", url, "due to", err))
this.updateAsync()
return true // data is loaded, we can safely unregister
})
} else {
this.LoadJSONFrom(url, eventsource, layer)
.then((fs) => console.debug("Loaded", fs.length, "features from", url))
.catch((err) => console.warn("Could not load ", url, "due to", err))
this.updateAsync()
}
this.features = eventsource
}
public async updateAsync(): Promise<void> {
if (!this.currentlyRunning) {
this.currentlyRunning = this.LoadJSONFrom()
}
await this.currentlyRunning
}
/**
* Init the download, write into the specified event source for the given layer.
* Note this method caches the requested geojson for five minutes
*/
private async LoadJSONFrom(
url: string,
eventSource: UIEventSource<Feature[]>,
layer: LayerConfig,
options?: {
maxCacheAgeSec?: number | 300
private async LoadJSONFrom(options?: { maxCacheAgeSec?: number | 300 }): Promise<Feature[]> {
if (this._isDownloaded) {
return
}
): Promise<Feature[]> {
const self = this
const url = this.url
try {
let json = await Utils.downloadJsonCached(url, (options?.maxCacheAgeSec ?? 300) * 1000)
if (json.features === undefined || json.features === null) {
json.features = []
}
if (layer.source.mercatorCrs) {
if (this.layer.source.mercatorCrs) {
json = GeoOperations.GeoJsonToWGS84(json)
}
const time = new Date()
const newFeatures: Feature[] = []
let i = 0
for (const feature of json.features) {
@ -120,8 +124,8 @@ export default class GeoJsonSource implements FeatureSource {
}
}
if (self.idKey !== undefined) {
props.id = props[self.idKey]
if (this.idKey !== undefined) {
props.id = props[this.idKey]
}
if (props.id === undefined) {
@ -129,14 +133,18 @@ export default class GeoJsonSource implements FeatureSource {
feature.id = url + "/" + i
i++
}
if (self.seenids.has(props.id)) {
if (this.seenids.has(props.id)) {
continue
}
self.seenids.add(props.id)
this.seenids.add(props.id)
newFeatures.push(feature)
}
eventSource.setData(newFeatures)
this._features.setData(newFeatures)
this._isDownloaded = true
return newFeatures
} catch (e) {
console.warn("Could not load ", url, "due to", e)
}
}
}

27
src/Logic/FeatureSource/Sources/LayoutSource.ts
View file

@ -1,17 +1,17 @@
import GeoJsonSource from "./GeoJsonSource"
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
import { FeatureSource } from "../FeatureSource"
import { UpdatableFeatureSource } from "../FeatureSource"
import { Or } from "../../Tags/Or"
import FeatureSwitchState from "../../State/FeatureSwitchState"
import OverpassFeatureSource from "./OverpassFeatureSource"
import { Store, UIEventSource } from "../../UIEventSource"
import OsmFeatureSource from "./OsmFeatureSource"
import FeatureSourceMerger from "./FeatureSourceMerger"
import DynamicGeoJsonTileSource from "../TiledFeatureSource/DynamicGeoJsonTileSource"
import { BBox } from "../../BBox"
import LocalStorageFeatureSource from "../TiledFeatureSource/LocalStorageFeatureSource"
import FullNodeDatabaseSource from "../TiledFeatureSource/FullNodeDatabaseSource"
import DynamicMvtileSource from "../TiledFeatureSource/DynamicMvtTileSource"
import FeatureSourceMerger from "./FeatureSourceMerger"
/**
* This source will fetch the needed data from various sources for the given layout.
@ -24,6 +24,8 @@ export default class LayoutSource extends FeatureSourceMerger {
*/
public readonly isLoading: Store<boolean>
private readonly supportsForceDownload: UpdatableFeatureSource[]
constructor(
layers: LayerConfig[],
featureSwitches: FeatureSwitchState,
@ -33,6 +35,8 @@ export default class LayoutSource extends FeatureSourceMerger {
mvtAvailableLayers: Set<string>,
fullNodeDatabaseSource?: FullNodeDatabaseSource
) {
const supportsForceDownload: UpdatableFeatureSource[] = []
const { bounds, zoom } = mapProperties
// remove all 'special' layers
layers = layers.filter((layer) => layer.source !== null && layer.source !== undefined)
@ -46,7 +50,7 @@ export default class LayoutSource extends FeatureSourceMerger {
maxAge: l.maxAgeOfCache,
})
)
const mvtSources: FeatureSource[] = osmLayers
const mvtSources: UpdatableFeatureSource[] = osmLayers
.filter((f) => mvtAvailableLayers.has(f.id))
.map((l) => LayoutSource.setupMvtSource(l, mapProperties, isDisplayed(l.id)))
const nonMvtSources = []
@ -79,24 +83,29 @@ export default class LayoutSource extends FeatureSourceMerger {
const loading = overpassSource?.runningQuery?.data || osmApiSource?.isRunning?.data
isLoading.setData(loading)
}
overpassSource?.runningQuery?.addCallbackAndRun((_) => setIsLoading())
osmApiSource?.isRunning?.addCallbackAndRun((_) => setIsLoading())
supportsForceDownload.push(overpassSource)
}
const geojsonSources: FeatureSource[] = geojsonlayers.map((l) =>
const geojsonSources: UpdatableFeatureSource[] = geojsonlayers.map((l) =>
LayoutSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id))
)
super(...geojsonSources, ...fromCache, ...mvtSources, ...nonMvtSources)
this.isLoading = isLoading
supportsForceDownload.push(...geojsonSources)
supportsForceDownload.push(...mvtSources) // Non-mvt sources are handled by overpass
this.supportsForceDownload = supportsForceDownload
}
private static setupMvtSource(
layer: LayerConfig,
mapProperties: { zoom: Store<number>; bounds: Store<BBox> },
isActive?: Store<boolean>
): FeatureSource {
): UpdatableFeatureSource {
return new DynamicMvtileSource(layer, mapProperties, { isActive })
}
@ -104,7 +113,7 @@ export default class LayoutSource extends FeatureSourceMerger {
layer: LayerConfig,
mapProperties: { zoom: Store<number>; bounds: Store<BBox> },
isActive?: Store<boolean>
): FeatureSource {
): UpdatableFeatureSource {
const source = layer.source
isActive = mapProperties.zoom.map(
(z) => (isActive?.data ?? true) && z >= layer.minzoom,
@ -190,4 +199,10 @@ export default class LayoutSource extends FeatureSourceMerger {
}
)
}
public async downloadAll() {
console.log("Downloading all data")
await Promise.all(this.supportsForceDownload.map((i) => i.updateAsync()))
console.log("Done")
}
}

82
src/Logic/FeatureSource/Sources/MvtSource.ts
View file

@ -1,8 +1,7 @@
import { Geometry } from "geojson"
import { Feature as GeojsonFeature } from "geojson"
import { Feature as GeojsonFeature, Geometry } from "geojson"
import { Store, UIEventSource } from "../../UIEventSource"
import { FeatureSourceForTile } from "../FeatureSource"
import { FeatureSourceForTile, UpdatableFeatureSource } from "../FeatureSource"
import Pbf from "pbf"
type Coords = [number, number][]
@ -205,6 +204,7 @@ class Layer {
end
)
}
static _readField(tag, obj, pbf) {
if (tag === 15) obj.version = pbf.readVarint()
else if (tag === 1) obj.name = pbf.readString()
@ -213,6 +213,7 @@ class Layer {
else if (tag === 4) obj.values.push(Value.read(pbf, pbf.readVarint() + pbf.pos))
else if (tag === 5) obj.extent = pbf.readVarint()
}
public static write(obj, pbf) {
if (obj.version) pbf.writeVarintField(15, obj.version)
if (obj.name) pbf.writeStringField(1, obj.name)
@ -230,12 +231,14 @@ class Feature {
static read(pbf, end) {
return pbf.readFields(Feature._readField, { id: 0, tags: [], type: 0, geometry: [] }, end)
}
static _readField(tag, obj, pbf) {
if (tag === 1) obj.id = pbf.readVarint()
else if (tag === 2) pbf.readPackedVarint(obj.tags)
else if (tag === 3) obj.type = pbf.readVarint()
else if (tag === 4) pbf.readPackedVarint(obj.geometry)
}
public static write(obj, pbf) {
if (obj.id) pbf.writeVarintField(1, obj.id)
if (obj.tags) pbf.writePackedVarint(2, obj.tags)
@ -260,6 +263,7 @@ class Value {
end
)
}
static _readField = function (tag, obj, pbf) {
if (tag === 1) obj.string_value = pbf.readString()
else if (tag === 2) obj.float_value = pbf.readFloat()
@ -269,6 +273,7 @@ class Value {
else if (tag === 6) obj.sint_value = pbf.readSVarint()
else if (tag === 7) obj.bool_value = pbf.readBoolean()
}
public static write(obj, pbf) {
if (obj.string_value) pbf.writeStringField(1, obj.string_value)
if (obj.float_value) pbf.writeFloatField(2, obj.float_value)
@ -279,21 +284,10 @@ class Value {
if (obj.bool_value) pbf.writeBooleanField(7, obj.bool_value)
}
}
class Tile {
// code generated by pbf v3.2.1
public static read(pbf, end) {
return pbf.readFields(Tile._readField, { layers: [] }, end)
}
static _readField(tag, obj, pbf) {
if (tag === 3) obj.layers.push(Layer.read(pbf, pbf.readVarint() + pbf.pos))
}
static write(obj, pbf) {
if (obj.layers)
for (var i = 0; i < obj.layers.length; i++)
pbf.writeMessage(3, Layer.write, obj.layers[i])
}
static GeomType = {
UNKNOWN: {
value: 0,
@ -312,10 +306,27 @@ class Tile {
options: {},
},
}
public static read(pbf, end) {
return pbf.readFields(Tile._readField, { layers: [] }, end)
}
static _readField(tag, obj, pbf) {
if (tag === 3) obj.layers.push(Layer.read(pbf, pbf.readVarint() + pbf.pos))
}
static write(obj, pbf) {
if (obj.layers)
for (var i = 0; i < obj.layers.length; i++)
pbf.writeMessage(3, Layer.write, obj.layers[i])
}
}
export default class MvtSource implements FeatureSourceForTile {
export default class MvtSource implements FeatureSourceForTile, UpdatableFeatureSource {
public readonly features: Store<GeojsonFeature<Geometry, { [name: string]: any }>[]>
public readonly x: number
public readonly y: number
public readonly z: number
private readonly _url: string
private readonly _layerName: string
private readonly _features: UIEventSource<
@ -326,9 +337,7 @@ export default class MvtSource implements FeatureSourceForTile {
}
>[]
> = new UIEventSource<GeojsonFeature<Geometry, { [p: string]: any }>[]>([])
public readonly x: number
public readonly y: number
public readonly z: number
private currentlyRunning: Promise<any>
constructor(
url: string,
@ -343,7 +352,7 @@ export default class MvtSource implements FeatureSourceForTile {
this.x = x
this.y = y
this.z = z
this.downloadSync()
this.updateAsync()
this.features = this._features.map(
(fs) => {
if (fs === undefined || isActive?.data === false) {
@ -355,6 +364,13 @@ export default class MvtSource implements FeatureSourceForTile {
)
}
async updateAsync() {
if (!this.currentlyRunning) {
this.currentlyRunning = this.download()
}
await this.currentlyRunning
}
private getValue(v: {
// Exactly one of these values must be present in a valid message
string_value?: string
@ -389,24 +405,12 @@ export default class MvtSource implements FeatureSourceForTile {
return undefined
}
private downloadSync() {
this.download()
.then((d) => {
if (d.length === 0) {
return
}
return this._features.setData(d)
})
.catch((e) => {
console.error(e)
})
}
private async download(): Promise<GeojsonFeature[]> {
private async download(): Promise<void> {
try {
const result = await fetch(this._url)
if (result.status !== 200) {
console.error("Could not download tile " + this._url)
return []
return
}
const buffer = await result.arrayBuffer()
const data = Tile.read(new Pbf(buffer), undefined)
@ -419,7 +423,7 @@ export default class MvtSource implements FeatureSourceForTile {
layer = layers.find((l) => l.name === this._layerName)
}
if (!layer) {
return []
return
}
const builder = new MvtFeatureBuilder(layer.extent, this.x, this.y, this.z)
const features: GeojsonFeature[] = []
@ -428,8 +432,10 @@ export default class MvtSource implements FeatureSourceForTile {
const properties = this.inflateProperties(feature.tags, layer.keys, layer.values)
features.push(builder.toGeoJson(feature.geometry, feature.type, properties))
}
return features
this._features.setData(features)
} catch (e) {
console.error("Could not download MVT tile due to", e)
}
}
private inflateProperties(tags: number[], keys: string[], values: { string_value: string }[]) {

19
src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts
View file

@ -1,5 +1,5 @@
import { Feature } from "geojson"
import { FeatureSource } from "../FeatureSource"
import { FeatureSource, UpdatableFeatureSource } from "../FeatureSource"
import { ImmutableStore, Store, UIEventSource } from "../../UIEventSource"
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
import { Or } from "../../Tags/Or"
@ -12,7 +12,7 @@ import { BBox } from "../../BBox"
* A wrapper around the 'Overpass'-object.
* It has more logic and will automatically fetch the data for the right bbox and the active layers
*/
export default class OverpassFeatureSource implements FeatureSource {
export default class OverpassFeatureSource implements UpdatableFeatureSource {
/**
* The last loaded features, as geojson
*/
@ -99,21 +99,15 @@ export default class OverpassFeatureSource implements FeatureSource {
) {
return undefined
}
const result = await this.updateAsync()
if (!result) {
return
}
const [bounds, _, __] = result
this._lastQueryBBox = bounds
await this.updateAsync()
}
/**
* Download the relevant data from overpass. Attempt to use a different server; only downloads the relevant layers
* @private
*/
private async updateAsync(): Promise<[BBox, Date, LayerConfig[]]> {
public async updateAsync(): Promise<void> {
let data: any = undefined
let date: Date = undefined
let lastUsed = 0
const layersToDownload = []
@ -172,7 +166,7 @@ export default class OverpassFeatureSource implements FeatureSource {
return undefined
}
this.runningQuery.setData(true)
;[data, date] = await overpass.queryGeoJson(bounds)
data = await overpass.queryGeoJson(bounds)[0]
} catch (e) {
self.retries.data++
self.retries.ping()
@ -205,10 +199,9 @@ export default class OverpassFeatureSource implements FeatureSource {
console.log("Overpass returned", data.features.length, "features")
self.features.setData(data.features)
return [bounds, date, layersToDownload]
this._lastQueryBBox = bounds
} catch (e) {
console.error("Got the overpass response, but could not process it: ", e, e.stack)
return undefined
} finally {
self.retries.setData(0)
self.runningQuery.setData(false)

4
src/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts
View file

@ -1,11 +1,11 @@
import { ImmutableStore, Store } from "../../UIEventSource"
import DynamicTileSource from "./DynamicTileSource"
import { UpdatableDynamicTileSource } from "./DynamicTileSource"
import { Utils } from "../../../Utils"
import GeoJsonSource from "../Sources/GeoJsonSource"
import { BBox } from "../../BBox"
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
export default class DynamicGeoJsonTileSource extends DynamicTileSource {
export default class DynamicGeoJsonTileSource extends UpdatableDynamicTileSource {
private static whitelistCache = new Map<string, any>()
constructor(

72
src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts
View file

@ -1,34 +1,37 @@
import { Store } from "../../UIEventSource"
import DynamicTileSource from "./DynamicTileSource"
import { UpdatableDynamicTileSource } from "./DynamicTileSource"
import { Utils } from "../../../Utils"
import { BBox } from "../../BBox"
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
import MvtSource from "../Sources/MvtSource"
import { Tiles } from "../../../Models/TileRange"
import Constants from "../../../Models/Constants"
import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
import { UpdatableFeatureSourceMerger } from "../Sources/FeatureSourceMerger"
import { LineSourceMerger } from "./LineSourceMerger"
import { PolygonSourceMerger } from "./PolygonSourceMerger"
class PolygonMvtSource extends PolygonSourceMerger{
constructor( layer: LayerConfig,
class PolygonMvtSource extends PolygonSourceMerger {
constructor(
layer: LayerConfig,
mapProperties: {
zoom: Store<number>
bounds: Store<BBox>
},
options?: {
isActive?: Store<boolean>
}) {
const roundedZoom = mapProperties.zoom.mapD(z => Math.min(Math.floor(z/2)*2, 14))
}
) {
const roundedZoom = mapProperties.zoom.mapD((z) => Math.min(Math.floor(z / 2) * 2, 14))
super(
roundedZoom,
layer.minzoom,
(zxy) => {
const [z, x, y] = Tiles.tile_from_index(zxy)
const url = Utils.SubstituteKeys(Constants.VectorTileServer,
{
z, x, y, layer: layer.id,
const url = Utils.SubstituteKeys(Constants.VectorTileServer, {
z,
x,
y,
layer: layer.id,
type: "polygons",
})
return new MvtSource(url, x, y, z)
@ -36,29 +39,33 @@ class PolygonMvtSource extends PolygonSourceMerger{
mapProperties,
{
isActive: options?.isActive,
})
}
)
}
}
class LineMvtSource extends LineSourceMerger{
constructor( layer: LayerConfig,
class LineMvtSource extends LineSourceMerger {
constructor(
layer: LayerConfig,
mapProperties: {
zoom: Store<number>
bounds: Store<BBox>
},
options?: {
isActive?: Store<boolean>
}) {
const roundedZoom = mapProperties.zoom.mapD(z => Math.min(Math.floor(z/2)*2, 14))
}
) {
const roundedZoom = mapProperties.zoom.mapD((z) => Math.min(Math.floor(z / 2) * 2, 14))
super(
roundedZoom,
layer.minzoom,
(zxy) => {
const [z, x, y] = Tiles.tile_from_index(zxy)
const url = Utils.SubstituteKeys(Constants.VectorTileServer,
{
z, x, y, layer: layer.id,
const url = Utils.SubstituteKeys(Constants.VectorTileServer, {
z,
x,
y,
layer: layer.id,
type: "lines",
})
return new MvtSource(url, x, y, z)
@ -66,13 +73,12 @@ class LineMvtSource extends LineSourceMerger{
mapProperties,
{
isActive: options?.isActive,
})
}
)
}
}
class PointMvtSource extends DynamicTileSource {
class PointMvtSource extends UpdatableDynamicTileSource {
constructor(
layer: LayerConfig,
mapProperties: {
@ -81,17 +87,19 @@ class PointMvtSource extends DynamicTileSource {
},
options?: {
isActive?: Store<boolean>
},
}
) {
const roundedZoom = mapProperties.zoom.mapD(z => Math.min(Math.floor(z/2)*2, 14))
const roundedZoom = mapProperties.zoom.mapD((z) => Math.min(Math.floor(z / 2) * 2, 14))
super(
roundedZoom,
layer.minzoom,
(zxy) => {
const [z, x, y] = Tiles.tile_from_index(zxy)
const url = Utils.SubstituteKeys(Constants.VectorTileServer,
{
z, x, y, layer: layer.id,
const url = Utils.SubstituteKeys(Constants.VectorTileServer, {
z,
x,
y,
layer: layer.id,
type: "pois",
})
return new MvtSource(url, x, y, z)
@ -99,13 +107,12 @@ class PointMvtSource extends DynamicTileSource {
mapProperties,
{
isActive: options?.isActive,
},
}
)
}
}
export default class DynamicMvtileSource extends FeatureSourceMerger {
export default class DynamicMvtileSource extends UpdatableFeatureSourceMerger {
constructor(
layer: LayerConfig,
mapProperties: {
@ -114,13 +121,12 @@ export default class DynamicMvtileSource extends FeatureSourceMerger {
},
options?: {
isActive?: Store<boolean>
},
}
) {
super(
new PointMvtSource(layer, mapProperties, options),
new LineMvtSource(layer, mapProperties, options),
new PolygonMvtSource(layer, mapProperties, options)
)
}
}

94
src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts
View file

@ -1,7 +1,7 @@
import { Store, Stores } from "../../UIEventSource"
import { Tiles } from "../../../Models/TileRange"
import { BBox } from "../../BBox"
import { FeatureSource } from "../FeatureSource"
import { FeatureSource, UpdatableFeatureSource } from "../FeatureSource"
import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
/***
@ -11,6 +11,12 @@ import FeatureSourceMerger from "../Sources/FeatureSourceMerger"
export default class DynamicTileSource<
Src extends FeatureSource = FeatureSource
> extends FeatureSourceMerger<Src> {
private readonly loadedTiles = new Set<number>()
private readonly zDiff: number
private readonly zoomlevel: Store<number>
private readonly constructSource: (tileIndex: number) => Src
private readonly bounds: Store<BBox>
/**
*
* @param zoomlevel If {z} is specified in the source, the 'zoomlevel' will be used as zoomlevel to download from
@ -33,12 +39,14 @@ export default class DynamicTileSource<
}
) {
super()
const loadedTiles = new Set<number>()
const zDiff = options?.zDiff ?? 0
this.constructSource = constructSource
this.zoomlevel = zoomlevel
this.zDiff = options?.zDiff ?? 0
this.bounds = mapProperties.bounds
const neededTiles: Store<number[]> = Stores.ListStabilized(
mapProperties.bounds
.mapD(
(bounds) => {
.mapD(() => {
if (options?.isActive && !options?.isActive.data) {
return undefined
}
@ -46,7 +54,28 @@ export default class DynamicTileSource<
if (mapProperties.zoom.data < minzoom) {
return undefined
}
const z = Math.floor(zoomlevel.data) + zDiff
return this.getNeededTileIndices()
}, [options?.isActive, mapProperties.zoom])
.stabilized(250)
)
neededTiles.addCallbackAndRunD((neededIndexes) => this.downloadTiles(neededIndexes))
}
protected downloadTiles(neededIndexes: number[]): Src[] {
const sources: Src[] = []
for (const neededIndex of neededIndexes) {
this.loadedTiles.add(neededIndex)
const src = this.constructSource(neededIndex)
super.addSource(src)
sources.push(src)
}
return sources
}
protected getNeededTileIndices() {
const bounds = this.bounds.data
const z = Math.floor(this.zoomlevel.data) + this.zDiff
const tileRange = Tiles.TileRangeBetween(
z,
bounds.getNorth(),
@ -55,30 +84,41 @@ export default class DynamicTileSource<
bounds.getWest()
)
if (tileRange.total > 500) {
console.warn(
"Got a really big tilerange, bounds and location might be out of sync"
)
return undefined
console.warn("Got a really big tilerange, bounds and location might be out of sync")
return []
}
const needed = Tiles.MapRange(tileRange, (x, y) =>
Tiles.tile_index(z, x, y)
).filter((i) => !loadedTiles.has(i))
const needed = Tiles.MapRange(tileRange, (x, y) => Tiles.tile_index(z, x, y)).filter(
(i) => !this.loadedTiles.has(i)
)
if (needed.length === 0) {
return undefined
return []
}
return needed
},
[options?.isActive, mapProperties.zoom]
)
.stabilized(250)
)
neededTiles.addCallbackAndRunD((neededIndexes) => {
for (const neededIndex of neededIndexes) {
loadedTiles.add(neededIndex)
super.addSource(constructSource(neededIndex))
}
})
}
}
export class UpdatableDynamicTileSource<Src extends UpdatableFeatureSource = UpdatableFeatureSource>
extends DynamicTileSource<Src>
implements UpdatableFeatureSource
{
constructor(
zoomlevel: Store<number>,
minzoom: number,
constructSource: (tileIndex: number) => Src,
mapProperties: {
bounds: Store<BBox>
zoom: Store<number>
},
options?: {
isActive?: Store<boolean>
zDiff?: number
}
) {
super(zoomlevel, minzoom, constructSource, mapProperties, options)
}
async updateAsync() {
const sources = super.downloadTiles(super.getNeededTileIndices())
await Promise.all(sources.map((src) => src.updateAsync()))
}
}

44
src/Logic/FeatureSource/TiledFeatureSource/LineSourceMerger.ts
View file

@ -1,30 +1,31 @@
import { FeatureSourceForTile } from "../FeatureSource"
import { FeatureSourceForTile, UpdatableFeatureSource } from "../FeatureSource"
import { Store } from "../../UIEventSource"
import { BBox } from "../../BBox"
import { Utils } from "../../../Utils"
import { Feature, LineString, MultiLineString, Position } from "geojson"
import { Tiles } from "../../../Models/TileRange"
import { Feature, MultiLineString, Position } from "geojson"
import { GeoOperations } from "../../GeoOperations"
import DynamicTileSource from "./DynamicTileSource"
import { UpdatableDynamicTileSource } from "./DynamicTileSource"
/**
* The PolygonSourceMerger receives various small pieces of bigger polygons and stitches them together.
* This is used to reconstruct polygons of vector tiles
*/
export class LineSourceMerger extends DynamicTileSource<FeatureSourceForTile> {
export class LineSourceMerger extends UpdatableDynamicTileSource<
FeatureSourceForTile & UpdatableFeatureSource
> {
private readonly _zoomlevel: Store<number>
constructor(
zoomlevel: Store<number>,
minzoom: number,
constructSource: (tileIndex: number) => FeatureSourceForTile,
constructSource: (tileIndex: number) => FeatureSourceForTile & UpdatableFeatureSource,
mapProperties: {
bounds: Store<BBox>
zoom: Store<number>
},
options?: {
isActive?: Store<boolean>
},
}
) {
super(zoomlevel, minzoom, constructSource, mapProperties, options)
this._zoomlevel = zoomlevel
@ -35,32 +36,29 @@ export class LineSourceMerger extends DynamicTileSource<FeatureSourceForTile> {
const all: Map<string, Feature<MultiLineString>> = new Map()
const currentZoom = this._zoomlevel?.data ?? 0
for (const source of sources) {
if(source.z != currentZoom){
if (source.z != currentZoom) {
continue
}
const bboxCoors = Tiles.tile_bounds_lon_lat(source.z, source.x, source.y)
const bboxGeo = new BBox(bboxCoors).asGeoJson({})
for (const f of source.features.data) {
const id = f.properties.id
const coordinates : Position[][] = []
if(f.geometry.type === "LineString"){
const coordinates: Position[][] = []
if (f.geometry.type === "LineString") {
coordinates.push(f.geometry.coordinates)
}else if(f.geometry.type === "MultiLineString"){
} else if (f.geometry.type === "MultiLineString") {
coordinates.push(...f.geometry.coordinates)
}else {
} else {
console.error("Invalid geometry type:", f.geometry.type)
continue
}
const oldV = all.get(id)
if(!oldV){
if (!oldV) {
all.set(id, {
type: "Feature",
properties: f.properties,
geometry:{
type:"MultiLineString",
coordinates
}
geometry: {
type: "MultiLineString",
coordinates,
},
})
continue
}
@ -70,11 +68,13 @@ export class LineSourceMerger extends DynamicTileSource<FeatureSourceForTile> {
const keys = Array.from(all.keys())
for (const key of keys) {
all.set(key, <any> GeoOperations.attemptLinearize(<Feature<MultiLineString>>all.get(key)))
all.set(
key,
<any>GeoOperations.attemptLinearize(<Feature<MultiLineString>>all.get(key))
)
}
const newList = Array.from(all.values())
this.features.setData(newList)
this._featuresById.setData(all)
}
}

13
src/Logic/FeatureSource/TiledFeatureSource/PolygonSourceMerger.ts
View file

@ -1,27 +1,29 @@
import { FeatureSourceForTile } from "../FeatureSource"
import { FeatureSourceForTile, UpdatableFeatureSource } from "../FeatureSource"
import { Store } from "../../UIEventSource"
import { BBox } from "../../BBox"
import { Utils } from "../../../Utils"
import { Feature } from "geojson"
import { GeoOperations } from "../../GeoOperations"
import DynamicTileSource from "./DynamicTileSource"
import DynamicTileSource, { UpdatableDynamicTileSource } from "./DynamicTileSource"
/**
* The PolygonSourceMerger receives various small pieces of bigger polygons and stitches them together.
* This is used to reconstruct polygons of vector tiles
*/
export class PolygonSourceMerger extends DynamicTileSource<FeatureSourceForTile> {
export class PolygonSourceMerger extends UpdatableDynamicTileSource<
FeatureSourceForTile & UpdatableFeatureSource
> {
constructor(
zoomlevel: Store<number>,
minzoom: number,
constructSource: (tileIndex: number) => FeatureSourceForTile,
constructSource: (tileIndex: number) => FeatureSourceForTile & UpdatableFeatureSource,
mapProperties: {
bounds: Store<BBox>
zoom: Store<number>
},
options?: {
isActive?: Store<boolean>
},
}
) {
super(zoomlevel, minzoom, constructSource, mapProperties, options)
}
@ -69,5 +71,4 @@ export class PolygonSourceMerger extends DynamicTileSource<FeatureSourceForTile>
this.features.setData(newList)
this._featuresById.setData(all)
}
}

17
src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource.ts
View file

@ -14,6 +14,10 @@ export class SummaryTileSourceRewriter implements FeatureSource {
private filteredLayers: FilteredLayer[]
public readonly features: Store<Feature[]> = this._features
private readonly _summarySource: SummaryTileSource
private readonly _totalNumberOfFeatures: UIEventSource<number> = new UIEventSource<number>(
undefined
)
public readonly totalNumberOfFeatures: Store<number> = this._totalNumberOfFeatures
constructor(
summarySource: SummaryTileSource,
filteredLayers: ReadonlyMap<string, FilteredLayer>
@ -31,6 +35,7 @@ export class SummaryTileSourceRewriter implements FeatureSource {
}
private update() {
let fullTotal = 0
const newFeatures: Feature[] = []
const layersToCount = this.filteredLayers.filter((fl) => fl.isDisplayed.data)
const bitmap = layersToCount.map((l) => (l.isDisplayed.data ? "1" : "0")).join("")
@ -42,10 +47,17 @@ export class SummaryTileSourceRewriter implements FeatureSource {
}
newFeatures.push({
...f,
properties: { ...f.properties, id: f.properties.id + bitmap, total: newTotal },
properties: {
...f.properties,
id: f.properties.id + bitmap,
total: newTotal,
total_metric: Utils.numberWithMetrixPrefix(newTotal),
},
})
fullTotal += newTotal
}
this._features.setData(newFeatures)
this._totalNumberOfFeatures.setData(fullTotal)
}
}
@ -94,7 +106,7 @@ export class SummaryTileSource extends DynamicTileSource {
}
const lat = counts["lat"]
const lon = counts["lon"]
const total = Utils.numberWithMetrixPrefix(Number(counts["total"]))
const total = Number(counts["total"])
const tileBbox = new BBox(Tiles.tile_bounds_lon_lat(z, x, y))
if (!tileBbox.contains([lon, lat])) {
console.error(
@ -116,6 +128,7 @@ export class SummaryTileSource extends DynamicTileSource {
summary: "yes",
...counts,
total,
total_metric: Utils.numberWithMetrixPrefix(total),
layers: layersSummed,
},
geometry: {

32
src/Models/ThemeViewState.ts
View file

@ -114,6 +114,7 @@ export default class ThemeViewState implements SpecialVisualizationState {
readonly closestFeatures: NearbyFeatureSource
readonly newFeatures: WritableFeatureSource
readonly layerState: LayerState
readonly featureSummary: SummaryTileSourceRewriter
readonly perLayer: ReadonlyMap<string, GeoIndexedStoreForLayer>
readonly perLayerFiltered: ReadonlyMap<string, FilteringFeatureSource>
@ -378,6 +379,7 @@ export default class ThemeViewState implements SpecialVisualizationState {
)
this.favourites = new FavouritesFeatureSource(this)
this.featureSummary = this.setupSummaryLayer()
this.initActors()
this.drawSpecialLayers()
this.initHotkeys()
@ -660,7 +662,17 @@ export default class ThemeViewState implements SpecialVisualizationState {
)
}
private setupSummaryLayer(maxzoom: number) {
private setupSummaryLayer(): SummaryTileSourceRewriter {
/**
* MaxZoom for the summary layer
*/
const normalLayers = this.layout.layers.filter(
(l) =>
Constants.priviliged_layers.indexOf(<any>l.id) < 0 &&
!l.id.startsWith("note_import")
)
const maxzoom = Math.min(...normalLayers.map((l) => l.minzoom))
const layers = this.layout.layers.filter(
(l) =>
Constants.priviliged_layers.indexOf(<any>l.id) < 0 &&
@ -684,22 +696,6 @@ export default class ThemeViewState implements SpecialVisualizationState {
private drawSpecialLayers() {
type AddedByDefaultTypes = (typeof Constants.added_by_default)[number]
const empty = []
/**
* MaxZoom for the summary layer
*/
const normalLayers = this.layout.layers.filter(
(l) =>
Constants.priviliged_layers.indexOf(<any>l.id) < 0 &&
!l.id.startsWith("note_import")
)
const maxzoom = Math.min(...normalLayers.map((l) => l.minzoom))
console.log(
"Maxzoom for summary layer is",
maxzoom,
normalLayers.map((nl) => nl.id + " - " + nl.minzoom).join(", ")
)
/**
* A listing which maps the layerId onto the featureSource
*/
@ -721,7 +717,7 @@ export default class ThemeViewState implements SpecialVisualizationState {
),
current_view: this.currentView,
favourite: this.favourites,
summary: this.setupSummaryLayer(maxzoom),
summary: this.featureSummary,
}
this.closestFeatures.registerSource(specialLayers.favourite, "favourite")

20
src/UI/DownloadFlow/DownloadButton.svelte
View file

@ -3,10 +3,8 @@
import { ArrowDownTrayIcon } from "@babeard/svelte-heroicons/mini"
import Tr from "../Base/Tr.svelte"
import Translations from "../i18n/Translations"
import type { FeatureCollection } from "geojson"
import Loading from "../Base/Loading.svelte"
import { Translation } from "../i18n/Translation"
import DownloadHelper from "./DownloadHelper"
import { Utils } from "../../Utils"
import type { PriviligedLayerType } from "../../Models/Constants"
import { UIEventSource } from "../../Logic/UIEventSource"
@ -16,14 +14,11 @@
export let extension: string
export let mimetype: string
export let construct: (
geojsonCleaned: FeatureCollection,
title: string,
status?: UIEventSource<string>
) => (Blob | string) | Promise<void>
) => Promise<Blob | string>
export let mainText: Translation
export let helperText: Translation
export let metaIsIncluded: boolean
let downloadHelper: DownloadHelper = new DownloadHelper(state)
const t = Translations.t.general.download
@ -31,30 +26,21 @@
let isError = false
let status: UIEventSource<string> = new UIEventSource<string>(undefined)
async function clicked() {
isExporting = true
const gpsLayer = state.layerState.filteredLayers.get(<PriviligedLayerType>"gps_location")
state.userRelatedState.preferencesAsTags.data["__showTimeSensitiveIcons"] = "no"
state.userRelatedState.preferencesAsTags.ping()
const gpsIsDisplayed = gpsLayer.isDisplayed.data
try {
gpsLayer.isDisplayed.setData(false)
const geojson: FeatureCollection = downloadHelper.getCleanGeoJson(metaIsIncluded)
const name = state.layout.id
const title = `MapComplete_${name}_export_${new Date()
.toISOString()
.substr(0, 19)}.${extension}`
const promise = construct(geojson, title, status)
let data: Blob | string
if (typeof promise === "string") {
data = promise
} else if (typeof promise["then"] === "function") {
data = await (<Promise<Blob | string>>promise)
} else {
data = <Blob>promise
}
const data: Blob | string = await construct(title, status)
if (!data) {
return
}

3
src/UI/DownloadFlow/DownloadHelper.ts
View file

@ -153,7 +153,7 @@ export default class DownloadHelper {
return header + "\n" + elements.join("\n") + "\n</svg>"
}
public getCleanGeoJsonPerLayer(includeMetaData: boolean): Map<string, Feature[]> {
private getCleanGeoJsonPerLayer(includeMetaData: boolean): Map<string, Feature[]> {
const state = this._state
const featuresPerLayer = new Map<string, any[]>()
const neededLayers = state.layout.layers.filter((l) => l.source !== null).map((l) => l.id)
@ -161,6 +161,7 @@ export default class DownloadHelper {
for (const neededLayer of neededLayers) {
const indexedFeatureSource = state.perLayer.get(neededLayer)
let features = indexedFeatureSource.GetFeaturesWithin(bbox)
// The 'indexedFeatureSources' contains _all_ features, they are not filtered yet
const filter = state.layerState.filteredLayers.get(neededLayer)

22
src/UI/DownloadFlow/DownloadPanel.svelte
View file

@ -17,9 +17,16 @@
const downloadHelper = new DownloadHelper(state)
let metaIsIncluded = false
const name = state.layout.id
function offerSvg(noSelfIntersectingLines: boolean): string {
let numberOfFeatures = state.featureSummary.totalNumberOfFeatures
async function getGeojson() {
await state.indexedFeatures.downloadAll()
return downloadHelper.getCleanGeoJson(metaIsIncluded)
}
async function offerSvg(noSelfIntersectingLines: boolean): Promise<string> {
await state.indexedFeatures.downloadAll()
const maindiv = document.getElementById("maindiv")
const layers = state.layout.layers.filter((l) => l.source !== null)
return downloadHelper.asSvg({
@ -34,6 +41,8 @@
{#if $isLoading}
<Loading />
{:else if $numberOfFeatures > 100000}
<Tr cls="alert" t={Translations.t.general.download.toMuch} />
{:else}
<div class="flex w-full flex-col" />
<h3>
@ -44,20 +53,18 @@
{state}
extension="geojson"
mimetype="application/vnd.geo+json"
construct={(geojson) => JSON.stringify(geojson)}
construct={async () => JSON.stringify(await getGeojson())}
mainText={t.downloadGeojson}
helperText={t.downloadGeoJsonHelper}
{metaIsIncluded}
/>
<DownloadButton
{state}
extension="csv"
mimetype="text/csv"
construct={(geojson) => GeoOperations.toCSV(geojson)}
construct={async () => GeoOperations.toCSV(await getGeojson())}
mainText={t.downloadCSV}
helperText={t.downloadCSVHelper}
{metaIsIncluded}
/>
<label class="mb-8 mt-2">
@ -67,7 +74,6 @@
<DownloadButton
{state}
{metaIsIncluded}
extension="svg"
mimetype="image/svg+xml"
mainText={t.downloadAsSvg}
@ -77,7 +83,6 @@
<DownloadButton
{state}
{metaIsIncluded}
extension="svg"
mimetype="image/svg+xml"
mainText={t.downloadAsSvgLinesOnly}
@ -87,7 +92,6 @@
<DownloadButton
{state}
{metaIsIncluded}
extension="png"
mimetype="image/png"
mainText={t.downloadAsPng}

7
src/UI/DownloadFlow/DownloadPdf.svelte
View file

@ -19,7 +19,7 @@
let t = Translations.t.general.download
const downloadHelper = new DownloadHelper(state)
async function constructPdf(_, title: string, status: UIEventSource<string>) {
async function constructPdf(title: string, status: UIEventSource<string>): Promise<Blob> {
title =
title.substring(0, title.length - 4) + "_" + template.format + "_" + template.orientation
const templateUrls = SvgToPdf.templates[templateName].pages
@ -33,11 +33,11 @@
console.log("Creating an image for key", key)
if (key === "qr") {
const toShare = window.location.href.split("#")[0]
return new Qr(toShare).toImageElement(parseFloat(width), parseFloat(height))
return new Qr(toShare).toImageElement(parseFloat(width))
}
return downloadHelper.createImage(key, width, height)
},
textSubstitutions: <Record<string, string>>{
textSubstitutions: <Record<string, string | Translation>>{
"layout.title": state.layout.title,
layoutid: state.layout.id,
title: state.layout.title,
@ -61,7 +61,6 @@
construct={constructPdf}
extension="pdf"
helperText={t.downloadAsPdfHelper}
metaIsIncluded={false}
mainText={t.pdf.current_view_generic.Subs({
orientation: template.orientation,
paper_size: template.format.toUpperCase(),

5
src/UI/SpecialVisualization.ts
View file

@ -20,6 +20,8 @@ import { OsmTags } from "../Models/OsmFeature"
import FavouritesFeatureSource from "../Logic/FeatureSource/Sources/FavouritesFeatureSource"
import { ProvidedImage } from "../Logic/ImageProviders/ImageProvider"
import GeoLocationHandler from "../Logic/Actors/GeoLocationHandler"
import { SummaryTileSourceRewriter } from "../Logic/FeatureSource/TiledFeatureSource/SummaryTileSource"
import LayoutSource from "../Logic/FeatureSource/Sources/LayoutSource"
/**
* The state needed to render a special Visualisation.
@ -30,12 +32,13 @@ export interface SpecialVisualizationState {
readonly featureSwitches: FeatureSwitchState
readonly layerState: LayerState
readonly featureSummary: SummaryTileSourceRewriter
readonly featureProperties: {
getStore(id: string): UIEventSource<Record<string, string>>
trackFeature?(feature: { properties: OsmTags })
}
readonly indexedFeatures: IndexedFeatureSource
readonly indexedFeatures: IndexedFeatureSource & LayoutSource
/**
* Some features will create a new element that should be displayed.
* These can be injected by appending them to this featuresource (and pinging it)

1
src/UI/StatisticsGUI.ts
View file

@ -16,7 +16,6 @@ import mcChanges from "../../src/assets/generated/themes/mapcomplete-changes.jso
import SvelteUIElement from "./Base/SvelteUIElement"
import Filterview from "./BigComponents/Filterview.svelte"
import FilteredLayer from "../Models/FilteredLayer"
import DownloadButton from "./DownloadFlow/DownloadButton.svelte"
import { SubtleButton } from "./Base/SubtleButton"
import { GeoOperations } from "../Logic/GeoOperations"
import { Polygon } from "geojson"

12
src/Utils/svgToPdf.ts
View file

@ -553,7 +553,7 @@ class SvgToPdfInternals {
export interface SvgToPdfOptions {
freeComponentId: string
disableMaps?: false | true
textSubstitutions?: Record<string, string>
textSubstitutions?: Record<string, string | Translation>
beforePage?: (i: number) => void
overrideLocation?: { lat: number; lon: number }
disableDataLoading?: boolean | false
@ -711,9 +711,13 @@ class SvgToPdfPage {
this.options.beforePage(i)
}
const self = this
const internal = new SvgToPdfInternals(advancedApi, this, (key) =>
self.extractTranslation(key, language)
)
const internal = new SvgToPdfInternals(advancedApi, this, (key) => {
const tr = self.extractTranslation(key, language)
if (typeof tr === "string") {
return tr
}
return tr.txt
})
for (const child of Array.from(this._svgRoot.children)) {
internal.handleElement(<any>child)
}