Osmwithspace/MapComplete
1
0
Fork 0
forked from MapComplete/MapComplete
Code Pull requests Activity

Fix duplicate buildings for grb layer; add default flag for filters, performance improvement

Browse source
This commit is contained in:
Pieter Vander Vennet 2022-02-11 03:57:39 +01:00
parent 31205f3430
commit 695a0867c7
13 changed files with 157 additions and 111 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -204,7 +204,7 @@ export default class FeaturePipeline {
TiledFeatureSource.createHierarchy(src, {
layer: src.layer,
minZoomLevel: this.osmSourceZoomLevel,
dontEnforceMinZoom: true,
noDuplicates: true,
registerTile: (tile) => {
new RegisteringAllFromFeatureSourceActor(tile, state.allElements)
perLayerHierarchy.get(id).registerTile(tile)
@ -276,7 +276,7 @@ export default class FeaturePipeline {
(source) => TiledFeatureSource.createHierarchy(source, {
layer: source.layer,
minZoomLevel: source.layer.layerDef.minzoom,
dontEnforceMinZoom: true,
noDuplicates: true,
maxFeatureCount: state.layoutToUse.clustering.minNeededElements,
maxZoomLevel: state.layoutToUse.clustering.maxZoom,
registerTile: (tile) => {

23
Logic/FeatureSource/Sources/GeoJsonSource.ts
View file

@ -18,19 +18,13 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
public readonly layer: FilteredLayer;
public readonly tileIndex
public readonly bbox;
private readonly seenids: Set<string> = new Set<string>()
/**
* Only used if the actual source is a tiled geojson.
* A big feature might be contained in multiple tiles.
* However, we only want to load them once. The blacklist thus contains all ids of all features previously seen
* @private
*/
private readonly featureIdBlacklist?: UIEventSource<Set<string>>
private readonly seenids: Set<string>;
private readonly idKey ?: string;
public constructor(flayer: FilteredLayer,
zxy?: [number, number, number] | BBox,
options?: {
featureIdBlacklist?: UIEventSource<Set<string>>
featureIdBlacklist?: Set<string>
}) {
if (flayer.layerDef.source.geojsonZoomLevel !== undefined && zxy === undefined) {
@ -38,7 +32,8 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
}
this.layer = flayer;
this.featureIdBlacklist = options?.featureIdBlacklist
this.idKey = flayer.layerDef.source.idKey
this.seenids = options?.featureIdBlacklist ?? new Set<string>()
let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id);
if (zxy !== undefined) {
let tile_bbox: BBox;
@ -106,6 +101,10 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
}
}
if(self.idKey !== undefined){
props.id = props[self.idKey]
}
if (props.id === undefined) {
props.id = url + "/" + i;
feature.id = url + "/" + i;
@ -117,10 +116,6 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
}
self.seenids.add(props.id)
if (self.featureIdBlacklist?.data?.has(props.id)) {
continue;
}
let freshness: Date = time;
if (feature.properties["_last_edit:timestamp"] !== undefined) {
freshness = new Date(props["_last_edit:timestamp"])

2
Logic/FeatureSource/Sources/NewGeometryFromChangesFeatureSource.ts
View file

@ -20,7 +20,7 @@ export class NewGeometryFromChangesFeatureSource implements FeatureSource {
const features = this.features.data;
const self = this;
changes.pendingChanges.addCallbackAndRunD(changes => {
changes.pendingChanges.stabilized(100).addCallbackAndRunD(changes => {
if (changes.length === 0) {
return;
}

8
Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts
View file

@ -55,8 +55,7 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
}
}
const seenIds = new Set<string>();
const blackList = new UIEventSource(seenIds)
const blackList = (new Set<string>())
super(
layer,
source.geojsonZoomLevel,
@ -76,10 +75,7 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
featureIdBlacklist: blackList
}
)
src.features.addCallbackAndRunD(feats => {
feats.forEach(feat => seenIds.add(feat.feature.properties.id))
blackList.ping();
})
registerLayer(src)
return src
},

1
Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts
View file

@ -21,7 +21,6 @@ export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer
* Add another feature source for the given tile.
* Entries for this tile will be merged
* @param src
* @param index
*/
public registerTile(src: FeatureSource & Tiled) {

13
Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource.ts
View file

@ -146,7 +146,10 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
for (const feature of features) {
const bbox = BBox.get(feature.feature)
if (this.options.dontEnforceMinZoom) {
// There are a few strategies to deal with features that cross tile boundaries
if (this.options.noDuplicates) {
// Strategy 1: We put the feature into a somewhat matching tile
if (bbox.overlapsWith(this.upper_left.bbox)) {
ulf.push(feature)
} else if (bbox.overlapsWith(this.upper_right.bbox)) {
@ -159,6 +162,7 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
overlapsboundary.push(feature)
}
} else if (this.options.minZoomLevel === undefined) {
// Strategy 2: put it into a strictly matching tile (or in this tile, which is slightly too big)
if (bbox.isContainedIn(this.upper_left.bbox)) {
ulf.push(feature)
} else if (bbox.isContainedIn(this.upper_right.bbox)) {
@ -171,7 +175,7 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
overlapsboundary.push(feature)
}
} else {
// We duplicate a feature on a boundary into every tile as we need to get to the minZoomLevel
// Strategy 3: We duplicate a feature on a boundary into every tile as we need to get to the minZoomLevel
if (bbox.overlapsWith(this.upper_left.bbox)) {
ulf.push(feature)
}
@ -201,10 +205,9 @@ export interface TiledFeatureSourceOptions {
readonly minZoomLevel?: number,
/**
* IF minZoomLevel is set, and if a feature runs through a tile boundary, it would normally be duplicated.
* Setting 'dontEnforceMinZoomLevel' will still allow bigger zoom levels for those features.
* If 'pick_first' is set, the feature will not be duplicated but set to some tile
* Setting 'dontEnforceMinZoomLevel' will assign to feature to some matching subtile.
*/
readonly dontEnforceMinZoom?: boolean | "pick_first",
readonly noDuplicates?: boolean,
readonly registerTile?: (tile: TiledFeatureSource & FeatureSourceForLayer & Tiled) => void,
readonly layer?: FilteredLayer
}

22
Models/ThemeConfig/FilterConfig.ts
View file

@ -18,6 +18,7 @@ export default class FilterConfig {
originalTagsSpec: string | AndOrTagConfigJson
fields: { name: string, type: string }[]
}[];
public readonly defaultSelection : number
constructor(json: FilterConfigJson, context: string) {
if (json.options === undefined) {
@ -35,6 +36,7 @@ export default class FilterConfig {
throw `A filter was given where the options aren't a list at ${context}`
}
this.id = json.id;
let defaultSelection : number = undefined
this.options = json.options.map((option, i) => {
const ctx = `${context}.options[${i}]`;
const question = Translations.T(
@ -66,10 +68,19 @@ export default class FilterConfig {
}
})
if(option.default){
if(defaultSelection === undefined){
defaultSelection = i;
}else{
throw `Invalid filter: multiple filters are set as default, namely ${i} and ${defaultSelection} at ${context}`
}
}
return {question: question, osmTags: osmTags, fields, originalTagsSpec: option.osmTags};
});
this.defaultSelection = defaultSelection ?? 0
if (this.options.some(o => o.fields.length > 0) && this.options.length > 1) {
throw `Invalid filter at ${context}: a filter with textfields should only offer a single option.`
}
@ -77,6 +88,8 @@ export default class FilterConfig {
if (this.options.length > 1 && this.options[0].osmTags !== undefined) {
throw "Error in " + context + "." + this.id + ": the first option of a multi-filter should always be the 'reset' option and not have any filters"
}
}
public initState(): UIEventSource<FilterState> {
@ -88,7 +101,14 @@ export default class FilterConfig {
return "" + state.state
}
const defaultValue = this.options.length > 1 ? "0" : ""
let defaultValue = ""
if(this.options.length > 1){
defaultValue = ""+this.defaultSelection
}else{
if(this.defaultSelection > 0){
defaultValue = ""+this.defaultSelection
}
}
const qp = QueryParameters.GetQueryParameter("filter-" + this.id, defaultValue, "State of filter " + this.id)
if (this.options.length > 1) {

1
Models/ThemeConfig/Json/FilterConfigJson.ts
View file

@ -14,6 +14,7 @@ export default interface FilterConfigJson {
options: {
question: string | any;
osmTags?: AndOrTagConfigJson | string,
default?: boolean,
fields?: {
name: string,
type?: string | "string"

59
Models/ThemeConfig/Json/LayerConfigJson.ts
View file

@ -33,19 +33,34 @@ export interface LayerConfigJson {
/**
* This determines where the data for the layer is fetched.
* There are some options:
* This determines where the data for the layer is fetched: from OSM or from an external geojson dataset.
*
* # Query OSM directly
* source: {osmTags: "key=value"}
* will fetch all objects with given tags from OSM.
* Currently, this will create a query to overpass and fetch the data - in the future this might fetch from the OSM API
* If no 'geojson' is defined, data will be fetched from overpass and the OSM-API.
*
* # Query OSM Via the overpass API with a custom script
* Every source _must_ define which tags _must_ be present in order to be picked up.
*
*/
source:
({
/**
* Every source must set which tags have to be present in order to load the given layer.
*/
osmTags: AndOrTagConfigJson | string
/**
* The maximum amount of seconds that a tile is allowed to linger in the cache
*/
maxCacheAge?: number
}) &
({ /* # Query OSM Via the overpass API with a custom script
* source: {overpassScript: "<custom overpass tags>"} when you want to do special things. _This should be really rare_.
* This means that the data will be pulled from overpass with this script, and will ignore the osmTags for the query
* However, for the rest of the pipeline, the OsmTags will _still_ be used. This is important to enable layers etc...
*
*/
overpassScript?: string
} |
{
/**
* The actual source of the data to load, if loaded via geojson.
*
* # A single geojson-file
* source: {geoJson: "https://my.source.net/some-geo-data.geojson"}
@ -56,20 +71,26 @@ export interface LayerConfigJson {
* to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted by the location; {layer} is substituted with the id of the loaded layer
*
* Some API's use a BBOX instead of a tile, this can be used by specifying {y_min}, {y_max}, {x_min} and {x_max}
* Some API's use a mercator-projection (EPSG:900913) instead of WGS84. Set the flag `mercatorCrs: true` in the source for this
*
* Note that both geojson-options might set a flag 'isOsmCache' indicating that the data originally comes from OSM too
*
*
* NOTE: the previous format was 'overpassTags: AndOrTagConfigJson | string', which is interpreted as a shorthand for source: {osmTags: "key=value"}
* While still supported, this is considered deprecated
*/
source: ({ osmTags: AndOrTagConfigJson | string, overpassScript?: string } |
{ osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number, isOsmCache?: boolean, mercatorCrs?: boolean }) & ({
geoJson: string,
/**
* The maximum amount of seconds that a tile is allowed to linger in the cache
* To load a tiled geojson layer, set the zoomlevel of the tiles
*/
maxCacheAge?: number
geoJsonZoomLevel?: number,
/**
* Indicates that the upstream geojson data is OSM-derived.
* Useful for e.g. merging or for scripts generating this cache
*/
isOsmCache?: boolean,
/**
* Some API's use a mercator-projection (EPSG:900913) instead of WGS84. Set the flag `mercatorCrs: true` in the source for this
*/
mercatorCrs?: boolean,
/**
* Some API's have an id-field, but give it a different name.
* Setting this key will rename this field into 'id'
*/
idKey?: string
})
/**

7
Models/ThemeConfig/LayerConfig.ts
View file

@ -15,7 +15,6 @@ import LineRenderingConfig from "./LineRenderingConfig";
import PointRenderingConfigJson from "./Json/PointRenderingConfigJson";
import LineRenderingConfigJson from "./Json/LineRenderingConfigJson";
import {TagRenderingConfigJson} from "./Json/TagRenderingConfigJson";
import {UIEventSource} from "../../Logic/UIEventSource";
import BaseUIElement from "../../UI/BaseUIElement";
import Combine from "../../UI/Base/Combine";
import Title from "../../UI/Base/Title";
@ -112,7 +111,9 @@ export default class LayerConfig extends WithContextLoader {
geojsonSourceLevel: json.source["geoJsonZoomLevel"],
overpassScript: json.source["overpassScript"],
isOsmCache: json.source["isOsmCache"],
mercatorCrs: json.source["mercatorCrs"]
mercatorCrs: json.source["mercatorCrs"],
idKey: json.source["idKey"]
},
json.id
);
@ -236,7 +237,7 @@ export default class LayerConfig extends WithContextLoader {
console.log(json.mapRendering)
throw("The layer " + this.id + " does not have any maprenderings defined and will thus not show up on the map at all. If this is intentional, set maprenderings to 'null' instead of '[]'")
} else if (!hasCenterRendering && this.lineRendering.length === 0 && !this.source.geojsonSource?.startsWith("https://api.openstreetmap.org/api/0.6/notes.json")) {
throw "The layer " + this.id + " might not render ways. This might result in dropped information"
throw "The layer " + this.id + " might not render ways. This might result in dropped information (at "+context+")"
}
}

3
Models/ThemeConfig/SourceConfig.ts
View file

@ -9,6 +9,7 @@ export default class SourceConfig {
public geojsonZoomLevel?: number;
public isOsmCacheLayer: boolean;
public readonly mercatorCrs: boolean;
public readonly idKey : string
constructor(params: {
mercatorCrs?: boolean;
@ -17,6 +18,7 @@ export default class SourceConfig {
geojsonSource?: string,
isOsmCache?: boolean,
geojsonSourceLevel?: number,
idKey?: string
}, context?: string) {
let defined = 0;
@ -47,5 +49,6 @@ export default class SourceConfig {
this.geojsonZoomLevel = params.geojsonSourceLevel;
this.isOsmCacheLayer = params.isOsmCache ?? false;
this.mercatorCrs = params.mercatorCrs ?? false;
this.idKey= params.idKey
}
}

71
UI/Popup/AutoApplyButton.ts
View file

@ -62,36 +62,6 @@ class ApplyButton extends UIElement {
}
private async Run() {
this.buttonState.setData("running")
try {
console.log("Applying auto-action on " + this.target_feature_ids.length + " features")
for (const targetFeatureId of this.target_feature_ids) {
const featureTags = this.state.allElements.getEventSourceById(targetFeatureId)
const rendering = this.tagRenderingConfig.GetRenderValue(featureTags.data).txt
const specialRenderings = Utils.NoNull(SubstitutedTranslation.ExtractSpecialComponents(rendering)
.map(x => x.special))
.filter(v => v.func["supportsAutoAction"] === true)
if(specialRenderings.length == 0){
console.warn("AutoApply: feature "+targetFeatureId+" got a rendering without supported auto actions:", rendering)
}
for (const specialRendering of specialRenderings) {
const action = <AutoAction>specialRendering.func
await action.applyActionOn(this.state, featureTags, specialRendering.args)
}
}
console.log("Flushing changes...")
await this.state.changes.flushChanges("Auto button")
this.buttonState.setData("done")
} catch (e) {
console.error("Error while running autoApply: ", e)
this. buttonState.setData({error: e})
}
}
protected InnerRender(): string | BaseUIElement {
if (this.target_feature_ids.length === 0) {
return new FixedUiElement("No elements found to perform action")
@ -105,7 +75,13 @@ class ApplyButton extends UIElement {
const button = new SubtleButton(
new Img(this.icon),
this.text
).onClick(() => self.Run());
).onClick(() => {
this.buttonState.setData("running")
window.setTimeout(() => {
self.Run();
}, 50)
});
const explanation = new Combine(["The following objects will be updated: ",
...this.target_feature_ids.map(id => new Combine([new Link(id, "https:/ /openstreetmap.org/" + id, true), ", "]))]).SetClass("subtle")
@ -145,6 +121,37 @@ class ApplyButton extends UIElement {
))
}
private async Run() {
try {
console.log("Applying auto-action on " + this.target_feature_ids.length + " features")
for (const targetFeatureId of this.target_feature_ids) {
const featureTags = this.state.allElements.getEventSourceById(targetFeatureId)
const rendering = this.tagRenderingConfig.GetRenderValue(featureTags.data).txt
const specialRenderings = Utils.NoNull(SubstitutedTranslation.ExtractSpecialComponents(rendering)
.map(x => x.special))
.filter(v => v.func["supportsAutoAction"] === true)
if (specialRenderings.length == 0) {
console.warn("AutoApply: feature " + targetFeatureId + " got a rendering without supported auto actions:", rendering)
}
for (const specialRendering of specialRenderings) {
const action = <AutoAction>specialRendering.func
await action.applyActionOn(this.state, featureTags, specialRendering.args)
}
}
console.log("Flushing changes...")
await this.state.changes.flushChanges("Auto button")
this.buttonState.setData("done")
} catch (e) {
console.error("Error while running autoApply: ", e)
this.buttonState.setData({error: e})
}
}
}
export default class AutoApplyButton implements SpecialVisualization {
@ -224,8 +231,6 @@ export default class AutoApplyButton implements SpecialVisualization {
})
} catch (e) {
return new FixedUiElement("Could not generate a auto_apply-button for key " + argument[0] + " due to " + e).SetClass("alert")
}

6
assets/themes/grb_import/grb.json
View file

@ -369,7 +369,8 @@
]
}
]
}
},
"default": true
}
]
}
@ -441,7 +442,8 @@
"geoJson": "https://betadata.grbosm.site/grb?bbox={x_min},{y_min},{x_max},{y_max}",
"geoJsonZoomLevel": 18,
"mercatorCrs": true,
"maxCacheAge": 0
"maxCacheAge": 0,
"idKey": "osm_id"
},
"name": "GRB geometries",
"title": "GRB outline",