Merge master

src/Models/Constants.ts

@@ -24,6 +24,7 @@ export default class Constants {
         "range",
         "last_click",
         "favourite",
+        "summary",
     ] as const
     /**
      * Special layers which are not included in a theme by default
@@ -157,6 +158,11 @@ export default class Constants {
         "addSmall",
     ] as const
     public static readonly defaultPinIcons: string[] = <any>Constants._defaultPinIcons
+    /**
+     * The location that the MVT-layer is hosted.
+     * This is a MapLibre/MapBox vector tile server which hosts vector tiles for every (official) layer
+     */
+    public static VectorTileServer: string | undefined = Constants.config.mvt_layer_server
     public static readonly maptilerApiKey = "GvoVAJgu46I5rZapJuAy"
 
     private static isRetina(): boolean {

src/Models/ThemeConfig/Conversion/CreateNoteImportLayer.ts

@@ -92,7 +92,6 @@ export default class CreateNoteImportLayer extends Conversion<LayerConfigJson, L
                     this._includeClosedNotesDays +
                     "&bbox={x_min},{y_min},{x_max},{y_max}",
                 geoJsonZoomLevel: 10,
-                maxCacheAge: 0,
             },
             /* We need to set 'pass_all_features'
        There are probably many note_import-layers, and we don't want the first one to gobble up all notes and then discard them...

src/Models/ThemeConfig/Conversion/LegacyJsonConvert.ts

@@ -135,6 +135,10 @@ export class UpdateLegacyLayer extends DesugaringStep<
         delete config["rotation"]
         delete config["wayHandling"]
         delete config["hideUnderlayingFeaturesMinPercentage"]
+        const src = config.source
+        delete src["isOsmCache"]
+        delete src["maxCacheAge"]
+        delete src["widenFactor"]
 
         for (const mapRenderingElement of config["mapRendering"] ?? []) {
             if (mapRenderingElement["iconOverlays"] !== undefined) {
@@ -269,6 +273,7 @@ class UpdateLegacyTheme extends DesugaringStep<LayoutConfigJson> {
         oldThemeConfig.layers = Utils.NoNull(oldThemeConfig.layers)
         delete oldThemeConfig["language"]
         delete oldThemeConfig["version"]
+        delete oldThemeConfig["clustering"]
 
         if (oldThemeConfig.startLat === 0) {
             delete oldThemeConfig.startLat

src/Models/ThemeConfig/Conversion/PrepareTheme.ts

@@ -194,7 +194,6 @@ class AddDefaultLayers extends DesugaringStep<LayoutConfigJson> {
             if (v === undefined) {
                 const msg = `Default layer ${layerName} not found. ${state.sharedLayers.size} layers are available`
                 if (layerName === "favourite") {
-                    //  context.warn(msg)
                     continue
                 }
                 context.err(msg)

src/Models/ThemeConfig/Conversion/Validation.ts

@@ -283,6 +283,15 @@ export class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
             }
         }
 
+        for (let i = 0; i < theme.layers.length; i++) {
+            const layer = theme.layers[i]
+            if (!layer.id.match("[a-z][a-z0-9_]*")) {
+                context
+                    .enters("layers", i, "id")
+                    .err("Invalid ID:" + layer.id + "should match [a-z][a-z0-9_]*")
+            }
+        }
+
         return json
     }
 }
@@ -364,6 +373,34 @@ class MiscThemeChecks extends DesugaringStep<LayoutConfigJson> {
         if (json.socialImage === "") {
             context.warn("Social image for theme " + json.id + " is the emtpy string")
         }
+        if (json["clustering"]) {
+            context.warn("Obsolete field `clustering` is still around")
+        }
+        {
+            for (let i = 0; i < json.layers.length; i++) {
+                const l = json.layers[i]
+                if (l["override"]?.["source"] === undefined) {
+                    continue
+                }
+                if (l["override"]?.["source"]?.["geoJson"]) {
+                    continue // We don't care about external data as we won't cache it anyway
+                }
+                if (l["override"]["id"] !== undefined) {
+                    continue
+                }
+                context
+                    .enters("layers", i)
+                    .err("A layer which changes the source-tags must also change the ID")
+            }
+        }
+
+        if (json["overideAll"]) {
+            context
+                .enter("overideAll")
+                .err(
+                    "'overrideAll' is spelled with _two_ `r`s. You only wrote a single one of them."
+                )
+        }
         return json
     }
 }
@@ -1035,7 +1072,8 @@ export class ValidateTagRenderings extends Fuse<TagRenderingConfigJson> {
             new On("render", new ValidatePossibleLinks()),
             new On("question", new ValidatePossibleLinks()),
             new On("questionHint", new ValidatePossibleLinks()),
-            new On("mappings", new Each(new On("then", new ValidatePossibleLinks())))
+            new On("mappings", new Each(new On("then", new ValidatePossibleLinks()))),
+            new MiscTagRenderingChecks()
         )
     }
 }
@@ -1070,8 +1108,9 @@ export class PrevalidateLayer extends DesugaringStep<LayerConfigJson> {
             if (json.id?.toLowerCase() !== json.id) {
                 context.enter("id").err(`The id of a layer should be lowercase: ${json.id}`)
             }
-            if (json.id?.match(/[a-z0-9-_]/) == null) {
-                context.enter("id").err(`The id of a layer should match [a-z0-9-_]*: ${json.id}`)
+            const layerRegex = /[a-zA-Z][a-zA-Z_0-9]+/
+            if (json.id.match(layerRegex) === null) {
+                context.enter("id").err("Invalid ID. A layer ID should match " + layerRegex.source)
             }
         }
 
@@ -1572,6 +1611,22 @@ export class ValidateLayer extends Conversion<
             }
         }
 
+        if (json["doCount"]) {
+            context.enters("doCount").err("Use `isCounted` instead of `doCount`")
+        }
+
+        if (json.source) {
+            const src = json.source
+            if (src["isOsmCache"] !== undefined) {
+                context.enters("source").err("isOsmCache is deprecated")
+            }
+            if (src["maxCacheAge"] !== undefined) {
+                context
+                    .enters("source")
+                    .err("maxCacheAge is deprecated; it is " + src["maxCacheAge"])
+            }
+        }
+
         return { raw: json, parsed: layerConfig }
     }
 }
@@ -1774,3 +1829,79 @@ export class DetectDuplicatePresets extends DesugaringStep<LayoutConfig> {
         return json
     }
 }
+
+export class ValidateThemeEnsemble extends Conversion<
+    LayoutConfig[],
+    Map<
+        string,
+        {
+            tags: TagsFilter
+            foundInTheme: string[]
+        }
+    >
+> {
+    constructor() {
+        super(
+            "Validates that all themes together are logical, i.e. no duplicate ids exists within (overriden) themes",
+            [],
+            "ValidateThemeEnsemble"
+        )
+    }
+
+    convert(
+        json: LayoutConfig[],
+        context: ConversionContext
+    ): Map<
+        string,
+        {
+            tags: TagsFilter
+            foundInTheme: string[]
+        }
+    > {
+        const idToSource = new Map<string, { tags: TagsFilter; foundInTheme: string[] }>()
+
+        for (const theme of json) {
+            for (const layer of theme.layers) {
+                if (typeof layer.source === "string") {
+                    continue
+                }
+                if (Constants.priviliged_layers.indexOf(<any>layer.id) >= 0) {
+                    continue
+                }
+                if (!layer.source) {
+                    console.log(theme, layer, layer.source)
+                    context.enters(theme.id, "layers", "source", layer.id).err("No source defined")
+                    continue
+                }
+                if (layer.source.geojsonSource) {
+                    continue
+                }
+                const id = layer.id
+                const tags = layer.source.osmTags
+                if (!idToSource.has(id)) {
+                    idToSource.set(id, { tags, foundInTheme: [theme.id] })
+                    continue
+                }
+
+                const oldTags = idToSource.get(id).tags
+                const oldTheme = idToSource.get(id).foundInTheme
+                if (oldTags.shadows(tags) && tags.shadows(oldTags)) {
+                    // All is good, all is well
+                    oldTheme.push(theme.id)
+                    continue
+                }
+                context.err(
+                    [
+                        "The layer with id '" +
+                            id +
+                            "' is found in multiple themes with different tag definitions:",
+                        "\t In theme " + oldTheme + ":\t" + oldTags.asHumanString(false, false, {}),
+                        "\tIn theme " + theme.id + ":\t" + tags.asHumanString(false, false, {}),
+                    ].join("\n")
+                )
+            }
+        }
+
+        return idToSource
+    }
+}

src/Models/ThemeConfig/Json/LayerConfigJson.ts

@@ -74,15 +74,6 @@ export interface LayerConfigJson {
                * Every source must set which tags have to be present in order to load the given layer.
                */
               osmTags: TagConfigJson
-              /**
-               * question: How long (in seconds) is the data allowed to remain cached until it must be refreshed?
-               * The maximum amount of seconds that a tile is allowed to linger in the cache
-               *
-               * type: nat
-               * default: 30 days
-               * group: expert
-               */
-              maxCacheAge?: number
           }
         | {
               /**
@@ -109,17 +100,6 @@ export interface LayerConfigJson {
                * ifunset: This is not a tiled geojson
                */
               geoJsonZoomLevel?: number
-              /**
-               * Indicates that the upstream geojson data is OSM-derived.
-               * Useful for e.g. merging or for scripts generating this cache.
-               * This also indicates that making changes on this data is possible
-               *
-               * question: Is this geojson a cache of OpenStreetMap data?
-               * ifunset: This is not an OpenStreetMap cache
-               * iftrue: this is based on OpenStreetMap and can thus be edited
-               * group: expert
-               */
-              isOsmCache?: boolean
               /**
                * Some API's use a mercator-projection (EPSG:900913) instead of WGS84. Set the flag `mercatorCrs: true`  in the source for this
                *
@@ -176,6 +156,18 @@ export interface LayerConfigJson {
      */
     isShown?: TagConfigJson
 
+    /**
+     * question: should this layer be included in the summary counts?
+     *
+     * The layer server can give summary counts for a tile.
+     * This should however be disabled for some layers, e.g. because there are too many features (walls_and_buildings) or because the count is irrelevant.
+     *
+     * ifunset: Do count
+     * iffalse: Do not include the counts
+     * iftrue: Do include the count
+     */
+    isCounted?: true | boolean
+
     /**
      * The minimum needed zoomlevel required to start loading and displaying the data.
      * This can be used to only show common features (e.g. a bicycle parking) only when the map is zoomed in very much (17).

src/Models/ThemeConfig/Json/LayoutConfigJson.ts

@@ -417,14 +417,6 @@ export interface LayoutConfigJson {
      */
     overpassTimeout?: number
 
-    /**
-     * When a query is run, the data within bounds of the visible map is loaded.
-     * However, users tend to pan and zoom a lot. It is pretty annoying if every single pan means a reloading of the data.
-     * For this, the bounds are widened in order to make a small pan still within bounds of the loaded data.
-     *
-     * IF widenfactor is 1, this feature is disabled. A recommended value is between 1 and 3
-     */
-    widenFactor?: number
     /**
      * At low zoom levels, overpass is used to query features.
      * At high zoom level, the OSM api is used to fetch one or more BBOX aligning with a slippy tile.

src/Models/ThemeConfig/LayerConfig.ts

@@ -28,8 +28,6 @@ import { ImmutableStore } from "../../Logic/UIEventSource"
 import { OsmTags } from "../OsmFeature"
 import Constants from "../Constants"
 import { QuestionableTagRenderingConfigJson } from "./Json/QuestionableTagRenderingConfigJson"
-import SvelteUIElement from "../../UI/Base/SvelteUIElement"
-import Statistics from "../../assets/svg/Statistics.svelte"
 
 export default class LayerConfig extends WithContextLoader {
     public static readonly syncSelectionAllowed = ["no", "local", "theme-only", "global"] as const
@@ -46,7 +44,6 @@ export default class LayerConfig extends WithContextLoader {
     public readonly isShown: TagsFilter
     public minzoom: number
     public minzoomVisible: number
-    public readonly maxzoom: number
     public readonly title?: TagRenderingConfig
     public readonly titleIcons: TagRenderingConfig[]
     public readonly mapRendering: PointRenderingConfig[]
@@ -56,6 +53,7 @@ export default class LayerConfig extends WithContextLoader {
     public readonly allowMove: MoveConfig | null
     public readonly allowSplit: boolean
     public readonly shownByDefault: boolean
+    public readonly doCount: boolean
     /**
      * In seconds
      */
@@ -161,6 +159,7 @@ export default class LayerConfig extends WithContextLoader {
         }
         this.minzoomVisible = json.minzoomVisible ?? this.minzoom
         this.shownByDefault = json.shownByDefault ?? true
+        this.doCount = json.isCounted ?? true
         this.forceLoad = json.forceLoad ?? false
         if (json.presets === null) json.presets = undefined
         if (json.presets !== undefined && json.presets?.map === undefined) {
@@ -465,9 +464,7 @@ export default class LayerConfig extends WithContextLoader {
                     return [
                         new Combine([
                             new Link(
-                                Utils.runningFromConsole
-                                    ? "<img src='https://mapcomplete.org/assets/svg/statistics.svg' height='18px'>"
-                                    : new SvelteUIElement(Statistics, { class: "w-4 h-4 mr-2" }),
+                                "<img src='https://mapcomplete.org/assets/svg/statistics.svg' height='18px'>",
                                 "https://taginfo.openstreetmap.org/keys/" + values.key + "#values",
                                 true
                             ),

src/Models/ThemeConfig/LayoutConfig.ts

@@ -247,6 +247,14 @@ export default class LayoutConfig implements LayoutInformation {
         return this.layers.some((l) => l.isLeftRightSensitive())
     }
 
+    public hasNoteLayer() {
+        return this.layers.some((l) => l.id === "note")
+    }
+
+    public hasPresets() {
+        return this.layers.some((l) => l.presets?.length > 0)
+    }
+
     public missingTranslations(extraInspection: any): {
         untranslated: Map<string, string[]>
         total: number

src/Models/ThemeConfig/PointRenderingConfig.ts

@@ -314,7 +314,7 @@ export default class PointRenderingConfig extends WithContextLoader {
                 const label = self.label
                     ?.GetRenderValue(tags)
                     ?.Subs(tags)
-                    ?.SetClass("block center absolute text-center marker-label")
+                    ?.SetClass("flex items-center justify-center absolute marker-label")
                     ?.SetClass(cssClassesLabel)
                 if (cssLabel) {
                     label.SetStyle(cssLabel)

src/Models/ThemeViewState.ts

@@ -62,6 +62,12 @@ import FavouritesFeatureSource from "../Logic/FeatureSource/Sources/FavouritesFe
 import { ProvidedImage } from "../Logic/ImageProviders/ImageProvider"
 import { GeolocationControlState } from "../UI/BigComponents/GeolocationControl"
 import Zoomcontrol from "../UI/Zoomcontrol"
+import {
+    SummaryTileSource,
+    SummaryTileSourceRewriter,
+} from "../Logic/FeatureSource/TiledFeatureSource/SummaryTileSource"
+import summaryLayer from "../assets/generated/layers/summary.json"
+import { LayerConfigJson } from "./ThemeConfig/Json/LayerConfigJson"
 import Locale from "../UI/i18n/Locale"
 
 /**
@@ -107,6 +113,7 @@ export default class ThemeViewState implements SpecialVisualizationState {
     readonly closestFeatures: NearbyFeatureSource
     readonly newFeatures: WritableFeatureSource
     readonly layerState: LayerState
+    readonly featureSummary: SummaryTileSourceRewriter
     readonly perLayer: ReadonlyMap<string, GeoIndexedStoreForLayer>
     readonly perLayerFiltered: ReadonlyMap<string, FilteringFeatureSource>
 
@@ -139,9 +146,9 @@ export default class ThemeViewState implements SpecialVisualizationState {
      * Triggered by navigating the map with arrows or by pressing 'space' or 'enter'
      */
     public readonly visualFeedback: UIEventSource<boolean> = new UIEventSource<boolean>(false)
-    private readonly newPointDialog: FilteredLayer
+    public readonly toCacheSavers: ReadonlyMap<string, SaveFeatureSourceToLocalStorage>
 
-    constructor(layout: LayoutConfig) {
+    constructor(layout: LayoutConfig, mvtAvailableLayers: Set<string>) {
         Utils.initDomPurify()
         this.layout = layout
         this.featureSwitches = new FeatureSwitchState(layout)
@@ -224,6 +231,7 @@ export default class ThemeViewState implements SpecialVisualizationState {
                 this.mapProperties,
                 this.osmConnection.Backend(),
                 (id) => self.layerState.filteredLayers.get(id).isDisplayed,
+                mvtAvailableLayers,
                 this.fullNodeDatabase
             )
 
@@ -288,17 +296,6 @@ export default class ThemeViewState implements SpecialVisualizationState {
             )
             this.perLayer = perLayer.perLayer
         }
-        this.perLayer.forEach((fs) => {
-            new SaveFeatureSourceToLocalStorage(
-                this.osmConnection.Backend(),
-                fs.layer.layerDef.id,
-                15,
-                fs,
-                this.featureProperties,
-                fs.layer.layerDef.maxAgeOfCache
-            )
-        })
-        this.newPointDialog = this.layerState.filteredLayers.get("last_click")
 
         this.floors = this.featuresInView.features.stabilized(500).map((features) => {
             if (!features) {
@@ -332,10 +329,7 @@ export default class ThemeViewState implements SpecialVisualizationState {
             return sorted
         })
 
-        this.lastClickObject = new LastClickFeatureSource(
-            this.mapProperties.lastClickLocation,
-            this.layout
-        )
+        this.lastClickObject = new LastClickFeatureSource(this.layout)
 
         this.osmObjectDownloader = new OsmObjectDownloader(
             this.osmConnection.Backend(),
@@ -362,6 +356,8 @@ export default class ThemeViewState implements SpecialVisualizationState {
         )
         this.favourites = new FavouritesFeatureSource(this)
 
+        this.featureSummary = this.setupSummaryLayer()
+        this.toCacheSavers = this.initSaveToLocalStorage()
         this.initActors()
         this.drawSpecialLayers()
         this.initHotkeys()
@@ -387,6 +383,25 @@ export default class ThemeViewState implements SpecialVisualizationState {
         })
     }
 
+    public initSaveToLocalStorage() {
+        const toLocalStorage = new Map<string, SaveFeatureSourceToLocalStorage>()
+        this.perLayer.forEach((fs, layerId) => {
+            if (fs.layer.layerDef.source.geojsonSource !== undefined) {
+                return // We don't cache external data layers
+            }
+            console.log("Setting up a local store feature sink for", layerId)
+            const storage = new SaveFeatureSourceToLocalStorage(
+                this.osmConnection.Backend(),
+                fs.layer.layerDef.id,
+                LayoutSource.fromCacheZoomLevel,
+                fs,
+                this.featureProperties,
+                fs.layer.layerDef.maxAgeOfCache
+            )
+            toLocalStorage.set(layerId, storage)
+        })
+        return toLocalStorage
+    }
     public showNormalDataOn(map: Store<MlMap>): ReadonlyMap<string, FilteringFeatureSource> {
         const filteringFeatureSource = new Map<string, FilteringFeatureSource>()
         this.perLayer.forEach((fs, layerName) => {
@@ -458,16 +473,6 @@ export default class ThemeViewState implements SpecialVisualizationState {
 
         this.userRelatedState.markLayoutAsVisited(this.layout)
 
-        this.selectedElement.addCallbackAndRunD((feature) => {
-            // As soon as we have a selected element, we clear the selected element
-            // This is to work around maplibre, which'll _first_ register the click on the map and only _then_ on the feature
-            // The only exception is if the last element is the 'add_new'-button, as we don't want it to disappear
-            if (feature.properties.id === "last_click") {
-                return
-            }
-            this.lastClickObject.features.setData([])
-        })
-
         this.selectedElement.addCallback((selected) => {
             if (selected === undefined) {
                 Zoomcontrol.resetzoom()
@@ -495,13 +500,11 @@ export default class ThemeViewState implements SpecialVisualizationState {
                 if (!toSelect) {
                     return
                 }
-                const layer = this.layout.getMatchingLayer(toSelect.properties)
                 this.selectedElement.setData(undefined)
                 this.selectedElement.setData(toSelect)
             })
             return
         }
-        const layer = this.layout.getMatchingLayer(toSelect.properties)
         this.selectedElement.setData(undefined)
         this.selectedElement.setData(toSelect)
     }
@@ -650,6 +653,35 @@ export default class ThemeViewState implements SpecialVisualizationState {
         )
     }
 
+    private setupSummaryLayer(): SummaryTileSourceRewriter {
+        /**
+         * MaxZoom for the summary layer
+         */
+        const normalLayers = this.layout.layers.filter(
+            (l) =>
+                Constants.priviliged_layers.indexOf(<any>l.id) < 0 &&
+                !l.id.startsWith("note_import")
+        )
+        const maxzoom = Math.min(...normalLayers.map((l) => l.minzoom))
+
+        const layers = this.layout.layers.filter(
+            (l) =>
+                Constants.priviliged_layers.indexOf(<any>l.id) < 0 &&
+                l.source.geojsonSource === undefined &&
+                l.doCount
+        )
+        const url = new URL(Constants.VectorTileServer)
+        const summaryTileSource = new SummaryTileSource(
+            url.protocol + "//" + url.host + "/summary",
+            layers.map((l) => l.id),
+            this.mapProperties.zoom.map((z) => Math.max(Math.ceil(z), 0)),
+            this.mapProperties,
+            {
+                isActive: this.mapProperties.zoom.map((z) => z <= maxzoom),
+            }
+        )
+        return new SummaryTileSourceRewriter(summaryTileSource, this.layerState.filteredLayers)
+    }
     /**
      * Add the special layers to the map
      */
@@ -677,6 +709,7 @@ export default class ThemeViewState implements SpecialVisualizationState {
             ),
             current_view: this.currentView,
             favourite: this.favourites,
+            summary: this.featureSummary,
         }
 
         this.closestFeatures.registerSource(specialLayers.favourite, "favourite")
@@ -714,15 +747,15 @@ export default class ThemeViewState implements SpecialVisualizationState {
             rangeIsDisplayed?.syncWith(this.featureSwitches.featureSwitchIsTesting, true)
         }
 
-        // enumarate all 'normal' layers and match them with the appropriate 'special' layer - if applicable
+        // enumerate all 'normal' layers and match them with the appropriate 'special' layer - if applicable
         this.layerState.filteredLayers.forEach((flayer) => {
             const id = flayer.layerDef.id
             const features: FeatureSource = specialLayers[id]
             if (features === undefined) {
                 return
             }
-            if (id === "favourite") {
-                console.log("Matching special layer", id, flayer)
+            if (id === "summary") {
+                return
             }
 
             this.featureProperties.trackFeatureSource(features)
@@ -734,6 +767,13 @@ export default class ThemeViewState implements SpecialVisualizationState {
                 selectedElement: this.selectedElement,
             })
         })
+
+        new ShowDataLayer(this.map, {
+            features: specialLayers.summary,
+            layer: new LayerConfig(<LayerConfigJson>summaryLayer, "summaryLayer"),
+            // doShowLayer: this.mapProperties.zoom.map((z) => z < maxzoom),
+            selectedElement: this.selectedElement,
+        })
     }
 
     /**
@@ -742,9 +782,6 @@ export default class ThemeViewState implements SpecialVisualizationState {
     private initActors() {
         this.selectedElement.addCallback((selected) => {
             if (selected === undefined) {
-                console.trace("Unselected")
-                // We did _unselect_ an item - we always remove the lastclick-object
-                this.lastClickObject.features.setData([])
                 this.focusOnMap()
             }
         })