More work on layerserver, generate

Docs/ServerConfig/cache/Caddyfile

@@ -0,0 +1,9 @@
+cache.mapcomplete.org {
+    reverse_proxy /mvt {
+        to http://127.0.0.1:7800
+    }
+    
+    reverse_proxy /summary {
+        to http://127.0.0.1:2345
+    }
+}

Docs/ServerConfig/cache/cache.txt

@@ -0,0 +1,15 @@
+# Cache.mapComplete.org server config
+
+The "cache"-server is hosted at nerdlab.
+
+It has a full OSM-copy on disk, and has a Postgis/Postgres database with a table for every layer for quick access. It should run the tileserver and the summaryserver
+
+## Dyndns
+
+https://dynamicdns.park-your-domain.com/update?host=cache&domain=mapcomplete.org&password=[ddns_password]
+
+## Setup
+
+See SettingUpPSQL.md
+
+

Docs/ServerConfig/hetzner/hetzner.txt

@@ -0,0 +1,5 @@
+# Hetzner
+
+This server hosts the studio files and is used for expermintal builds.
+
+For used hosts, see the Caddyfile

Docs/SettingUpPSQL.md

@@ -54,7 +54,7 @@ pg_tileserv kan hier gedownload worden: https://github.com/CrunchyData/pg_tilese
 
 ````
 export DATABASE_URL=postgresql://user:password@localhost:5444/osm-poi
-./pg_tileserv
+nohup ./pg_tileserv &
 ````
 
 Tiles are available at: 
@@ -64,3 +64,8 @@ map.addSource("drinking_water", {
 "tiles": ["http://127.0.0.2:7800/public.drinking_water/{z}/{x}/{y}.pbf"] // http://127.0.0.2:7800/public.drinking_water.json",
 })
 ````
+
+# Rebooting:
+
+-> Restart the docker container
+-> 

assets/png/twoway.png.license

@@ -1,2 +1,2 @@
 SPDX-FileCopyrightText: Pieter Vander Vennet
-SPDX-License-Identifier: CC0
+SPDX-License-Identifier: CC0-1.0

assets/png/twoway.svg.license

@@ -1,2 +1,2 @@
 SPDX-FileCopyrightText: Pieter Vander Vennet
-SPDX-License-Identifier: CC0
+SPDX-License-Identifier: CC0-1.0

assets/themes/shops/shops.json

@@ -59,4 +59,3 @@
     "minzoom": 16
   }
 }
-}

package.json

@@ -21,7 +21,7 @@
       "oauth_secret": "NBWGhWDrD3QDB35xtVuxv4aExnmIt4FA_WgeLtwxasg",
       "url": "https://www.openstreetmap.org"
     },
-    "mvt_layer_server": "http://127.0.0.1:7800/public.{type}_{layer}/{z}/{x}/{y}.pbf",
+    "mvt_layer_server": "http://cache.mapcomplete.org/mvt/public.{type}_{layer}/{z}/{x}/{y}.pbf",
     "disabled:oauth_credentials": {
       "##": "DEV",
       "#": "This client-id is registered by 'MapComplete' on https://master.apis.dev.openstreetmap.org/",

scripts/osm2pgsql/tilecountServer.ts

@@ -28,7 +28,7 @@ class OsmPoiDatabase {
     private static readonly prefixes: ReadonlyArray<string> = ["pois", "lines", "polygons"]
     private readonly _client: Client
     private isConnected = false
-    private supportedLayers: string[] = undefined
+    private supportedLayers: Set<string> = undefined
     private metaCache: PoiDatabaseMeta = undefined
     private metaCacheDate: Date = undefined
 
@@ -64,7 +64,7 @@ class OsmPoiDatabase {
         this._client.end()
     }
 
-    async getLayers(): Promise<string[]> {
+    async getLayers(): Promise<Set<string>> {
         if (this.supportedLayers !== undefined) {
             return this.supportedLayers
         }
@@ -74,8 +74,8 @@ class OsmPoiDatabase {
                 "WHERE table_schema = 'public' AND table_name LIKE 'lines_%';"
         )
         const layers = result.rows.map((r) => r.table_name.substring("lines_".length))
-        this.supportedLayers = layers
+        this.supportedLayers = new Set(layers)
-        return layers
+        return this.supportedLayers
     }
 
     async getMeta(): Promise<PoiDatabaseMeta> {
@@ -218,7 +218,12 @@ const server = new Server(2345, [
 
             let sum = 0
             let properties: Record<string, number> = {}
+            const availableLayers = await tcs.getLayers()
             for (const layer of layers.split("+")) {
+                console.log("Getting layer", layer)
+                if (!availableLayers.has(layer)) {
+                    continue
+                }
                 const count = await tcs.getCount(
                     layer,
                     Tiles.tile_bounds_lon_lat(Number(z), Number(x), Number(y))

src/Logic/FeatureSource/Sources/LayoutSource.ts

@@ -11,9 +11,7 @@ import DynamicGeoJsonTileSource from "../TiledFeatureSource/DynamicGeoJsonTileSo
 import { BBox } from "../../BBox"
 import LocalStorageFeatureSource from "../TiledFeatureSource/LocalStorageFeatureSource"
 import FullNodeDatabaseSource from "../TiledFeatureSource/FullNodeDatabaseSource"
-import { Features } from "@rgossiaux/svelte-headlessui/types"
 import DynamicMvtileSource from "../TiledFeatureSource/DynamicMvtTileSource"
-import { layouts } from "chart.js"
 
 /**
  * This source will fetch the needed data from various sources for the given layout.
@@ -21,17 +19,18 @@ import { layouts } from "chart.js"
  * Note that special layers (with `source=null` will be ignored)
  */
 export default class LayoutSource extends FeatureSourceMerger {
-    private readonly _isLoading: UIEventSource<boolean> = new UIEventSource<boolean>(false)
     /**
      * Indicates if a data source is loading something
      */
-    public readonly isLoading: Store<boolean> = this._isLoading
+    public readonly isLoading: Store<boolean>
+
     constructor(
         layers: LayerConfig[],
         featureSwitches: FeatureSwitchState,
         mapProperties: { bounds: Store<BBox>; zoom: Store<number> },
         backend: string,
         isDisplayed: (id: string) => Store<boolean>,
+        mvtAvailableLayers: Set<string>,
         fullNodeDatabaseSource?: FullNodeDatabaseSource
     ) {
         const { bounds, zoom } = mapProperties
@@ -47,18 +46,25 @@ export default class LayoutSource extends FeatureSourceMerger {
                     maxAge: l.maxAgeOfCache,
                 })
         )
-        console.log(mapProperties)
+        const mvtSources: FeatureSource[] = osmLayers
-        const mvtSources: FeatureSource[] = osmLayers.map(l => LayoutSource.setupMvtSource(l, mapProperties, isDisplayed(l.id)))
+            .filter((f) => mvtAvailableLayers.has(f.id))
+            .map((l) => LayoutSource.setupMvtSource(l, mapProperties, isDisplayed(l.id)))
+        const nonMvtSources = []
+        const nonMvtLayers = osmLayers.filter((l) => !mvtAvailableLayers.has(l.id))
 
-
+        const isLoading = new UIEventSource(false)
-        /*
+        if (nonMvtLayers.length > 0) {
+            console.log(
+                "Layers ",
+                nonMvtLayers.map((l) => l.id),
+                " cannot be fetched from the cache server, defaulting to overpass/OSM-api"
+            )
             const overpassSource = LayoutSource.setupOverpass(
                 osmLayers,
                 bounds,
                 zoom,
                 featureSwitches
-        )//*/
+            )
-/*
             const osmApiSource = LayoutSource.setupOsmApiSource(
                 osmLayers,
                 bounds,
@@ -66,28 +72,34 @@ export default class LayoutSource extends FeatureSourceMerger {
                 backend,
                 featureSwitches,
                 fullNodeDatabaseSource
-        )*/
+            )
+            nonMvtSources.push(overpassSource, osmApiSource)
+
+            function setIsLoading() {
+                const loading = overpassSource?.runningQuery?.data || osmApiSource?.isRunning?.data
+                isLoading.setData(loading)
+            }
+            overpassSource?.runningQuery?.addCallbackAndRun((_) => setIsLoading())
+            osmApiSource?.isRunning?.addCallbackAndRun((_) => setIsLoading())
+        }
 
         const geojsonSources: FeatureSource[] = geojsonlayers.map((l) =>
             LayoutSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id))
         )
 
+        super(...geojsonSources, ...fromCache, ...mvtSources, ...nonMvtSources)
 
-        super(...geojsonSources, ...fromCache, ...mvtSources)
+        this.isLoading = isLoading
-
-        const self = this
-        function setIsLoading() {
-           // const loading = overpassSource?.runningQuery?.data || osmApiSource?.isRunning?.data
-           // self._isLoading.setData(loading)
     }
 
-        // overpassSource?.runningQuery?.addCallbackAndRun((_) => setIsLoading())
+    private static setupMvtSource(
-       // osmApiSource?.isRunning?.addCallbackAndRun((_) => setIsLoading())
+        layer: LayerConfig,
-    }
+        mapProperties: { zoom: Store<number>; bounds: Store<BBox> },
-
+        isActive?: Store<boolean>
-    private static setupMvtSource(layer: LayerConfig, mapProperties:  { zoom: Store<number>; bounds: Store<BBox> },    isActive?: Store<boolean>): FeatureSource{
+    ): FeatureSource {
         return new DynamicMvtileSource(layer, mapProperties, { isActive })
     }
+
     private static setupGeojsonSource(
         layer: LayerConfig,
         mapProperties: { zoom: Store<number>; bounds: Store<BBox> },

src/index.ts

@@ -7,6 +7,7 @@ import Combine from "./UI/Base/Combine"
 import { SubtleButton } from "./UI/Base/SubtleButton"
 import { Utils } from "./Utils"
 import Download from "./assets/svg/Download.svelte"
+import Constants from "./Models/Constants"
 
 function webgl_support() {
     try {
@@ -19,29 +20,32 @@ function webgl_support() {
         return false
     }
 }
-
+async function availableLayers(): Promise<Set<string>> {
+    const status = await Utils.downloadJson(Constants.VectorTileServer + "/status.json")
+    return new Set<string>(status.layers)
+}
+async function main() {
     // @ts-ignore
     try {
         if (!webgl_support()) {
             throw "WebGL is not supported or not enabled. This is essential for MapComplete to function, please enable this."
         }
-    DetermineLayout.GetLayout()
+        const [layout, availableLayers] = await Promise.all([
-        .then((layout) => {
+            DetermineLayout.GetLayout(),
+            await availableLayers(),
+        ])
         const state = new ThemeViewState(layout)
         const main = new SvelteUIElement(ThemeViewGUI, { state })
         main.AttachTo("maindiv")
-        })
+    } catch (err) {
-        .catch((err) => {
         console.error("Error while initializing: ", err, err.stack)
         const customDefinition = DetermineLayout.getCustomDefinition()
         new Combine([
             new FixedUiElement(err).SetClass("block alert"),
 
             customDefinition?.length > 0
-                    ? new SubtleButton(
+                ? new SubtleButton(new SvelteUIElement(Download), "Download the raw file").onClick(
-                          new SvelteUIElement(Download),
+                      () =>
-                          "Download the raw file"
-                      ).onClick(() =>
                           Utils.offerContentsAsDownloadableFile(
                               DetermineLayout.getCustomDefinition(),
                               "mapcomplete-theme.json",
@@ -50,7 +54,7 @@ try {
                   )
                 : undefined,
         ]).AttachTo("maindiv")
-        })
-} catch (err) {
-    new FixedUiElement(err).SetClass("block alert").AttachTo("maindiv")
     }
+}
+
+main().then((_) => {})