diff --git a/UI/BigComponents/BackgroundMapSwitch.ts b/UI/BigComponents/BackgroundMapSwitch.ts
index 5af1c305c..16fd17bb3 100644
--- a/UI/BigComponents/BackgroundMapSwitch.ts
+++ b/UI/BigComponents/BackgroundMapSwitch.ts
@@ -84,9 +84,16 @@ class SingleLayerSelectionButton extends Toggle {
             previousLayer.setData(previousLayer.data ?? available.data)
             options.currentBackground.setData(previousLayer.data)
         })
+        
+        options.currentBackground.addCallbackAndRunD(background => {
+            if (background.category === options.preferredType) {
+                previousLayer.setData(background)
+            }
+        })
 
 
-        available.addCallbackAndRunD(availableLayer => {
+        available.addCallbackD(availableLayer => {
+            // Called whenever a better layer is available
 
             if (previousLayer.data === undefined) {
                 // PreviousLayer is unset -> we definitively weren't using this category -> no need to switch
@@ -96,6 +103,11 @@ class SingleLayerSelectionButton extends Toggle {
                 // The previously used layer doesn't match the current layer -> no need to switch
                 return;
             }
+            
+            // Is the previous layer still valid? If so, we don't bother to switch
+            if(previousLayer.data.feature === null || GeoOperations.inside(locationControl.data, previousLayer.data.feature)){
+                return
+            }
 
             if (availableLayer.category === options.preferredType) {
                 // Allright, we can set this different layer
@@ -112,12 +124,6 @@ class SingleLayerSelectionButton extends Toggle {
             }
         })
 
-        options.currentBackground.addCallbackAndRunD(background => {
-            if (background.category === options.preferredType) {
-                previousLayer.setData(background)
-            }
-        })
-
         this.activate = () => {
             checkPreviousLayer()
             if (available.data.category !== options.preferredType) {