frame: keep overlay backgrounds with their content

fixes #3350
port 1.6

quickshell/Modals/Common/DankModalConnected.qml

@@ -37,7 +37,7 @@ Item {
 
     readonly property string resolvedConnectedBarSide: frameConnectedMode ? preferredConnectedBarSide : ""
 
-    readonly property bool frameOwnsConnectedChrome: frameConnectedMode && resolvedConnectedBarSide !== "" && !allowStacking && CompositorService.usesConnectedFrameChromeForScreen(effectiveScreen)
+    readonly property bool frameOwnsConnectedChrome: frameConnectedMode && resolvedConnectedBarSide !== "" && !allowStacking && effectiveModalLayer === WlrLayer.Top && CompositorService.usesConnectedFrameChromeForScreen(effectiveScreen)
 
     function _dockOccupiesSide(side) {
         if (!SettingsData.showDock)
@@ -83,6 +83,12 @@ Item {
     property bool keepPopoutsOpen: false
     property var customKeyboardFocus: null
     property bool useOverlayLayer: false
+    readonly property var effectiveModalLayer: root.useOverlayLayer ? WlrLayer.Overlay : LayerShell.fromEnv("DMS_MODAL_LAYER", WlrLayer.Top, {
+        "allow": ["top", "overlay"],
+        "invalidLayer": WlrLayer.Top,
+        "label": "modals",
+        "error": true
+    })
     property real frozenMotionOffsetX: 0
     property real frozenMotionOffsetY: 0
     readonly property alias contentWindow: contentWindow
@@ -140,7 +146,7 @@ Item {
             "phase": phase,
             "visible": presented,
             "presented": presented,
-            "layer": root.useOverlayLayer ? "overlay" : "top",
+            "layer": root.effectiveModalLayer === WlrLayer.Overlay ? "overlay" : "top",
             "barSide": resolvedConnectedBarSide,
             "bodyRect": bodyRect,
             "animationOffset": animationOffset,
@@ -397,12 +403,7 @@ Item {
         }
 
         WlrLayershell.namespace: root.layerNamespace
-        WlrLayershell.layer: root.useOverlayLayer ? WlrLayer.Overlay : LayerShell.fromEnv("DMS_MODAL_LAYER", WlrLayer.Top, {
-            "allow": ["top", "overlay"],
-            "invalidLayer": WlrLayer.Top,
-            "label": "modals",
-            "error": true
-        })
+        WlrLayershell.layer: root.effectiveModalLayer
         WlrLayershell.exclusiveZone: -1
         WlrLayershell.keyboardFocus: KeyboardFocus.keyboardFocus(shouldHaveFocus, customKeyboardFocus)
 

quickshell/Modals/DankLauncherV2/DankLauncherV2ModalConnected.qml

@@ -85,7 +85,7 @@ Item {
 
     readonly property string resolvedConnectedBarSide: frameConnectedMode ? preferredConnectedBarSide : ""
 
-    readonly property bool frameOwnsConnectedChrome: frameConnectedMode && resolvedConnectedBarSide !== "" && CompositorService.usesConnectedFrameChromeForScreen(effectiveScreen)
+    readonly property bool frameOwnsConnectedChrome: frameConnectedMode && resolvedConnectedBarSide !== "" && effectiveLauncherLayer === WlrLayer.Top && CompositorService.usesConnectedFrameChromeForScreen(effectiveScreen)
     readonly property bool launcherArcExtenderActive: frameOwnsConnectedChrome && SettingsData.frameLauncherArcExtender && (resolvedConnectedBarSide === "top" || resolvedConnectedBarSide === "bottom")
 
     function _dockOccupiesSide(side) {
@@ -246,7 +246,7 @@ Item {
             "phase": phase,
             "visible": presented,
             "presented": presented,
-            "layer": root.usesOverlayLayer ? "overlay" : "top",
+            "layer": root.effectiveLauncherLayer === WlrLayer.Overlay ? "overlay" : "top",
             "barSide": resolvedConnectedBarSide,
             "bodyRect": bodyRect,
             "animationOffset": animationOffset,

quickshell/Widgets/DankPopoutConnected.qml

@@ -266,7 +266,7 @@ Item {
             "phase": phase,
             "visible": visible,
             "presented": presented,
-            "layer": root.triggerUsesOverlayLayer ? "overlay" : "top",
+            "layer": root.effectivePopoutLayer === WlrLayer.Overlay ? "overlay" : "top",
             "barSide": contentContainer.connectedBarSide,
             "bodyRect": bodyRect,
             "animationOffset": animationOffset,
@@ -423,7 +423,7 @@ Item {
         }
     }
 
-    readonly property bool frameOwnsConnectedChrome: CompositorService.usesConnectedFrameChromeForScreen(root.screen)
+    readonly property bool frameOwnsConnectedChrome: effectivePopoutLayer === WlrLayer.Top && CompositorService.usesConnectedFrameChromeForScreen(root.screen)
     readonly property bool usesConnectedSurfaceChrome: Theme.isConnectedEffect
     readonly property bool usesLocalConnectedSurfaceChrome: usesConnectedSurfaceChrome && !frameOwnsConnectedChrome
     onFrameOwnsConnectedChromeChanged: _syncPopoutChromeState()