Gaming: raise HDT overlay in X so its buttons get clicks

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

apps/hdt-wine-overlay/WineOverlayPlugin.cs

@@ -17,8 +17,9 @@
 // and guide tooltips came up as large black squares). Those get a 1/255 clear
 // color on their render target as soon as they appear.
 //
-// Clicks: XWayland routes them by X stacking order, which Hyprland changes
+// Clicks on overlay buttons need the overlay above the game in X stacking,
-// behind Wine's back; the poll in OnUpdate keeps the overlay raised in X.
+// which Wine never syncs for this window; the hdt-overlay-raise user service
+// in hosts/FredOS-Gaming.nix does that from outside.
 //
 // Both overlay fixes have to land before the overlay is first shown. HDT loads enabled
 // plugins (plugins.xml) before it starts its overlay update loop, so OnLoad is
@@ -49,7 +50,7 @@ namespace HdtWineOverlay
 		public string Description => "Transparent, correctly placed overlay under Wine/Proton (Wayland). Restart HDT after enabling.";
 		public string ButtonText => "";
 		public string Author => "fred";
-		public Version Version => new Version(0, 2, 0);
+		public Version Version => new Version(0, 4, 0);
 		public MenuItem MenuItem => null;
 
 		private static readonly Color AlmostTransparent = Color.FromArgb(1, 0, 0, 0);
@@ -61,13 +62,6 @@ namespace HdtWineOverlay
 		private DateTime _nextPoll = DateTime.MinValue;
 		private int _handBacks;
 
-		private static readonly IntPtr HwndTopmost = new IntPtr(-1);
-		private static readonly IntPtr HwndNoTopmost = new IntPtr(-2);
-		private const uint SwpNoSize = 0x0001;
-		private const uint SwpNoMove = 0x0002;
-		private const uint SwpNoActivate = 0x0010;
-		private const uint SwpNoOwnerZOrder = 0x0200;
-
 		public void OnLoad()
 		{
 			if(!IsWine())
@@ -136,14 +130,30 @@ namespace HdtWineOverlay
 				return;
 			_nextPoll = DateTime.UtcNow.AddMilliseconds(250);
 
+			var overlay = Core.Overlay;
+			var game = FindWindow("UnityWndClass", "Hearthstone");
+			if(!overlay.IsContentVisible || game == IntPtr.Zero)
+				return;
+
+			if(GetActiveWindow() == _hookedSource.Handle)
+			{
+				// Moving the overlay now would make Wine hand it to the compositor
+				// for good. Give activation back to the game first, a bounded
+				// number of times so this can't become a focus fight.
+				if(_handBacks < 5)
+				{
+					_handBacks++;
+					Log.Warn("Overlay is the active window, handing focus back to Hearthstone");
+					SetForegroundWindow(game);
+				}
+				return;
+			}
+			_handBacks = 0;
+
 			// HDT follows the game window through an out-of-context WinEvent hook
 			// (EVENT_OBJECT_LOCATIONCHANGE), which Wine doesn't deliver for
 			// compositor-driven moves. Poll the game's client rect instead.
-			var overlay = Core.Overlay;
+			if(!GetClientRect(game, out var rect))
-			if(!overlay.IsContentVisible)
-				return;
-			var game = FindWindow("UnityWndClass", "Hearthstone");
-			if(game == IntPtr.Zero || !GetClientRect(game, out var rect))
 				return;
 			var origin = new POINT();
 			if(!ClientToScreen(game, ref origin))
@@ -157,37 +167,6 @@ namespace HdtWineOverlay
 				_lastGameRect = rect;
 				overlay.UpdatePosition();
 			}
-
-			var hwnd = _hookedSource.Handle;
-			if(GetActiveWindow() == hwnd)
-			{
-				// A SetWindowPos now would make Wine hand the overlay to the
-				// compositor for good. Give activation back to the game instead,
-				// a bounded number of times so this can't become a focus fight.
-				if(_handBacks < 5)
-				{
-					_handBacks++;
-					Log.Warn("Overlay is the active window, handing focus back to Hearthstone");
-					SetForegroundWindow(game);
-				}
-				return;
-			}
-			_handBacks = 0;
-
-			// XWayland gives a click to the topmost X window under the pointer, in
-			// X's stacking order, not the order things are drawn in. Hyprland
-			// restacks the game above the overlay in X whenever it focuses it, so
-			// the overlay still shows on top but its buttons' clicks land in the
-			// game. Re-raise it while the game has focus. Going non-topmost first
-			// makes the second call a real z-order change Wine passes on to X
-			// (a TOPMOST call on an already topmost window is a no-op). The game's
-			// own clicks are unaffected: the overlay's input shape (HDT's
-			// WS_EX_TRANSPARENT toggling) decides those, not the stacking.
-			if(GetForegroundWindow() != game)
-				return;
-			const uint flags = SwpNoSize | SwpNoMove | SwpNoActivate | SwpNoOwnerZOrder;
-			SetWindowPos(hwnd, HwndNoTopmost, 0, 0, 0, 0, flags);
-			SetWindowPos(hwnd, HwndTopmost, 0, 0, 0, 0, flags);
 		}
 
 		private void OnToolTipOpened(object sender, RoutedEventArgs e)
@@ -207,7 +186,7 @@ namespace HdtWineOverlay
 			if(_patched.TryGetValue(source, out _))
 				return;
 			_patched.Add(source, null);
-			// Diagnostics while the black-square report is open: every window HDT opens.
+			// Every window HDT opens, to identify the odd black square if it recurs.
 			GetWindowRect(source.Handle, out var r);
 			Log.Info($"New window: {source.RootVisual?.GetType().Name} \"{(source.RootVisual as Window)?.Title}\" layered={source.UsesPerPixelOpacity} {r.Right - r.Left}x{r.Bottom - r.Top} at {r.Left},{r.Top}");
 			if(!source.UsesPerPixelOpacity)
@@ -318,15 +297,9 @@ namespace HdtWineOverlay
 		[DllImport("user32.dll")]
 		private static extern IntPtr GetActiveWindow();
 
-		[DllImport("user32.dll")]
-		private static extern IntPtr GetForegroundWindow();
-
 		[DllImport("user32.dll")]
 		private static extern bool SetForegroundWindow(IntPtr hWnd);
 
-		[DllImport("user32.dll")]
-		private static extern bool SetWindowPos(IntPtr hWnd, IntPtr hWndInsertAfter, int x, int y, int cx, int cy, uint uFlags);
-
 		[DllImport("user32.dll")]
 		private static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect);
 

hosts/FredOS-Gaming.nix

@@ -147,6 +147,63 @@
       # Not "Proton-CachyOS Latest": Faugus owns that name and, on every
       # launch, re-downloaded 330MB trying to replace this symlink with its
       # own copy. Faugus entries use this as runner "Proton-CachyOS Nix".
+      # Hearthstone Deck Tracker's overlay buttons (apps/hdt-wine-overlay for
+      # the rest of the overlay fixes). XWayland gives a click to the topmost
+      # X window under the pointer, and Hyprland raises the game above the
+      # override-redirect overlay in X whenever it focuses it, so the buttons'
+      # clicks landed in the game. Wine never passes the overlay's own
+      # restacks on to X, so raise it from outside while the game is active.
+      systemd.user.services.hdt-overlay-raise = {
+        Unit = {
+          Description = "Keep HDT's overlay above Hearthstone in X";
+          After = [ "graphical-session.target" ];
+          PartOf = [ "graphical-session.target" ];
+        };
+        Service = {
+          ExecStart = lib.getExe (pkgs.writers.writePython3Bin "hdt-overlay-raise" {
+            libraries = [ pkgs.python3Packages.xlib ];
+          } ''
+            import time
+            from Xlib import X, display
+
+            d = display.Display()
+            root = d.screen().root
+            active_atom = d.intern_atom("_NET_ACTIVE_WINDOW")
+            names = {}
+
+            while True:
+                kids = root.query_tree().children  # bottom -> top
+                ids = [w.id for w in kids]
+                for gone in set(names) - set(ids):
+                    del names[gone]
+                found = {}
+                for w in kids:
+                    if w.id not in names:
+                        try:
+                            names[w.id] = w.get_wm_name()
+                        except Exception:
+                            names[w.id] = None
+                    if names[w.id] in ("Hearthstone", "HearthstoneOverlay"):
+                        found[names[w.id]] = w
+                overlay = found.get("HearthstoneOverlay")
+                game = found.get("Hearthstone")
+                if not (overlay and game):
+                    time.sleep(1)
+                    continue
+                prop = root.get_full_property(active_atom, X.AnyPropertyType)
+                active = prop.value[0] if prop else 0
+                if active == game.id and ids.index(overlay.id) < ids.index(game.id):
+                    overlay.configure(stack_mode=X.Above)
+                    d.sync()
+                time.sleep(0.1)
+          '');
+          # X connection lost (Hyprland/XWayland restart): start over.
+          Restart = "always";
+          RestartSec = 5;
+        };
+        Install.WantedBy = [ "graphical-session.target" ];
+      };
+
       home.file.".local/share/Steam/compatibilitytools.d/Proton-CachyOS Nix".source =
         inputs.proton-cachyos-nix.packages.x86_64-linux.proton-cachyos-x86_64-v3.steamcompattool;
     };