nixos/apps/hdt-wine-overlay/WineOverlayPlugin.cs

// HDT plugin: makes the in-game overlay work under Wine/Proton on Hyprland.
//
// Wine's X11 driver decides per window whether it is "managed" (handed to the
// compositor, which tiles/floats/focuses it) or override-redirect (placed where
// the app says, stacked on top, left alone). A popup becomes managed for good
// once it is the active window during a show or SetWindowPos. WPF's Show()
// activates by default, so HDT's overlay ends up managed: Hyprland places it
// wherever it likes instead of over the game.
//
// Separately, Wine cuts every alpha-0 pixel of a layered window out of the X11
// shape, and XWayland shows those pixels as opaque black. Painting the window
// background at alpha 1/255 keeps every pixel inside the shape while staying
// invisible.
//
// The same alpha-0 problem hits every other transparent HWND HDT opens: WPF
// tooltips and popups are separate layered windows (the Battlegrounds trinket
// and guide tooltips came up as large black squares). Those get a 1/255 clear
// color on their render target as soon as they appear.
//
// Clicks on overlay buttons need the overlay above the game in X stacking,
// which Wine never syncs for this window; the hdt-overlay-raise user service
// in hosts/FredOS-Gaming.nix does that from outside.
//
// Both overlay fixes have to land before the overlay is first shown. HDT loads enabled
// plugins (plugins.xml) before it starts its overlay update loop, so OnLoad is
// early enough; enabling the plugin mid-session needs an HDT restart.
//
// Approach from alexzfe/hdt-omarchy (Utility/Wine.cs), done as a plugin so the
// official HDT build and its updater stay in place.

using System;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Controls.Primitives;
using System.Windows.Interop;
using System.Windows.Media;
using System.Windows.Threading;
using Hearthstone_Deck_Tracker;
using Hearthstone_Deck_Tracker.Plugins;
using Hearthstone_Deck_Tracker.Utility.Logging;

namespace HdtWineOverlay
{
	public class WineOverlayPlugin : IPlugin
	{
		public string Name => "Wine Overlay Fix";
		public string Description => "Transparent, correctly placed overlay under Wine/Proton (Wayland). Restart HDT after enabling.";
		public string ButtonText => "";
		public string Author => "fred";
		public Version Version => new Version(0, 4, 0);
		public MenuItem MenuItem => null;

		private static readonly Color AlmostTransparent = Color.FromArgb(1, 0, 0, 0);
		private static bool _active;
		private static bool _classHandlerRegistered;
		private readonly ConditionalWeakTable<HwndSource, object> _patched = new ConditionalWeakTable<HwndSource, object>();
		private HwndSource _hookedSource;
		private RECT _lastGameRect;
		private DateTime _nextPoll = DateTime.MinValue;
		private int _handBacks;

		public void OnLoad()
		{
			if(!IsWine())
			{
				Log.Info("Not running under Wine, nothing to do");
				return;
			}
			_active = true;

			// Tooltips: patch the moment they open so they never flash black.
			// Class handlers can't be removed, hence the _active guard.
			if(!_classHandlerRegistered)
			{
				EventManager.RegisterClassHandler(typeof(ToolTip), ToolTip.OpenedEvent, new RoutedEventHandler(OnToolTipOpened));
				_classHandlerRegistered = true;
			}
			var overlay = Core.Overlay;
			var handle = new WindowInteropHelper(overlay).Handle;
			if(handle != IntPtr.Zero && overlay.IsVisible)
				Log.Warn("Overlay was already shown before this plugin loaded; restart HDT for the fix to apply");

			// Never activate on show: keeps Wine from making the overlay a managed window.
			overlay.ShowActivated = false;

			if(!(overlay.Background is SolidColorBrush { Color.A: > 0 }))
			{
				var brush = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0));
				brush.Freeze();
				overlay.Background = brush;
			}

			if(handle != IntPtr.Zero)
				HookMouseActivate(handle);
			else
				overlay.SourceInitialized += OnOverlaySourceInitialized;

			Log.Info("Applied Wine overlay workarounds");

			if(System.IO.File.Exists(System.IO.Path.Combine(Config.AppDataPath, "wine-overlay-test")))
				ShowTestPopup();
		}

		public void OnUnload()
		{
			_active = false;
			Core.Overlay.SourceInitialized -= OnOverlaySourceInitialized;
			_hookedSource?.RemoveHook(WndProc);
			_hookedSource = null;
		}

		public void OnButtonPress()
		{
		}

		// Called by HDT every 100ms on the UI thread.
		public void OnUpdate()
		{
			if(!_active)
				return;

			// Popups, context menus and toasts that aren't tooltips.
			foreach(var source in PresentationSource.CurrentSources.OfType<HwndSource>().ToList())
				PatchTransparentSource(source);

			if(_hookedSource == null || DateTime.UtcNow < _nextPoll)
				return;
			_nextPoll = DateTime.UtcNow.AddMilliseconds(250);

			var overlay = Core.Overlay;
			var game = FindWindow("UnityWndClass", "Hearthstone");
			if(!overlay.IsContentVisible || game == IntPtr.Zero)
				return;

			if(GetActiveWindow() == _hookedSource.Handle)
			{
				// Moving the overlay now would make Wine hand it to the compositor
				// for good. Give activation back to the game first, a bounded
				// number of times so this can't become a focus fight.
				if(_handBacks < 5)
				{
					_handBacks++;
					Log.Warn("Overlay is the active window, handing focus back to Hearthstone");
					SetForegroundWindow(game);
				}
				return;
			}
			_handBacks = 0;

			// HDT follows the game window through an out-of-context WinEvent hook
			// (EVENT_OBJECT_LOCATIONCHANGE), which Wine doesn't deliver for
			// compositor-driven moves. Poll the game's client rect instead.
			if(!GetClientRect(game, out var rect))
				return;
			var origin = new POINT();
			if(!ClientToScreen(game, ref origin))
				return;
			rect.Left += origin.X;
			rect.Right += origin.X;
			rect.Top += origin.Y;
			rect.Bottom += origin.Y;
			if(!rect.Equals(_lastGameRect))
			{
				_lastGameRect = rect;
				overlay.UpdatePosition();
			}
		}

		private void OnToolTipOpened(object sender, RoutedEventArgs e)
		{
			if(_active && PresentationSource.FromVisual((Visual)sender) is HwndSource source)
				PatchTransparentSource(source);
		}

		// A layered (per-pixel alpha) window clears to transparent before WPF
		// renders into it; clearing to 1/255 instead keeps every pixel inside
		// Wine's window shape. The overlay itself is handled via Background in
		// OnLoad and skipped here.
		private void PatchTransparentSource(HwndSource source)
		{
			if(source.IsDisposed || source.CompositionTarget == null)
				return;
			if(_patched.TryGetValue(source, out _))
				return;
			_patched.Add(source, null);
			// Every window HDT opens, to identify the odd black square if it recurs.
			GetWindowRect(source.Handle, out var r);
			Log.Info($"New window: {source.RootVisual?.GetType().Name} \"{(source.RootVisual as Window)?.Title}\" layered={source.UsesPerPixelOpacity} {r.Right - r.Left}x{r.Bottom - r.Top} at {r.Left},{r.Top}");
			if(!source.UsesPerPixelOpacity)
				return;
			if(source.RootVisual == Core.Overlay)
				return;
			source.CompositionTarget.BackgroundColor = AlmostTransparent;
			Log.Info($"Patched transparent window: {source.RootVisual?.GetType().Name}");
		}

		// Remote check without hovering anything: transparent popups whose
		// edges, corners and inner ring are alpha 0. Black anywhere = not fixed.
		// Left one gets the patch, right one is left alone for comparison.
		// Shown at startup while %AppData%/HearthstoneDeckTracker/wine-overlay-test exists.
		private void ShowTestPopup()
		{
			var patched = OpenTestPopup(100, "patched");
			var control = OpenTestPopup(600, "unpatched");
			if(PresentationSource.FromVisual(control.Child) is HwndSource source)
				_patched.Add(source, null);
			var timer = new DispatcherTimer { Interval = TimeSpan.FromSeconds(30) };
			timer.Tick += (s, e) =>
			{
				patched.IsOpen = false;
				control.IsOpen = false;
				timer.Stop();
			};
			timer.Start();
			Log.Info("Opened test popups for 30s");
		}

		private static Popup OpenTestPopup(double x, string label)
		{
			var popup = new Popup
			{
				AllowsTransparency = true,
				// Owned by the overlay like HDT's own tooltips: Wine keeps owned
				// popups override-redirect, unowned ones go to the compositor.
				PlacementTarget = Core.Overlay,
				Placement = PlacementMode.Relative,
				HorizontalOffset = x,
				VerticalOffset = 100,
				Child = new Border
				{
					Width = 400,
					Height = 300,
					CornerRadius = new CornerRadius(60),
					BorderThickness = new Thickness(40),
					BorderBrush = Brushes.OrangeRed,
					Child = new TextBlock { Text = label, Foreground = Brushes.White, FontSize = 28, Margin = new Thickness(20) }
				}
			};
			popup.IsOpen = true;
			return popup;
		}

		private void OnOverlaySourceInitialized(object sender, EventArgs e)
			=> HookMouseActivate(new WindowInteropHelper(Core.Overlay).Handle);

		private void HookMouseActivate(IntPtr handle)
		{
			_hookedSource = HwndSource.FromHwnd(handle);
			_hookedSource?.AddHook(WndProc);
		}

		// Clicking an overlay button would otherwise activate the window, which
		// makes Wine turn it into a managed window on the next position update.
		private static IntPtr WndProc(IntPtr hwnd, int msg, IntPtr wParam, IntPtr lParam, ref bool handled)
		{
			const int wmMouseActivate = 0x0021;
			const int wmActivate = 0x0006;
			if(msg == wmActivate && wParam != IntPtr.Zero)
				Log.Warn("Overlay was activated; Wine may now hand it to the compositor");
			const int maNoActivate = 3;
			if(msg != wmMouseActivate)
				return IntPtr.Zero;
			handled = true;
			return new IntPtr(maNoActivate);
		}

		private static bool IsWine()
		{
			var ntdll = GetModuleHandle("ntdll.dll");
			return ntdll != IntPtr.Zero && GetProcAddress(ntdll, "wine_get_version") != IntPtr.Zero;
		}

		[StructLayout(LayoutKind.Sequential)]
		private struct RECT
		{
			public int Left, Top, Right, Bottom;
		}

		[StructLayout(LayoutKind.Sequential)]
		private struct POINT
		{
			public int X, Y;
		}

		[DllImport("kernel32.dll", CharSet = CharSet.Ansi)]
		private static extern IntPtr GetModuleHandle(string lpModuleName);

		[DllImport("kernel32.dll", CharSet = CharSet.Ansi)]
		private static extern IntPtr GetProcAddress(IntPtr hModule, string lpProcName);

		[DllImport("user32.dll", CharSet = CharSet.Unicode)]
		private static extern IntPtr FindWindow(string lpClassName, string lpWindowName);

		[DllImport("user32.dll")]
		private static extern IntPtr GetActiveWindow();

		[DllImport("user32.dll")]
		private static extern bool SetForegroundWindow(IntPtr hWnd);

		[DllImport("user32.dll")]
		private static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect);

		[DllImport("user32.dll")]
		private static extern bool GetClientRect(IntPtr hWnd, out RECT lpRect);

		[DllImport("user32.dll")]
		private static extern bool ClientToScreen(IntPtr hWnd, ref POINT lpPoint);
	}
}