From e45bfee67278bc9fa03d7fc0687e35bb64406179 Mon Sep 17 00:00:00 2001 From: leafOfTree Date: Sat, 10 Oct 2026 19:41:10 +0800 Subject: [PATCH] Run the regression suites out of sight and wait for the colour popup WinForms keeps dropdowns and tooltips on a screen even when a suite places their owner off-screen, so test runs flashed popups in the screen corners and could take focus. Run-Tests.ps1 now starts every suite on a hidden desktop of its own; -VisibleDesktop runs them on the current one. SettingsEditors pumped messages only once after clicking the colour swatch and occasionally found no popup yet, failing with a NullReferenceException. It now waits up to two seconds and reports "Colour popup did not open". --- AGENTS.md | 5 ++- docs/testing.md | 6 ++- tests/Run-Tests.ps1 | 93 ++++++++++++++++++++++++++++++++++++--- tests/SettingsEditors.fsx | 18 ++++++-- 4 files changed, 107 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6189551..7633ff5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,8 +14,9 @@ powershell -NoProfile -ExecutionPolicy Bypass -Command "& ./tests/Run-Tests.ps1 - `Run-Tests.ps1` builds a separate Debug copy into `tests/Debug`, compiles each `tests/.fsx` into its own STA exe through `tests/TestHost.fsproj` and runs them - serially. Native UI suites need an interactive desktop and must not run in parallel - with another UI test run. Logs are written to `tests/Debug`. + serially on a hidden desktop (`-VisibleDesktop` to watch them). Native UI suites need + a desktop session and must not run in parallel with another UI test run. Logs are + written to `tests/Debug`. - A running `WindowTabs.exe` locks the output; exit it from the tray before building. - New suites must be added to the `$Suites` default in `tests/Run-Tests.ps1`, start with `TestInit.run main`, and reference the app as `#r "Debug/WindowTabs.exe"`. diff --git a/docs/testing.md b/docs/testing.md index d937d19..bc12a70 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3,8 +3,10 @@ ## Local commands Use Windows with .NET SDK 10 and .NET Framework 4.8. Native UI tests run -serially and require a desktop session. They must not share focus with another -UI test run. +serially and require a desktop session. They run on a hidden desktop of their own, +because WinForms keeps dropdowns and tooltips on a screen even when the suite places +their owner off-screen; `-VisibleDesktop` runs them on the current desktop instead. +They must not share focus with another UI test run. The regression runner takes a session-wide mutex before building or running; another regression run, including from another checkout, fails explicitly. diff --git a/tests/Run-Tests.ps1 b/tests/Run-Tests.ps1 index ba88e0b..b20a062 100644 --- a/tests/Run-Tests.ps1 +++ b/tests/Run-Tests.ps1 @@ -4,6 +4,8 @@ param([ValidateRange(1, 3600)][int]$TimeoutSeconds = 120, [switch]$Coverage, [ValidateRange(1, 100)][int]$Repeat = 1, # Retained for existing callers; failures are no longer retried. [switch]$NoRetry, + # Run the suites on the current desktop instead of a hidden one, to watch them. + [switch]$VisibleDesktop, [ValidateSet('Reliability', 'Architecture', 'DpiLayout', 'SettingsTheme', 'SettingsEditors', 'TabShadow', 'WindowIcon', 'GroupLifecycle', 'GroupOperations', 'PopupRendering', 'TabInteraction', 'TaskbarPreview', 'PrintWindowProbe')] [string[]]$Suites = @('Reliability', 'Architecture', 'DpiLayout', 'SettingsTheme', 'SettingsEditors', 'TabShadow', 'WindowIcon', 'GroupLifecycle', 'GroupOperations', 'PopupRendering', 'TabInteraction', 'TaskbarPreview', 'PrintWindowProbe')) $ErrorActionPreference = 'Stop' @@ -70,20 +72,96 @@ try { dotnet altcover "--inputDirectory=$output" "--outputDirectory=$runOutput" "--report=$report" '--assemblyFilter=^(?!(WindowTabs|Win32)$)' --localSource --visibleBranches --save if ($LASTEXITCODE -ne 0) { throw 'Coverage instrumentation failed.' } } + # WinForms keeps dropdowns and tooltips on a screen, so suites that place their + # windows off-screen still flash popups in a corner and can take focus. A desktop + # of their own keeps them out of sight; child processes inherit it. + $testDesktop = $null + if (-not $VisibleDesktop) { + Add-Type -TypeDefinition @' +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; +public sealed class DesktopProcess : IDisposable { + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] + struct StartupInfo { + public int cb; public string reserved, desktop, title; + public int x, y, width, height, columns, rows, fill, flags; + public short show, reserved2; public IntPtr reserved3, stdin, stdout, stderr; + } + [StructLayout(LayoutKind.Sequential)] + struct ProcessInformation { public IntPtr process, thread; public int processId, threadId; } + [StructLayout(LayoutKind.Sequential)] + struct SecurityAttributes { public int length; public IntPtr descriptor; public bool inherit; } + [DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + public static extern IntPtr CreateDesktop(string name, IntPtr device, IntPtr mode, int flags, uint access, IntPtr attributes); + [DllImport("user32.dll")] + public static extern bool CloseDesktop(IntPtr desktop); + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + static extern bool CreateProcess(string application, string commandLine, IntPtr processAttributes, IntPtr threadAttributes, + bool inheritHandles, uint flags, IntPtr environment, string directory, ref StartupInfo startup, out ProcessInformation info); + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + static extern IntPtr CreateFile(string name, uint access, uint share, ref SecurityAttributes attributes, uint disposition, uint flags, IntPtr template); + [DllImport("kernel32.dll")] + static extern uint WaitForSingleObject(IntPtr handle, uint milliseconds); + [DllImport("kernel32.dll")] + static extern bool GetExitCodeProcess(IntPtr process, out int code); + [DllImport("kernel32.dll")] + static extern bool TerminateProcess(IntPtr process, int code); + [DllImport("kernel32.dll")] + static extern bool CloseHandle(IntPtr handle); + IntPtr process; + static IntPtr OpenLog(string path) { + var attributes = new SecurityAttributes { length = Marshal.SizeOf(typeof(SecurityAttributes)), inherit = true }; + // GENERIC_WRITE, shared read/write, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL. + var handle = CreateFile(path, 0x40000000, 3, ref attributes, 2, 0x80, IntPtr.Zero); + if (handle == new IntPtr(-1)) throw new Win32Exception(); + return handle; + } + public DesktopProcess(string executable, string desktop, string stdout, string stderr) { + var output = OpenLog(stdout); + var error = OpenLog(stderr); + try { + // STARTF_USESTDHANDLES; CREATE_NO_WINDOW. + var startup = new StartupInfo { cb = Marshal.SizeOf(typeof(StartupInfo)), desktop = desktop, flags = 0x100, stdout = output, stderr = error }; + ProcessInformation info; + if (!CreateProcess(null, "\"" + executable + "\"", IntPtr.Zero, IntPtr.Zero, true, 0x08000000, IntPtr.Zero, + System.IO.Path.GetDirectoryName(executable), ref startup, out info)) + throw new Win32Exception(); + CloseHandle(info.thread); + process = info.process; + } finally { CloseHandle(output); CloseHandle(error); } + } + public bool WaitForExit(int milliseconds) { return WaitForSingleObject(process, (uint)milliseconds) == 0; } + public void Kill() { TerminateProcess(process, -1); } + public int ExitCode { get { int code; GetExitCodeProcess(process, out code); return code; } } + public void Dispose() { if (process != IntPtr.Zero) { CloseHandle(process); process = IntPtr.Zero; } } +} +'@ + $testDesktopName = "WindowTabsTests-$PID" + # GENERIC_ALL: the suites create windows and hooks on it. + $testDesktop = [DesktopProcess]::CreateDesktop($testDesktopName, [IntPtr]::Zero, [IntPtr]::Zero, 0, 0x10000000, [IntPtr]::Zero) + if ($testDesktop -eq [IntPtr]::Zero) { throw 'Could not create the hidden test desktop; pass -VisibleDesktop to run on this one.' } + } function Invoke-Test($name, $attempt) { $logName = if ($Repeat -gt 1) { "$name.run-$iteration" } else { $name } $stdout = Join-Path $output "$logName.attempt-$attempt.stdout.log" $stderr = Join-Path $output "$logName.attempt-$attempt.stderr.log" $executable = Join-Path $runOutput "$name.exe" - $process = Start-Process -FilePath $executable -WindowStyle Hidden -PassThru -RedirectStandardOutput $stdout -RedirectStandardError $stderr - # Keep the native process handle open so Windows PowerShell retains ExitCode. - $processHandle = $process.Handle + if ($testDesktop) { + $process = [DesktopProcess]::new($executable, $testDesktopName, $stdout, $stderr) + } else { + $process = Start-Process -FilePath $executable -WindowStyle Hidden -PassThru -RedirectStandardOutput $stdout -RedirectStandardError $stderr + # Keep the native process handle open so Windows PowerShell retains ExitCode. + $processHandle = $process.Handle + } $timedOut = -not $process.WaitForExit($TimeoutSeconds * 1000) if ($timedOut) { $process.Kill() } - $process.WaitForExit() - $record = [pscustomobject]@{ Suite = $name; Run = $iteration; Attempt = $attempt; ExitCode = $process.ExitCode; TimedOut = $timedOut } + $process.WaitForExit(-1) | Out-Null + $exitCode = $process.ExitCode + if ($testDesktop) { $process.Dispose() } + $record = [pscustomobject]@{ Suite = $name; Run = $iteration; Attempt = $attempt; ExitCode = $exitCode; TimedOut = $timedOut } $record | ConvertTo-Json -Compress | Add-Content -LiteralPath (Join-Path $output 'test-results.jsonl') if ($Coverage) { # Flush each process separately; do not leave multiple recorder streams @@ -94,10 +172,10 @@ try { Get-Content -LiteralPath $stdout | Out-Host Get-Content -LiteralPath $stderr | Out-Host if ($timedOut) { throw "$name timed out after $TimeoutSeconds seconds." } - if ($process.ExitCode -eq 0 -and -not (Select-String -LiteralPath $stdout -Pattern '^TEST_BODY_COMPLETE$' -Quiet)) { + if ($exitCode -eq 0 -and -not (Select-String -LiteralPath $stdout -Pattern '^TEST_BODY_COMPLETE$' -Quiet)) { throw "$name exited without completing its test body and cleanup." } - [pscustomobject]@{ ExitCode = $process.ExitCode } + [pscustomobject]@{ ExitCode = $exitCode } } # Native UI tests share desktop focus and must not run in parallel. Set-Content -LiteralPath (Join-Path $output 'test-results.jsonl') -Value '' @@ -132,6 +210,7 @@ try { } if ($failures.Count -gt 0) { throw ($failures -join [Environment]::NewLine) } } finally { + if ($testDesktop) { [DesktopProcess]::CloseDesktop($testDesktop) | Out-Null } if ($taskOwnsRegressionMutex) { $taskRegressionMutex.ReleaseMutex() } $taskRegressionMutex.Dispose() Pop-Location diff --git a/tests/SettingsEditors.fsx b/tests/SettingsEditors.fsx index 1bfc3d8..0df7746 100644 --- a/tests/SettingsEditors.fsx +++ b/tests/SettingsEditors.fsx @@ -18,7 +18,7 @@ module Native = extern uint32 GetCurrentThreadId() [] extern bool EnumThreadWindows(uint32 threadId, WindowCallback callback, IntPtr data) -let visiblePopup() = +let tryVisiblePopup() = let mutable result = None let callback = WindowCallback(fun hwnd _ -> match Control.FromHandle(hwnd) with @@ -26,7 +26,18 @@ let visiblePopup() = | _ -> () true) Native.EnumThreadWindows(Native.GetCurrentThreadId(),callback,IntPtr.Zero) |> ignore - result.Value + result +/// A popup can need more than one pass of the message loop before it shows. +let visiblePopup what = + let clock = Diagnostics.Stopwatch.StartNew() + let mutable result = tryVisiblePopup() + while result.IsNone && clock.ElapsedMilliseconds<2000L do + Application.DoEvents() + Threading.Thread.Sleep(5) + result <- tryVisiblePopup() + match result with + | Some popup -> popup + | None -> failwith (what+" did not open") let mutable preferences = { geometry=Theme.defaultGeometry;legacyPalette=Theme.lightPalette @@ -192,8 +203,7 @@ let main() = for _ in 1..3 do swatch.PerformClick() Application.DoEvents() - let popup = visiblePopup() - assertTrue popup.Visible "Popup opened" + let popup = visiblePopup "Colour popup" let picker = (popup.Items.[0] :?> ToolStripControlHost).Control :?> SettingsColorPicker let other = if mode=DarkTheme then preferences.lightPalette else preferences.darkPalette let swatches = picker.Controls |> Seq.cast |> Seq.choose(function :? Button as button -> Some button | _ -> None) |> Seq.toArray