Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@ powershell -NoProfile -ExecutionPolicy Bypass -Command "& ./tests/Run-Tests.ps1

- `Run-Tests.ps1` builds a separate Debug copy into `tests/Debug`, compiles each
`tests/<Suite>.fsx` into its own STA exe through `tests/TestHost.fsproj` and runs them
serially. Native UI suites need an interactive desktop and must not run in parallel
with another UI test run. Logs are written to `tests/Debug`.
serially on a hidden desktop (`-VisibleDesktop` to watch them). Native UI suites need
a desktop session and must not run in parallel with another UI test run. Logs are
written to `tests/Debug`.
- A running `WindowTabs.exe` locks the output; exit it from the tray before building.
- New suites must be added to the `$Suites` default in `tests/Run-Tests.ps1`, start
with `TestInit.run main`, and reference the app as `#r "Debug/WindowTabs.exe"`.
Expand Down
6 changes: 4 additions & 2 deletions docs/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
## Local commands

Use Windows with .NET SDK 10 and .NET Framework 4.8. Native UI tests run
serially and require a desktop session. They must not share focus with another
UI test run.
serially and require a desktop session. They run on a hidden desktop of their own,
because WinForms keeps dropdowns and tooltips on a screen even when the suite places
their owner off-screen; `-VisibleDesktop` runs them on the current desktop instead.
They must not share focus with another UI test run.
The regression runner takes a session-wide mutex before building or running;
another regression run, including from another checkout, fails explicitly.

Expand Down
93 changes: 86 additions & 7 deletions tests/Run-Tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ param([ValidateRange(1, 3600)][int]$TimeoutSeconds = 120, [switch]$Coverage,
[ValidateRange(1, 100)][int]$Repeat = 1,
# Retained for existing callers; failures are no longer retried.
[switch]$NoRetry,
# Run the suites on the current desktop instead of a hidden one, to watch them.
[switch]$VisibleDesktop,
[ValidateSet('Reliability', 'Architecture', 'DpiLayout', 'SettingsTheme', 'SettingsEditors', 'TabShadow', 'WindowIcon', 'GroupLifecycle', 'GroupOperations', 'PopupRendering', 'TabInteraction', 'TaskbarPreview', 'PrintWindowProbe')]
[string[]]$Suites = @('Reliability', 'Architecture', 'DpiLayout', 'SettingsTheme', 'SettingsEditors', 'TabShadow', 'WindowIcon', 'GroupLifecycle', 'GroupOperations', 'PopupRendering', 'TabInteraction', 'TaskbarPreview', 'PrintWindowProbe'))
$ErrorActionPreference = 'Stop'
Expand Down Expand Up @@ -70,20 +72,96 @@ try {
dotnet altcover "--inputDirectory=$output" "--outputDirectory=$runOutput" "--report=$report" '--assemblyFilter=^(?!(WindowTabs|Win32)$)' --localSource --visibleBranches --save
if ($LASTEXITCODE -ne 0) { throw 'Coverage instrumentation failed.' }
}
# WinForms keeps dropdowns and tooltips on a screen, so suites that place their
# windows off-screen still flash popups in a corner and can take focus. A desktop
# of their own keeps them out of sight; child processes inherit it.
$testDesktop = $null
if (-not $VisibleDesktop) {
Add-Type -TypeDefinition @'
using System;
using System.ComponentModel;
using System.Runtime.InteropServices;
public sealed class DesktopProcess : IDisposable {
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
struct StartupInfo {
public int cb; public string reserved, desktop, title;
public int x, y, width, height, columns, rows, fill, flags;
public short show, reserved2; public IntPtr reserved3, stdin, stdout, stderr;
}
[StructLayout(LayoutKind.Sequential)]
struct ProcessInformation { public IntPtr process, thread; public int processId, threadId; }
[StructLayout(LayoutKind.Sequential)]
struct SecurityAttributes { public int length; public IntPtr descriptor; public bool inherit; }
[DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
public static extern IntPtr CreateDesktop(string name, IntPtr device, IntPtr mode, int flags, uint access, IntPtr attributes);
[DllImport("user32.dll")]
public static extern bool CloseDesktop(IntPtr desktop);
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
static extern bool CreateProcess(string application, string commandLine, IntPtr processAttributes, IntPtr threadAttributes,
bool inheritHandles, uint flags, IntPtr environment, string directory, ref StartupInfo startup, out ProcessInformation info);
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
static extern IntPtr CreateFile(string name, uint access, uint share, ref SecurityAttributes attributes, uint disposition, uint flags, IntPtr template);
[DllImport("kernel32.dll")]
static extern uint WaitForSingleObject(IntPtr handle, uint milliseconds);
[DllImport("kernel32.dll")]
static extern bool GetExitCodeProcess(IntPtr process, out int code);
[DllImport("kernel32.dll")]
static extern bool TerminateProcess(IntPtr process, int code);
[DllImport("kernel32.dll")]
static extern bool CloseHandle(IntPtr handle);
IntPtr process;
static IntPtr OpenLog(string path) {
var attributes = new SecurityAttributes { length = Marshal.SizeOf(typeof(SecurityAttributes)), inherit = true };
// GENERIC_WRITE, shared read/write, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL.
var handle = CreateFile(path, 0x40000000, 3, ref attributes, 2, 0x80, IntPtr.Zero);
if (handle == new IntPtr(-1)) throw new Win32Exception();
return handle;
}
public DesktopProcess(string executable, string desktop, string stdout, string stderr) {
var output = OpenLog(stdout);
var error = OpenLog(stderr);
try {
// STARTF_USESTDHANDLES; CREATE_NO_WINDOW.
var startup = new StartupInfo { cb = Marshal.SizeOf(typeof(StartupInfo)), desktop = desktop, flags = 0x100, stdout = output, stderr = error };
ProcessInformation info;
if (!CreateProcess(null, "\"" + executable + "\"", IntPtr.Zero, IntPtr.Zero, true, 0x08000000, IntPtr.Zero,
System.IO.Path.GetDirectoryName(executable), ref startup, out info))
throw new Win32Exception();
CloseHandle(info.thread);
process = info.process;
} finally { CloseHandle(output); CloseHandle(error); }
}
public bool WaitForExit(int milliseconds) { return WaitForSingleObject(process, (uint)milliseconds) == 0; }
public void Kill() { TerminateProcess(process, -1); }
public int ExitCode { get { int code; GetExitCodeProcess(process, out code); return code; } }
public void Dispose() { if (process != IntPtr.Zero) { CloseHandle(process); process = IntPtr.Zero; } }
}
'@
$testDesktopName = "WindowTabsTests-$PID"
# GENERIC_ALL: the suites create windows and hooks on it.
$testDesktop = [DesktopProcess]::CreateDesktop($testDesktopName, [IntPtr]::Zero, [IntPtr]::Zero, 0, 0x10000000, [IntPtr]::Zero)
if ($testDesktop -eq [IntPtr]::Zero) { throw 'Could not create the hidden test desktop; pass -VisibleDesktop to run on this one.' }
}
function Invoke-Test($name, $attempt) {
$logName = if ($Repeat -gt 1) { "$name.run-$iteration" } else { $name }
$stdout = Join-Path $output "$logName.attempt-$attempt.stdout.log"
$stderr = Join-Path $output "$logName.attempt-$attempt.stderr.log"
$executable = Join-Path $runOutput "$name.exe"
$process = Start-Process -FilePath $executable -WindowStyle Hidden -PassThru -RedirectStandardOutput $stdout -RedirectStandardError $stderr
# Keep the native process handle open so Windows PowerShell retains ExitCode.
$processHandle = $process.Handle
if ($testDesktop) {
$process = [DesktopProcess]::new($executable, $testDesktopName, $stdout, $stderr)
} else {
$process = Start-Process -FilePath $executable -WindowStyle Hidden -PassThru -RedirectStandardOutput $stdout -RedirectStandardError $stderr
# Keep the native process handle open so Windows PowerShell retains ExitCode.
$processHandle = $process.Handle
}
$timedOut = -not $process.WaitForExit($TimeoutSeconds * 1000)
if ($timedOut) {
$process.Kill()
}
$process.WaitForExit()
$record = [pscustomobject]@{ Suite = $name; Run = $iteration; Attempt = $attempt; ExitCode = $process.ExitCode; TimedOut = $timedOut }
$process.WaitForExit(-1) | Out-Null
$exitCode = $process.ExitCode
if ($testDesktop) { $process.Dispose() }
$record = [pscustomobject]@{ Suite = $name; Run = $iteration; Attempt = $attempt; ExitCode = $exitCode; TimedOut = $timedOut }
$record | ConvertTo-Json -Compress | Add-Content -LiteralPath (Join-Path $output 'test-results.jsonl')
if ($Coverage) {
# Flush each process separately; do not leave multiple recorder streams
Expand All @@ -94,10 +172,10 @@ try {
Get-Content -LiteralPath $stdout | Out-Host
Get-Content -LiteralPath $stderr | Out-Host
if ($timedOut) { throw "$name timed out after $TimeoutSeconds seconds." }
if ($process.ExitCode -eq 0 -and -not (Select-String -LiteralPath $stdout -Pattern '^TEST_BODY_COMPLETE$' -Quiet)) {
if ($exitCode -eq 0 -and -not (Select-String -LiteralPath $stdout -Pattern '^TEST_BODY_COMPLETE$' -Quiet)) {
throw "$name exited without completing its test body and cleanup."
}
[pscustomobject]@{ ExitCode = $process.ExitCode }
[pscustomobject]@{ ExitCode = $exitCode }
}
# Native UI tests share desktop focus and must not run in parallel.
Set-Content -LiteralPath (Join-Path $output 'test-results.jsonl') -Value ''
Expand Down Expand Up @@ -132,6 +210,7 @@ try {
}
if ($failures.Count -gt 0) { throw ($failures -join [Environment]::NewLine) }
} finally {
if ($testDesktop) { [DesktopProcess]::CloseDesktop($testDesktop) | Out-Null }
if ($taskOwnsRegressionMutex) { $taskRegressionMutex.ReleaseMutex() }
$taskRegressionMutex.Dispose()
Pop-Location
Expand Down
18 changes: 14 additions & 4 deletions tests/SettingsEditors.fsx
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,26 @@ module Native =
extern uint32 GetCurrentThreadId()
[<System.Runtime.InteropServices.DllImport("user32.dll")>]
extern bool EnumThreadWindows(uint32 threadId, WindowCallback callback, IntPtr data)
let visiblePopup() =
let tryVisiblePopup() =
let mutable result = None
let callback = WindowCallback(fun hwnd _ ->
match Control.FromHandle(hwnd) with
| :? SettingsChoicePopup as p when p.Visible -> result <- Some p
| _ -> ()
true)
Native.EnumThreadWindows(Native.GetCurrentThreadId(),callback,IntPtr.Zero) |> ignore
result.Value
result
/// A popup can need more than one pass of the message loop before it shows.
let visiblePopup what =
let clock = Diagnostics.Stopwatch.StartNew()
let mutable result = tryVisiblePopup()
while result.IsNone && clock.ElapsedMilliseconds<2000L do
Application.DoEvents()
Threading.Thread.Sleep(5)
result <- tryVisiblePopup()
match result with
| Some popup -> popup
| None -> failwith (what+" did not open")

let mutable preferences = {
geometry=Theme.defaultGeometry;legacyPalette=Theme.lightPalette
Expand Down Expand Up @@ -192,8 +203,7 @@ let main() =
for _ in 1..3 do
swatch.PerformClick()
Application.DoEvents()
let popup = visiblePopup()
assertTrue popup.Visible "Popup opened"
let popup = visiblePopup "Colour popup"
let picker = (popup.Items.[0] :?> ToolStripControlHost).Control :?> SettingsColorPicker
let other = if mode=DarkTheme then preferences.lightPalette else preferences.darkPalette
let swatches = picker.Controls |> Seq.cast<Control> |> Seq.choose(function :? Button as button -> Some button | _ -> None) |> Seq.toArray
Expand Down