From f95be9f364eecaaf4e88762747f8403171359fc2 Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:18:06 -0600 Subject: [PATCH 01/19] fix(rendering): find the bundled DXC from example binaries Examples live in target//examples, next to deps/, but the DXC lookup only stepped up from deps/. Examples fell back to FXC, which took ~13s to compile the shaders on every editor open and skewed every benchmark built as an example. --- crates/rendering/src/lib.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index c1a0e2ff6bd..aa604376463 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -259,9 +259,12 @@ fn bundled_dxc_compiler() -> wgpu::Dx12Compiler { fn bundled_dxc_paths() -> Option<(PathBuf, PathBuf)> { let executable = std::env::current_exe().ok()?; let executable_dir = executable.parent()?; - let profile_dir = (executable_dir.file_name()? == "deps") - .then(|| executable_dir.parent()) - .flatten(); + let profile_dir = matches!( + executable_dir.file_name()?.to_str(), + Some("deps" | "examples") + ) + .then(|| executable_dir.parent()) + .flatten(); [Some(executable_dir), profile_dir] .into_iter() From b22788134c0db33230f4a3ac5469732f2530f8d9 Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:18:34 -0600 Subject: [PATCH 02/19] chore: add capbench, a repeatable A/B runner for media pipeline benchmarks scripts/capbench.py drives the existing measurement binaries (cap export, cap record + cap-test validate, and the cap-editor playback/decode/startup examples) against the CI performance fixture and records wall time, whole-process-tree CPU time and peak commit (Windows job object), plus per-benchmark metrics parsed from their output. Variants (builds and/or env overrides) are interleaved per repetition with alternating order so drift and thermal state hit both sides equally; `compare` reports the median change with a bootstrap 95% interval and SSIM of the exported files. Single-shot runs on the reference machine varied by ~25%, which is larger than most individual changes. The playback example gains --timer-1ms to measure under the 1ms timer resolution the editor holds while playing. --- .github/workflows/sync-tests.yml | 3 + .../examples/playback-pipeline-benchmark.rs | 7 + scripts/capbench.py | 775 ++++++++++++++++++ scripts/test-capbench.py | 140 ++++ 4 files changed, 925 insertions(+) create mode 100644 scripts/capbench.py create mode 100644 scripts/test-capbench.py diff --git a/.github/workflows/sync-tests.yml b/.github/workflows/sync-tests.yml index cb63034c85b..4b83daeb745 100644 --- a/.github/workflows/sync-tests.yml +++ b/.github/workflows/sync-tests.yml @@ -33,6 +33,8 @@ on: - "scripts/recording-reliability.py" - "scripts/recording-reliability-owned-process.cs" - "scripts/test-recording-reliability.py" + - "scripts/capbench.py" + - "scripts/test-capbench.py" - ".github/workflows/sync-tests.yml" concurrency: @@ -97,6 +99,7 @@ jobs: run: | cargo test --locked -p cap-project --lib python -B scripts/test-recording-reliability.py + python -B scripts/test-capbench.py - name: Compile recording process supervisor (Windows) if: runner.os == 'Windows' diff --git a/crates/editor/examples/playback-pipeline-benchmark.rs b/crates/editor/examples/playback-pipeline-benchmark.rs index 8e772d10372..bca3bc02fa4 100644 --- a/crates/editor/examples/playback-pipeline-benchmark.rs +++ b/crates/editor/examples/playback-pipeline-benchmark.rs @@ -849,6 +849,12 @@ async fn main() { .and_then(|s| s.parse().ok()) .unwrap_or(300); let force_ffmpeg = args.iter().any(|arg| arg == "--force-ffmpeg"); + // The editor holds a 1ms timer resolution while playing; this measures under the same condition. + let timer_1ms = args.iter().any(|arg| arg == "--timer-1ms"); + #[cfg(target_os = "windows")] + if timer_1ms { + unsafe { windows::Win32::Media::timeBeginPeriod(1) }; + } println!("{}", "=".repeat(60)); println!(" CAP PLAYBACK PIPELINE BENCHMARK"); @@ -858,6 +864,7 @@ async fn main() { println!("Target FPS: {fps}"); println!("Max frames: {frame_count}"); println!("Force FFmpeg: {force_ffmpeg}"); + println!("1ms timer: {timer_1ms}"); println!(); let (recording_meta, meta, project, recordings) = match load_recording(&recording_path).await { diff --git a/scripts/capbench.py b/scripts/capbench.py new file mode 100644 index 00000000000..96d210f813d --- /dev/null +++ b/scripts/capbench.py @@ -0,0 +1,775 @@ +#!/usr/bin/env python3 +"""Repeatable export / playback / decode / recording benchmarks for the Rust media pipeline. + +Fixture: clone CapSoftware/cap-performance-fixtures into ./performance-fixtures (as CI does). +Build the measured binaries into target/perf (release codegen; see `profile.perf` below): + + cargo build --profile perf -p cap --bin cap -p cap-test --bin cap-test -p cap-editor \\ + --example playback-pipeline-benchmark --example decode-benchmark --example editor-startup-benchmark + +`profile.perf` is not checked in; add it to .cargo/config.toml (or use --bin-dir target/release): + + [profile.perf] + inherits = "release" + codegen-units = 16 + lto = false + +Then, e.g.: + + python scripts/capbench.py run offline --out bench/base + python scripts/capbench.py run offline --variant base=bins/base --variant cand=bins/cand --out bench/ab + python scripts/capbench.py run record --screen --animate X,Y,W,H --out bench/rec + python scripts/capbench.py compare bench/base/run.json bench/cand/run.json + +On Windows each benchmark runs in a job object (whole-process-tree CPU time and peak commit) +at above-normal priority, so background load disturbs both variants less; absolute numbers are +therefore slightly optimistic. Variants are interleaved per repetition with alternating order, +each variant gets its own copy of the fixture, and comparisons report the median change with a +bootstrap 95% interval (verdicts need at least 5 runs per side). +""" +import argparse +import ctypes +import json +import os +from pathlib import Path +import platform +import random +import re +import shutil +import statistics +import subprocess +import sys +import threading +import time + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_FIXTURE = ROOT / "performance-fixtures" / "reference-recording.cap" +EXE = ".exe" if os.name == "nt" else "" + + +class Job: + def __init__(self, affinity=None): + self.handle = None + if os.name != "nt": + return + from ctypes import wintypes + + self.kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + self.kernel32.CreateJobObjectW.restype = wintypes.HANDLE + self.handle = self.kernel32.CreateJobObjectW(None, None) + if not self.handle: + raise OSError(ctypes.get_last_error(), "CreateJobObjectW failed") + limits = JobExtendedLimits() + limits.BasicLimitInformation.LimitFlags = 0x2000 | 0x20 + limits.BasicLimitInformation.PriorityClass = 0x8000 + if affinity: + limits.BasicLimitInformation.LimitFlags |= 0x10 + limits.BasicLimitInformation.Affinity = affinity + if not self.kernel32.SetInformationJobObject( + wintypes.HANDLE(self.handle), 9, ctypes.byref(limits), ctypes.sizeof(limits) + ): + raise OSError(ctypes.get_last_error(), "SetInformationJobObject failed") + + def assign(self, process): + if self.handle is None: + return + from ctypes import wintypes + + if not self.kernel32.AssignProcessToJobObject( + wintypes.HANDLE(self.handle), wintypes.HANDLE(int(process._handle)) + ): + raise OSError(ctypes.get_last_error(), "AssignProcessToJobObject failed") + + def stats(self): + if self.handle is None: + return {} + from ctypes import wintypes + + accounting = JobAccounting() + limits = JobExtendedLimits() + handle = wintypes.HANDLE(self.handle) + self.kernel32.QueryInformationJobObject( + handle, 1, ctypes.byref(accounting), ctypes.sizeof(accounting), None + ) + self.kernel32.QueryInformationJobObject( + handle, 9, ctypes.byref(limits), ctypes.sizeof(limits), None + ) + return { + "cpu_s": (accounting.TotalUserTime + accounting.TotalKernelTime) / 1e7, + "peak_commit_mb": limits.PeakJobMemoryUsed / 2**20, + "processes": accounting.TotalProcesses, + } + + def close(self): + if self.handle is not None: + from ctypes import wintypes + + self.kernel32.CloseHandle(wintypes.HANDLE(self.handle)) + self.handle = None + + +if os.name == "nt": + + class JobBasicLimits(ctypes.Structure): + _fields_ = [ + ("PerProcessUserTimeLimit", ctypes.c_int64), + ("PerJobUserTimeLimit", ctypes.c_int64), + ("LimitFlags", ctypes.c_uint32), + ("MinimumWorkingSetSize", ctypes.c_size_t), + ("MaximumWorkingSetSize", ctypes.c_size_t), + ("ActiveProcessLimit", ctypes.c_uint32), + ("Affinity", ctypes.c_size_t), + ("PriorityClass", ctypes.c_uint32), + ("SchedulingClass", ctypes.c_uint32), + ] + + class IoCounters(ctypes.Structure): + _fields_ = [(name, ctypes.c_uint64) for name in ("r", "w", "o", "rb", "wb", "ob")] + + class JobExtendedLimits(ctypes.Structure): + _fields_ = [ + ("BasicLimitInformation", JobBasicLimits), + ("IoInfo", IoCounters), + ("ProcessMemoryLimit", ctypes.c_size_t), + ("JobMemoryLimit", ctypes.c_size_t), + ("PeakProcessMemoryUsed", ctypes.c_size_t), + ("PeakJobMemoryUsed", ctypes.c_size_t), + ] + + class JobAccounting(ctypes.Structure): + _fields_ = [ + ("TotalUserTime", ctypes.c_int64), + ("TotalKernelTime", ctypes.c_int64), + ("ThisPeriodTotalUserTime", ctypes.c_int64), + ("ThisPeriodTotalKernelTime", ctypes.c_int64), + ("TotalPageFaultCount", ctypes.c_uint32), + ("TotalProcesses", ctypes.c_uint32), + ("ActiveProcesses", ctypes.c_uint32), + ("TotalTerminatedProcesses", ctypes.c_uint32), + ] + + +def run_measured(argv, timeout, env=None, cwd=None, affinity=None): + job = Job(affinity) + started = time.perf_counter() + process = subprocess.Popen( + argv, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + cwd=cwd, + text=True, + encoding="utf-8", + errors="replace", + ) + try: + job.assign(process) + lines = [] + + def pump(): + for line in process.stdout: + lines.append((time.perf_counter() - started, line.rstrip("\n"))) + + reader = threading.Thread(target=pump, daemon=True) + reader.start() + try: + code = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + code = "timeout" + wall = time.perf_counter() - started + reader.join(5) + process.stdout.close() + return {"exit": code, "wall_s": wall, **job.stats()}, lines + finally: + job.close() + + +def float_after(lines, pattern, group=1): + regex = re.compile(pattern) + for _, line in lines: + match = regex.search(line) + if match: + return float(match.group(group)) + return None + + +def section_floats(lines, start_pattern, pattern, group=1): + regex = re.compile(pattern) + inside = False + for _, line in lines: + if re.search(start_pattern, line): + inside = True + continue + if inside: + match = regex.search(line) + if match: + return float(match.group(group)) + return None + + +def ffprobe_video(path): + result = subprocess.run( + [ + "ffprobe", "-v", "error", "-count_packets", "-select_streams", "v:0", + "-show_entries", "stream=nb_read_packets,width,height:format=duration,size", + "-of", "json", str(path), + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + return {} + data = json.loads(result.stdout) + stream = (data.get("streams") or [{}])[0] + fmt = data.get("format", {}) + return { + "out_frames": float(stream.get("nb_read_packets", 0)), + "out_duration_s": float(fmt.get("duration", 0)), + "out_size_mb": float(fmt.get("size", 0)) / 2**20, + "out_width": float(stream.get("width", 0)), + "out_height": float(stream.get("height", 0)), + } + + +def parse_export(lines): + progress = [] + for at, line in lines: + if not line.startswith("{"): + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if event.get("type") == "Progress": + progress.append((at, event.get("rendered_count", 0), event.get("total_frames", 0))) + rendering = [event for event in progress if event[1] > 0] + if not rendering: + return {} + first_at, first_rendered, _ = rendering[0] + last_at, rendered, total = rendering[-1] + metrics = {"first_frame_s": first_at, "frames": float(total or rendered)} + if last_at > first_at: + metrics["steady_fps"] = (rendered - first_rendered) / (last_at - first_at) + return metrics + + +def export_bench(fps, resolution, quality, extra=()): + def argv(ctx, out_dir): + return [ + str(ctx.bin("cap")), "export", str(ctx.fixture), "-o", str(out_dir / "export.mp4"), + "--fps", str(fps), "--resolution", resolution, "--quality", quality, "--json", *extra, + ] + + def post(ctx, out_dir, metrics): + probe = ffprobe_video(out_dir / "export.mp4") + metrics.update(probe) + if metrics.get("frames") and metrics.get("wall_s"): + metrics["export_fps"] = metrics["frames"] / metrics["wall_s"] + return out_dir / "export.mp4" + + return { + "argv": argv, + "parse": parse_export, + "post": post, + "timeout": 900, + "env": {"CAP_DISABLE_ENCODER_SELF_TEST": None}, + "better": { + "wall_s": "lower", "cpu_s": "lower", "peak_commit_mb": "lower", + "export_fps": "higher", "steady_fps": "higher", "first_frame_s": "lower", + }, + } + + +def report_blocks(lines): + blocks = {} + block = stat = None + texts = [line for _, line in lines] + for index, line in enumerate(texts): + stripped = line.strip() + if index + 1 < len(texts) and set(texts[index - 1].strip()) == {"="} and set(texts[index + 1].strip()) == {"="}: + block = blocks.setdefault(stripped, {}) + stat = None + continue + if block is None: + continue + fps = re.match(r"Effective FPS:\s*([\d.]+)", stripped) + if fps: + block["fps"] = float(fps.group(1)) + elif stripped.endswith(":") and not stripped.startswith(("avg", "p50")): + stat = block.setdefault(stripped[:-1], {}) + elif stat is not None: + for key, value in re.findall(r"(avg|p50|p95|p99)=([\d.]+)ms", stripped): + stat[key] = float(value) + return blocks + + +def parse_playback_pipeline(lines): + blocks = report_blocks(lines) + decode = blocks.get("DECODE-ONLY", {}) + full = blocks.get("FULL PIPELINE - Full (1920x1080)", {}) + scrub = blocks.get("SCRUBBING (Half resolution)", {}) + metrics = { + "decode_fps": decode.get("fps"), + "pipeline_fps": full.get("fps"), + "render_p50_ms": full.get("GPU Render+Readback", {}).get("p50"), + "pipeline_p95_ms": full.get("Total (decode+render)", {}).get("p95"), + "scrub_decode_p50_ms": scrub.get("Decode", {}).get("p50"), + "scrub_p50_ms": scrub.get("Total (decode+render)", {}).get("p50"), + "scrub_p95_ms": scrub.get("Total (decode+render)", {}).get("p95"), + } + return {key: value for key, value in metrics.items() if value is not None} + + +def parse_decode(lines): + metrics = { + "decoder_create_ms": float_after(lines, r"Time to create decoder:\s*([\d.]+)ms"), + "sequential_fps": section_floats(lines, r"SEQUENTIAL DECODE PERFORMANCE", r"Effective FPS:\s*([\d.]+)"), + "random_avg_ms": float_after(lines, r"Avg access time:\s*([\d.]+)ms"), + "random_p95_ms": section_floats(lines, r"RANDOM ACCESS PERFORMANCE", r"P95:\s*([\d.]+)ms"), + } + return {key: value for key, value in metrics.items() if value is not None} + + +def parse_startup(lines): + metrics = { + "startup_p50_ms": float_after(lines, r"^Startup: .*p50=([\d.]+)ms"), + "first_preview_p50_ms": float_after(lines, r"^First preview: .*p50=([\d.]+)ms"), + } + return {key: value for key, value in metrics.items() if value is not None} + + +def example_bench(name, args, parse, better, timeout=600, env=None): + def argv(ctx, out_dir): + return [str(ctx.bin(name, example=True)), *args(ctx)] + + return {"argv": argv, "parse": parse, "timeout": timeout, "env": env or {}, "better": better} + + +def record_bench(mode, fps, seconds): + def argv(ctx, out_dir): + return [ + str(ctx.bin("cap")), "record", "--screen", ctx.screen, "--mode", mode, + "--fps", str(fps), "--duration", str(seconds), "--path", str(out_dir / "rec.cap"), + ] + + def post(ctx, out_dir, metrics): + project = out_dir / "rec.cap" + validation = subprocess.run( + [str(ctx.bin("cap-test")), "validate", str(project), "--json"], + capture_output=True, text=True, + ) + try: + video = json.loads(validation.stdout).get("video_info") or {} + except json.JSONDecodeError: + video = {} + frames = video.get("frame_count") + captured = video.get("duration_secs") + if frames and captured: + metrics["frames"] = float(frames) + metrics["captured_s"] = float(captured) + metrics["captured_ratio"] = float(captured) / seconds + metrics["delivered_fps"] = float(frames) / float(captured) + if metrics.get("cpu_s") is not None: + metrics["cpu_per_rec_s"] = metrics["cpu_s"] / seconds + metrics["overhead_s"] = metrics["wall_s"] - seconds + return None + + return { + "argv": argv, + "parse": lambda lines: {}, + "post": post, + "timeout": seconds + 120, + "env": {}, + "needs_screen": True, + "better": { + "cpu_per_rec_s": "lower", "peak_commit_mb": "lower", "delivered_fps": "higher", + "captured_ratio": "higher", "overhead_s": "lower", + }, + } + + +BENCHES = { + "export-1080p60": export_bench(60, "1920x1080", "maximum"), + "export-4k60": export_bench(60, "3840x2160", "maximum"), + "export-720p30": export_bench(30, "1280x720", "social"), + "playback-pipeline": example_bench( + "playback-pipeline-benchmark", + lambda ctx: ["--recording-path", str(ctx.fixture), "--fps", "60", "--frames", "300", "--force-ffmpeg"], + parse_playback_pipeline, + { + "cpu_s": "lower", "decode_fps": "higher", "pipeline_fps": "higher", "render_p50_ms": "lower", + "pipeline_p95_ms": "lower", "scrub_decode_p50_ms": "lower", "scrub_p50_ms": "lower", + "scrub_p95_ms": "lower", + }, + ), + "playback-pipeline-1ms": example_bench( + "playback-pipeline-benchmark", + lambda ctx: ["--recording-path", str(ctx.fixture), "--fps", "60", "--frames", "300", "--force-ffmpeg", "--timer-1ms"], + parse_playback_pipeline, + { + "cpu_s": "lower", "decode_fps": "higher", "pipeline_fps": "higher", "render_p50_ms": "lower", + "pipeline_p95_ms": "lower", "scrub_decode_p50_ms": "lower", "scrub_p50_ms": "lower", + "scrub_p95_ms": "lower", + }, + ), + "decode-display": example_bench( + "decode-benchmark", + lambda ctx: [ + "--video", str(ctx.fixture / "content" / "segments" / "segment-0" / "display.mp4"), + "--fps", "60", "--iterations", "50", + ], + parse_decode, + {"cpu_s": "lower", "sequential_fps": "higher", "random_avg_ms": "lower", "random_p95_ms": "lower", "decoder_create_ms": "lower"}, + ), + "editor-startup": example_bench( + "editor-startup-benchmark", + lambda ctx: ["--recording-path", str(ctx.fixture), "--runs", "3"], + parse_startup, + {"startup_p50_ms": "lower", "first_preview_p50_ms": "lower"}, + ), + "record-studio-60": record_bench("studio", 60, 10), + "record-instant-30": record_bench("instant", 30, 10), +} + +SUITES = { + "quick": ["export-1080p60", "playback-pipeline"], + "export": ["export-1080p60", "export-4k60", "export-720p30"], + "playback": ["playback-pipeline", "playback-pipeline-1ms", "decode-display", "editor-startup"], + "record": ["record-studio-60", "record-instant-30"], + "offline": ["export-1080p60", "export-4k60", "export-720p30", "playback-pipeline", "decode-display", "editor-startup"], +} +SUITES["all"] = SUITES["offline"] + SUITES["record"] + + +class Context: + def __init__(self, args): + self.source_fixture = Path(args.fixture).resolve() + self.fixture = self.source_fixture + self.bin_dir = Path(args.bin_dir).resolve() + self.screen = args.screen + self.animate = args.animate + self.affinity = int(args.affinity, 0) if args.affinity else None + self.extra_env = {} + + def bin(self, name, example=False): + path = self.bin_dir / ("examples" if example else "") / f"{name}{EXE}" + if not path.exists(): + raise FileNotFoundError(f"missing binary {path}; build it with --profile perf first") + return path + + +def bench_env(spec, ctx): + env = dict(os.environ) + env["PATH"] = str(ctx.bin_dir) + os.pathsep + env.get("PATH", "") + for key, value in spec.get("env", {}).items(): + if value is None: + env.pop(key, None) + else: + env[key] = value + env.update(ctx.extra_env) + return env + + +def start_animation(bounds): + x, y, width, height = bounds.split(",") + process = subprocess.Popen( + ["ffplay", "-hide_banner", "-loglevel", "quiet", "-noborder", "-alwaysontop", + "-left", x, "-top", y, "-x", width, "-y", height, + "-f", "lavfi", f"testsrc2=s={width}x{height}:r=60"], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + time.sleep(1.5) + return process + + +def run_once(name, spec, ctx, out_dir, keep_log): + out_dir.mkdir(parents=True, exist_ok=True) + argv = spec["argv"](ctx, out_dir) + animation = start_animation(ctx.animate) if spec.get("needs_screen") and ctx.animate else None + try: + measured, lines = run_measured(argv, spec["timeout"], env=bench_env(spec, ctx), cwd=out_dir, affinity=ctx.affinity) + finally: + if animation: + animation.kill() + animation.wait() + metrics = {key: value for key, value in measured.items() if key != "exit"} + metrics.update(spec["parse"](lines)) + artifact = None + if measured["exit"] == 0 and spec.get("post"): + artifact = spec["post"](ctx, out_dir, metrics) + if keep_log or measured["exit"] != 0: + (out_dir / "output.log").write_text("\n".join(line for _, line in lines), encoding="utf-8") + return measured["exit"], metrics, artifact + + +def build_info(bin_dir): + path = bin_dir / "build-info.txt" + return path.read_text(encoding="utf-8").strip() if path.exists() else None + + +def runner_git_info(): + def git(*args): + result = subprocess.run(["git", "-C", str(ROOT), *args], capture_output=True, text=True) + return result.stdout.strip() + + return {"commit": git("rev-parse", "HEAD"), "branch": git("rev-parse", "--abbrev-ref", "HEAD"), + "dirty": bool(git("status", "--porcelain", "--untracked-files=no"))} + + +def parse_variant_env(labels, items): + env = {label: {} for label in labels} + for item in items: + label, _, assignment = item.partition(":") + key, _, value = assignment.partition("=") + if label not in env or not key: + sys.exit(f"--env expects LABEL:KEY=VALUE with a known variant label, got {item!r}") + env[label][key] = value + return env + + +def cmd_run(args): + ctx = Context(args) + names = [] + for item in args.bench: + names.extend(SUITES.get(item, [item])) + unknown = [name for name in names if name not in BENCHES] + if unknown: + sys.exit(f"unknown benchmark(s): {', '.join(unknown)}. Known: {', '.join(BENCHES)}; suites: {', '.join(SUITES)}") + if any(BENCHES[name].get("needs_screen") for name in names) and not ctx.screen: + sys.exit("recording benchmarks need --screen (see `cap record screens`)") + if not ctx.source_fixture.exists(): + sys.exit(f"fixture not found: {ctx.source_fixture}") + + if any("=" not in item for item in args.variant): + sys.exit("--variant expects LABEL=BIN_DIR") + variants = [tuple(item.split("=", 1)) for item in args.variant] or [(args.label, args.bin_dir)] + if len({label for label, _ in variants}) != len(variants): + sys.exit("variant labels must be unique") + variant_env = parse_variant_env([label for label, _ in variants], args.env) + + out_root = Path(args.out).resolve() + work = out_root / "work" + for _, bin_dir in variants: + ctx.bin_dir = Path(bin_dir).resolve() + try: + for name in names: + BENCHES[name]["argv"](ctx, work) + except FileNotFoundError as error: + sys.exit(str(error)) + if work.exists(): + shutil.rmtree(work) + # Exports migrate the project config in place, so variants must not share a fixture. + fixtures = {} + for label, _ in variants: + fixtures[label] = work / "fixture" / label / ctx.source_fixture.name + shutil.copytree(ctx.source_fixture, fixtures[label], ignore=shutil.ignore_patterns(".git", ".recovery.lock")) + all_results = {} + for label, bin_dir in variants: + ctx.bin_dir = Path(bin_dir).resolve() + all_results[label] = { + "meta": { + "label": label, "runner_git": runner_git_info(), "fixture": str(ctx.source_fixture), + "bin_dir": str(ctx.bin_dir), "build_info": build_info(ctx.bin_dir), "env": variant_env[label], + "machine": platform.platform(), "cpu": platform.processor(), + "reps": args.reps, "warmup": args.warmup, "affinity": args.affinity, + "interleaved_with": [other for other, _ in variants if other != label], + "started": time.strftime("%Y-%m-%dT%H:%M:%S"), + }, + "benches": {name: {"runs": [], "failures": [], "better": BENCHES[name]["better"]} for name in names}, + } + + schedule = [("warmup", rep) for rep in range(args.warmup)] + [("rep", rep) for rep in range(args.reps)] + for phase, rep in schedule: + for name in names: + spec = BENCHES[name] + for label, bin_dir in variants if rep % 2 == 0 else variants[::-1]: + ctx.bin_dir = Path(bin_dir).resolve() + ctx.fixture = fixtures[label] + ctx.extra_env = variant_env[label] + results = all_results[label] + out_dir = work / label / name / f"{phase}{rep}" + code, metrics, artifact = run_once(name, spec, ctx, out_dir, args.keep_logs) + status = "ok" if code == 0 else f"exit={code}" + summary = ", ".join(f"{key}={value:.3f}" for key, value in sorted(metrics.items()) if key in spec["better"]) + print(f"[{phase} {rep}] {label} {name}: {status} {summary}", flush=True) + missing = sorted(key for key in spec["better"] if key not in metrics) + if code == 0 and missing: + print(f" warning: no {', '.join(missing)} in this run's output", flush=True) + if phase == "warmup": + continue + if code != 0: + results["benches"][name]["failures"].append({"rep": rep, "exit": code, "log": str(out_dir / "output.log")}) + continue + results["benches"][name]["runs"].append(metrics) + if artifact and args.keep_outputs: + kept = out_root / "artifacts" / label / f"{name}{artifact.suffix}" + kept.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(artifact, kept) + results["benches"][name]["artifact"] = str(kept) + + if not args.keep_work: + shutil.rmtree(work, ignore_errors=True) + out_root.mkdir(parents=True, exist_ok=True) + for label, results in all_results.items(): + for bench in results["benches"].values(): + bench["median"] = summarize(bench["runs"]) + path = out_root / f"{label}.json" + path.write_text(json.dumps(results, indent=2), encoding="utf-8") + print(f"\n=== {label} ===") + print_table(results) + print(f"\nresults: {path}") + if len(variants) == 2: + print() + print_comparison(all_results[variants[0][0]], all_results[variants[1][0]]) + if any(bench["failures"] for results in all_results.values() for bench in results["benches"].values()): + sys.exit(1) + + +def summarize(runs): + keys = sorted({key for run in runs for key in run}) + return {key: statistics.median([run[key] for run in runs if key in run]) for key in keys} + + +def spread(values): + if len(values) < 2: + return 0.0 + center = statistics.median(values) + return statistics.median([abs(value - center) for value in values]) / center * 100 if center else 0.0 + + +def print_table(results): + for name, bench in results["benches"].items(): + print(f"\n{name} (n={len(bench['runs'])}, failures={len(bench['failures'])})") + for key, direction in bench["better"].items(): + values = [run[key] for run in bench["runs"] if key in run] + if values: + print(f" {key:<20} {statistics.median(values):>12.3f} +/-{spread(values):4.1f}% MAD ({direction} is better)") + + +def bootstrap_ratio_ci(base, cand, rounds=4000, seed=0): + rng = random.Random(seed) + ratios = [] + for _ in range(rounds): + b = statistics.median(rng.choices(base, k=len(base))) + c = statistics.median(rng.choices(cand, k=len(cand))) + if b: + ratios.append(c / b) + ratios.sort() + if not ratios: + return None + return ratios[int(0.025 * len(ratios))], ratios[int(0.975 * len(ratios)) - 1] + + +def compare(base, cand): + rows = [] + for name, cand_bench in cand["benches"].items(): + base_bench = base["benches"].get(name) + if not base_bench: + continue + for key, direction in cand_bench["better"].items(): + b = [run[key] for run in base_bench["runs"] if key in run] + c = [run[key] for run in cand_bench["runs"] if key in run] + if not b or not c or statistics.median(b) == 0: + continue + change = statistics.median(c) / statistics.median(b) - 1 + ci = bootstrap_ratio_ci(b, c) + significant = ci is not None and (ci[0] > 1 or ci[1] < 1) + improved = (change < 0) == (direction == "lower") + verdict = "same" + if min(len(b), len(c)) < 5: + verdict = "n<5" + elif significant: + verdict = "BETTER" if improved else "WORSE" + rows.append({ + "bench": name, "metric": key, "base": statistics.median(b), "cand": statistics.median(c), + "change_pct": change * 100, + "ci_pct": None if ci is None else [(ci[0] - 1) * 100, (ci[1] - 1) * 100], + "verdict": verdict, + }) + return rows + + +def ssim(reference, distorted): + result = subprocess.run( + ["ffmpeg", "-hide_banner", "-nostats", "-i", str(distorted), "-i", str(reference), + "-lavfi", "[0:v][1:v]ssim", "-f", "null", "-"], + capture_output=True, text=True, + ) + match = re.search(r"All:([\d.]+)", result.stderr) + return float(match.group(1)) if match else None + + +def cmd_compare(args): + base = json.loads(Path(args.base).read_text(encoding="utf-8")) + cand = json.loads(Path(args.cand).read_text(encoding="utf-8")) + rows = print_comparison(base, cand) + if args.json: + Path(args.json).write_text(json.dumps(rows, indent=2), encoding="utf-8") + + +def print_comparison(base, cand): + rows = compare(base, cand) + def build(meta): + return (meta.get("build_info") or meta["bin_dir"]).splitlines()[0] + + print(f"base: {base['meta']['label']} ({build(base['meta'])})") + print(f"cand: {cand['meta']['label']} ({build(cand['meta'])})") + current = None + for row in rows: + if row["bench"] != current: + current = row["bench"] + print(f"\n{current}") + ci = row["ci_pct"] + ci_text = "" if ci is None else f"[{ci[0]:+6.1f}%, {ci[1]:+6.1f}%]" + print(f" {row['metric']:<20} {row['base']:>11.3f} -> {row['cand']:>11.3f} " + f"{row['change_pct']:+7.1f}% {ci_text:<20} {row['verdict']}") + for name, cand_bench in cand["benches"].items(): + base_artifact = base["benches"].get(name, {}).get("artifact") + cand_artifact = cand_bench.get("artifact") + if base_artifact and cand_artifact and Path(base_artifact).exists() and Path(cand_artifact).exists(): + print(f"\n{name} output SSIM vs base: {ssim(base_artifact, cand_artifact)}") + return rows + + +def main(argv=None): + parser = argparse.ArgumentParser(description="Repeatable Cap performance benchmarks (export, playback, decode, recording).") + sub = parser.add_subparsers(dest="command", required=True) + + run = sub.add_parser("run", help="run benchmarks and write results.json") + run.add_argument("bench", nargs="+", help=f"benchmarks or suites: {', '.join(list(SUITES) + list(BENCHES))}") + run.add_argument("--out", required=True, help="directory for results.json and artifacts") + run.add_argument("--label", default="run") + run.add_argument("--reps", type=int, default=5) + run.add_argument("--warmup", type=int, default=1) + run.add_argument("--fixture", default=str(DEFAULT_FIXTURE)) + run.add_argument("--bin-dir", default=str(ROOT / "target" / "perf")) + run.add_argument("--variant", action="append", default=[], metavar="LABEL=BIN_DIR", + help="benchmark several builds interleaved in one session (repeatable); overrides --label/--bin-dir") + run.add_argument("--env", action="append", default=[], metavar="LABEL:KEY=VALUE", + help="extra environment variable for one variant (repeatable), for A/B testing runtime knobs") + run.add_argument("--screen", help="screen id for recording benchmarks") + run.add_argument("--animate", metavar="X,Y,W,H", help="play a 60fps ffplay test pattern over these screen bounds during recording benchmarks") + run.add_argument("--affinity", help="CPU affinity mask for benchmark processes, e.g. 0xFFFF for the first 16 logical CPUs") + run.add_argument("--keep-outputs", action="store_true", help="keep one export per bench for SSIM comparison") + run.add_argument("--keep-logs", action="store_true") + run.add_argument("--keep-work", action="store_true") + run.set_defaults(func=cmd_run) + + cmp = sub.add_parser("compare", help="compare two results.json files") + cmp.add_argument("base") + cmp.add_argument("cand") + cmp.add_argument("--json") + cmp.set_defaults(func=cmd_compare) + + sub.add_parser("list", help="list benchmarks and suites").set_defaults( + func=lambda _: print("\n".join([*(f"suite {k}: {' '.join(v)}" for k, v in SUITES.items()), *BENCHES])) + ) + + args = parser.parse_args(argv) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-capbench.py b/scripts/test-capbench.py new file mode 100644 index 00000000000..4c450054bde --- /dev/null +++ b/scripts/test-capbench.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +import importlib.util +from pathlib import Path +import sys +import unittest + + +SPEC = importlib.util.spec_from_file_location("capbench", Path(__file__).with_name("capbench.py")) +capbench = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(capbench) + +BANNER = "=" * 60 + +PLAYBACK_OUTPUT = f""" +--- DECODE-ONLY BENCHMARK --- +{BANNER} + DECODE-ONLY +{BANNER} + Frames rendered: 300 + Effective FPS: 197.2 + Decode: + avg=5.07ms min=0.00ms max=184.49ms + p50=3.75ms p95=7.81ms p99=27.60ms +{BANNER} + FULL PIPELINE - Full (1920x1080) +{BANNER} + Effective FPS: 32.6 + Decode: + avg=6.09ms min=0.04ms max=17.48ms + p50=6.00ms p95=11.47ms p99=14.38ms + GPU Render+Readback: + avg=24.59ms min=5.36ms max=50.49ms + p50=25.11ms p95=31.70ms p99=35.18ms + Total (decode+render): + avg=30.68ms min=5.40ms max=55.99ms + p50=30.95ms p95=39.37ms p99=47.18ms +{BANNER} + FULL PIPELINE - Half (1248x702) +{BANNER} + Effective FPS: 31.9 + Total (decode+render): + avg=31.33ms min=12.97ms max=60.85ms + p50=31.03ms p95=37.79ms p99=47.13ms +{BANNER} + SCRUBBING (Half resolution) +{BANNER} + Effective FPS: 9.2 + Decode: + avg=82.37ms min=23.55ms max=846.34ms + p50=68.54ms p95=105.12ms p99=846.34ms + Total (decode+render): + avg=108.65ms min=46.38ms max=909.39ms + p50=93.93ms p95=141.57ms p99=909.39ms +""" + +DECODE_OUTPUT = """ +DECODER CREATION + Time to create decoder: 97.00ms +SEQUENTIAL DECODE PERFORMANCE + Frames decoded: 100 + Effective FPS: 189.0 +RANDOM ACCESS PERFORMANCE + Samples: 50 + Avg access time: 107.38ms + P50: 105.14ms + P95: 150.12ms +""" + + +def timed(text): + return [(index * 0.1, line) for index, line in enumerate(text.splitlines())] + + +class CapbenchTests(unittest.TestCase): + def test_playback_picks_full_and_scrub_totals(self): + metrics = capbench.parse_playback_pipeline(timed(PLAYBACK_OUTPUT)) + self.assertEqual(metrics["decode_fps"], 197.2) + self.assertEqual(metrics["pipeline_fps"], 32.6) + self.assertEqual(metrics["render_p50_ms"], 25.11) + self.assertEqual(metrics["pipeline_p95_ms"], 39.37) + self.assertEqual(metrics["scrub_decode_p50_ms"], 68.54) + self.assertEqual(metrics["scrub_p50_ms"], 93.93) + self.assertEqual(metrics["scrub_p95_ms"], 141.57) + + def test_decode_metrics(self): + metrics = capbench.parse_decode(timed(DECODE_OUTPUT)) + self.assertEqual(metrics, { + "decoder_create_ms": 97.0, "sequential_fps": 189.0, + "random_avg_ms": 107.38, "random_p95_ms": 150.12, + }) + + def test_startup_metrics(self): + lines = timed("Startup: avg=1159.3ms p50=671.2ms p95=2229.8ms\nFirst preview: avg=37.0ms p50=44.4ms p95=45.9ms") + self.assertEqual(capbench.parse_startup(lines), {"startup_p50_ms": 671.2, "first_preview_p50_ms": 44.4}) + + def test_export_progress_timing(self): + lines = [ + (0.5, "Updated project-config.json migrated settings"), + (1.0, '{"type":"Progress","rendered_count":0,"total_frames":100}'), + (2.0, '{"type":"Progress","rendered_count":50,"total_frames":100}'), + (3.0, '{"type":"Progress","rendered_count":100,"total_frames":100}'), + (3.1, '{"type":"Completed","path":"x.mp4"}'), + ] + metrics = capbench.parse_export(lines) + self.assertEqual(metrics["first_frame_s"], 2.0) + self.assertEqual(metrics["frames"], 100.0) + self.assertAlmostEqual(metrics["steady_fps"], 50.0) + + def test_compare_flags_only_separated_changes(self): + def results(values): + return {"benches": {"b": {"runs": [{"wall_s": v} for v in values], "better": {"wall_s": "lower"}}}} + + faster = capbench.compare(results([10.0, 10.1, 9.9, 10.2, 10.0]), results([8.0, 8.1, 7.9, 8.2, 8.0])) + self.assertEqual(faster[0]["verdict"], "BETTER") + slower = capbench.compare(results([10.0, 10.1, 9.9, 10.2, 10.0]), results([12.0, 12.1, 11.9, 12.2, 12.0])) + self.assertEqual(slower[0]["verdict"], "WORSE") + noise = capbench.compare(results([10.0, 12.0, 8.0, 11.0, 9.0]), results([10.5, 8.5, 11.5, 9.5, 12.5])) + self.assertEqual(noise[0]["verdict"], "same") + + def test_variant_env_is_scoped_per_label(self): + env = capbench.parse_variant_env(["base", "t4"], ["t4:CAP_EXP_DECODE_THREADS=4", "t4:X=a=b"]) + self.assertEqual(env, {"base": {}, "t4": {"CAP_EXP_DECODE_THREADS": "4", "X": "a=b"}}) + with self.assertRaises(SystemExit): + capbench.parse_variant_env(["base"], ["other:K=V"]) + + def test_job_accounts_child_process_cpu(self): + if sys.platform != "win32": + self.skipTest("job object accounting is Windows-only") + measured, lines = capbench.run_measured( + [sys.executable, "-c", "import subprocess,sys; subprocess.run([sys.executable,'-c','sum(range(3*10**7))']); print('done')"], + timeout=60, + ) + self.assertEqual(measured["exit"], 0) + self.assertGreaterEqual(measured["processes"], 2) + self.assertGreater(measured["cpu_s"], 0.1) + self.assertEqual(lines[-1][1], "done") + + +if __name__ == "__main__": + unittest.main() From 6794aabc2cf31cbcd1dd204f650a0029e450d5e3 Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:02:41 -0600 Subject: [PATCH 03/19] fix(wgpu-hal): give each DX12 fence wait its own event Backport of gfx-rs/wgpu#8273 (wgpu 27). Device::wait signalled the device's single auto-reset idler event, so when several threads waited on one device a wakeup could reach the wrong waiter and the other one slept until the 60s CLEANUP_WAIT_MS timeout while wgpu-core held the fence lock, blocking every queue.submit on that device. The desktop app shares one device between editor, preparing-preview and screenshot renderers, and readbacks are about to block on the fence, so concurrent waits become routine. --- vendor/wgpu-hal/src/dx12/device.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vendor/wgpu-hal/src/dx12/device.rs b/vendor/wgpu-hal/src/dx12/device.rs index 7de69e368f4..9580b82c52b 100644 --- a/vendor/wgpu-hal/src/dx12/device.rs +++ b/vendor/wgpu-hal/src/dx12/device.rs @@ -1992,7 +1992,11 @@ impl crate::Device for super::Device { return Ok(true); } - unsafe { fence.raw.SetEventOnCompletion(value, self.idler.event.0) } + // Backport of gfx-rs/wgpu#8273: the shared auto-reset idler event loses wakeups when + // several threads wait on the same device, stranding a waiter until its timeout. + let event = Event::create(false, false)?; + + unsafe { fence.raw.SetEventOnCompletion(value, event.0) } .into_device_result("Set event")?; let start_time = Instant::now(); @@ -2032,7 +2036,7 @@ impl crate::Device for super::Device { match unsafe { Threading::WaitForSingleObject( - self.idler.event.0, + event.0, remaining_wait_duration.as_millis().try_into().unwrap(), ) } { From 520b5987c71fbc2261a2146a7aed70063ec84793 Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:18:42 -0600 Subject: [PATCH 04/19] perf(rendering): wait on the GPU fence for readbacks instead of sleep-polling Preview and export readbacks polled map_async with yield, then sleep(100us), then sleep(1ms). Outside timeBeginPeriod every sleep rounds up to the ~15.6ms Windows timer tick, which pinned render+readback at ~24ms regardless of resolution. Block on the submission's fence in spawn_blocking (device.poll WaitForSubmissionIndex) so the wait ends when the GPU signals. capbench, 5 interleaved reps (Ryzen 9 9950X3D, RTX 4070 Ti, fixture 3024x1964 + camera): playback pipeline 31.3 -> 92.2 fps, render p50 24.2 -> 4.1 ms, p95 46.0 -> 17.0 ms scrub p50/p95 79.6/123.5 -> 63.5/90.2 ms editor first frame 38.7 -> 13.9 ms exports unchanged --- crates/rendering/src/frame_pipeline.rs | 149 +++++++++++-------------- 1 file changed, 67 insertions(+), 82 deletions(-) diff --git a/crates/rendering/src/frame_pipeline.rs b/crates/rendering/src/frame_pipeline.rs index 9384333838e..f9ab6928c6b 100644 --- a/crates/rendering/src/frame_pipeline.rs +++ b/crates/rendering/src/frame_pipeline.rs @@ -14,7 +14,6 @@ use cidre::{arc, cf, cv, io, mtl}; const GPU_BUFFER_WAIT_TIMEOUT_SECS: u64 = 10; const SOFTWARE_GPU_BUFFER_WAIT_TIMEOUT_SECS: u64 = 60; -const NV12_READBACK_SPIN: std::time::Duration = std::time::Duration::from_millis(8); /// Whether this process has selected a software (CPU rasterizer) wgpu adapter. /// Software adapters like Windows WARP legitimately take tens of seconds for the @@ -36,6 +35,43 @@ fn gpu_buffer_wait_timeout() -> std::time::Duration { std::time::Duration::from_secs(secs) } +enum MapWait { + Mapped, + Failed(wgpu::BufferAsyncError), + Abandoned, +} + +// Blocking on the submission's fence wakes the moment the GPU signals it. Sleep-polling +// instead rounds every wait up to the OS timer tick (~15.6ms on Windows outside +// timeBeginPeriod), which capped preview renders at ~25ms regardless of resolution. +async fn wait_for_map( + device: &wgpu::Device, + submission: &wgpu::SubmissionIndex, + rx: &mut oneshot::Receiver>, +) -> Result { + let device = device.clone(); + let submission = submission.clone(); + let waited = tokio::time::timeout( + gpu_buffer_wait_timeout(), + tokio::task::spawn_blocking(move || { + device.poll(wgpu::PollType::WaitForSubmissionIndex(submission)) + }), + ) + .await; + match waited { + Ok(Ok(result)) => { + result?; + } + _ => return Ok(MapWait::Abandoned), + } + // The wait fires the map callbacks of every submission up to this one. + Ok(match rx.try_recv() { + Ok(Ok(())) => MapWait::Mapped, + Ok(Err(error)) => MapWait::Failed(error), + Err(_) => MapWait::Abandoned, + }) +} + #[derive(Clone, Copy, Debug, Default)] pub struct FinishEncoderTimings { pub wait_previous_duration: std::time::Duration, @@ -810,7 +846,7 @@ impl RgbaToNv12Converter { encoder.copy_buffer_to_buffer(nv12_buffer, 0, &readback_buffer, 0, nv12_size); self.pending = Some(PendingNv12Output::Readback(PendingNv12Readback { - rx: None, + armed: None, mapped: false, buffer: readback_buffer, width, @@ -826,7 +862,7 @@ impl RgbaToNv12Converter { /// Arms the pending output after its command buffer has been submitted: /// readback mode starts the buffer map, surface mode registers a /// submitted-work-done completion flag. - pub fn after_submit(&mut self, queue: &wgpu::Queue) { + pub fn after_submit(&mut self, queue: &wgpu::Queue, submission: &wgpu::SubmissionIndex) { match self.pending.as_mut() { Some(PendingNv12Output::Readback(pending)) => { let (tx, rx) = oneshot::channel(); @@ -836,7 +872,7 @@ impl RgbaToNv12Converter { .map_async(wgpu::MapMode::Read, move |result| { let _ = tx.send(result); }); - pending.rx = Some(rx); + pending.armed = Some((rx, submission.clone())); pending.mapped = true; } #[cfg(target_os = "macos")] @@ -1014,7 +1050,10 @@ impl PendingNv12Surface { } pub struct PendingNv12Readback { - rx: Option>>, + armed: Option<( + oneshot::Receiver>, + wgpu::SubmissionIndex, + )>, mapped: bool, buffer: Arc, pub width: u32, @@ -1042,41 +1081,17 @@ impl PendingNv12Readback { device: &wgpu::Device, buffer_pool: Option<&mut NV12BufferPool>, ) -> Result { - let Some(mut rx) = self.rx.take() else { + let Some((mut rx, submission)) = self.armed.take() else { return Err(self.cancel()); }; - let start_time = Instant::now(); - let timeout_duration = gpu_buffer_wait_timeout(); - - loop { - if start_time.elapsed() > timeout_duration { - return Err(self.cancel()); - } - - match rx.try_recv() { - Ok(result) => match result { - Ok(()) => break, - Err(error) => { - self.unmap(); - return Err(RenderingError::BufferMapFailed(error)); - } - }, - Err(oneshot::error::TryRecvError::Empty) => { - device.poll(wgpu::PollType::Poll)?; - // tokio's timer ticks in whole milliseconds, so even a - // 100µs sleep outlasts a frame's GPU work and caps the - // export at the timer rate; sleep only once a frame is slow. - if start_time.elapsed() < NV12_READBACK_SPIN { - tokio::task::yield_now().await; - } else { - tokio::time::sleep(std::time::Duration::from_millis(1)).await; - } - } - Err(oneshot::error::TryRecvError::Closed) => { - return Err(self.cancel()); - } + match wait_for_map(device, &submission, &mut rx).await? { + MapWait::Mapped => {} + MapWait::Failed(error) => { + self.unmap(); + return Err(RenderingError::BufferMapFailed(error)); } + MapWait::Abandoned => return Err(self.cancel()), } let buffer_slice = self.buffer.slice(..); @@ -1559,6 +1574,7 @@ impl PendingSurface { pub struct PendingReadback { readiness_phase: Option, rx: oneshot::Receiver>, + submission: wgpu::SubmissionIndex, mapped: bool, buffer: Arc, padded_bytes_per_row: u32, @@ -1592,53 +1608,21 @@ impl PendingReadback { } pub async fn wait(mut self, device: &wgpu::Device) -> Result { - let mut poll_count = 0u32; let start_time = Instant::now(); - let timeout_duration = gpu_buffer_wait_timeout(); - - loop { - if start_time.elapsed() > timeout_duration { + match wait_for_map(device, &self.submission, &mut self.rx).await? { + MapWait::Mapped => {} + MapWait::Failed(error) => { + self.unmap(); + return Err(RenderingError::BufferMapFailed(error)); + } + MapWait::Abandoned => { tracing::error!( frame_number = self.frame_number, - elapsed_secs = start_time.elapsed().as_secs(), - poll_count = poll_count, - "GPU buffer mapping timed out after {}s", - timeout_duration.as_secs() + elapsed_ms = start_time.elapsed().as_millis() as u64, + "GPU buffer mapping timed out or was abandoned" ); return Err(self.cancel()); } - - match self.rx.try_recv() { - Ok(result) => match result { - Ok(()) => break, - Err(error) => { - self.unmap(); - return Err(RenderingError::BufferMapFailed(error)); - } - }, - Err(oneshot::error::TryRecvError::Empty) => { - device.poll(wgpu::PollType::Poll)?; - poll_count += 1; - if poll_count < 10 { - tokio::task::yield_now().await; - } else if poll_count < 100 { - tokio::time::sleep(std::time::Duration::from_micros(100)).await; - } else { - tokio::time::sleep(std::time::Duration::from_millis(1)).await; - } - if poll_count.is_multiple_of(10000) { - tracing::warn!( - frame_number = self.frame_number, - poll_count = poll_count, - elapsed_ms = start_time.elapsed().as_millis() as u64, - "GPU buffer mapping taking longer than expected" - ); - } - } - Err(oneshot::error::TryRecvError::Closed) => { - return Err(self.cancel()); - } - } } if let Some(phase) = &self.readiness_phase { @@ -1824,7 +1808,7 @@ impl PipelinedGpuReadback { if let Some(phase) = &phase { phase.mark("submit_start"); } - queue.submit(std::iter::once(render_encoder.finish())); + let submission = queue.submit(std::iter::once(render_encoder.finish())); if let Some(phase) = &phase { phase.mark("submit_returned"); } @@ -1842,6 +1826,7 @@ impl PipelinedGpuReadback { readiness_phase: phase, mapped: true, rx, + submission, buffer, padded_bytes_per_row, width: uniforms.output_size.0, @@ -2171,9 +2156,9 @@ pub async fn finish_encoder_nv12_pooled( })); } - queue.submit(std::iter::once(encoder.finish())); + let submission = queue.submit(std::iter::once(encoder.finish())); if submitted { - nv12_converter.after_submit(queue); + nv12_converter.after_submit(queue, &submission); } if let Some(prev) = previous { @@ -2374,8 +2359,8 @@ mod surface_output_tests { .await .expect("conversion submission") ); - queue.submit(std::iter::once(encoder.finish())); - converter.after_submit(queue); + let submission = queue.submit(std::iter::once(encoder.finish())); + converter.after_submit(queue, &submission); converter .take_pending() .expect("conversion pending") From e0f0ec26d3721706c6d47442ada71d147661b7c2 Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:18:58 -0600 Subject: [PATCH 05/19] perf(rendering): serve Media Foundation frame requests during readahead After answering a request the MF decoder reads ahead up to 8-30 frames before it looks at its channel again. A sequential reader (export, playback prefetch) therefore waited out the rest of the batch for its next frame, then found the decoder idle while it consumed the cached ones, so decode and render alternated instead of overlapping. Poll for requests between samples, answer cache hits immediately and slide the readahead window with them; misses are handed back to the normal request path. Answering from the cache mid-batch exposed a timing dependence that already existed: the decoders number samples at the recorded fps (59/57/30 on the fixture), so a VFR stream regularly has two samples rounding to the same frame and the later one replaces the first in the cache. Whether a request got the first or the second depended on whether it arrived before the second was decoded, so repeated exports differed by a source frame at a few dozen positions. A frame is now answered only once the decoder has moved past it, which always yields the last sample for that frame; repeated exports are byte-identical. capbench, interleaved, same build with the window slide toggled (n=4): export 1080p60 9.79 -> 8.45 s wall (-14%), 15.7 -> 18.0 s CPU export 720p30 8.11 -> 7.72 s wall (-5%), 13.7 -> 15.2 s CPU and against the previous build (n=5): MF random access 87.7 -> 69.1 ms avg, 119 -> 97 ms p95. The extra CPU is mostly NVIDIA driver threads now running concurrently with rendering; 4K exports are NVENC-bound and unchanged. --- .../rendering/src/decoder/media_foundation.rs | 88 +++++++++++++------ 1 file changed, 60 insertions(+), 28 deletions(-) diff --git a/crates/rendering/src/decoder/media_foundation.rs b/crates/rendering/src/decoder/media_foundation.rs index 1fb01862cd5..fa140793173 100644 --- a/crates/rendering/src/decoder/media_foundation.rs +++ b/crates/rendering/src/decoder/media_foundation.rs @@ -248,7 +248,8 @@ impl MFDecoder { sender: oneshot::Sender, } - while let Ok(r) = rx.recv() { + let mut carried: Option = None; + while let Some(r) = carried.take().or_else(|| rx.recv().ok()) { let mut pending_requests: Vec = Vec::with_capacity(8); let mut push_request = @@ -312,8 +313,8 @@ impl MFDecoder { ); } - let cache_min = requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); - let cache_max = requested_frame + FRAME_CACHE_SIZE as u32 / 2; + let mut cache_min = requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); + let mut cache_max = requested_frame + FRAME_CACHE_SIZE as u32 / 2; let needs_seek = last_decoded_frame .map(|last| { @@ -352,7 +353,8 @@ impl MFDecoder { } else { Duration::from_millis(800) }; - let batch_start = Instant::now(); + let mut batch_start = Instant::now(); + let mut readahead_target = requested_frame + readahead_frames; loop { if sender.as_ref().is_some_and(|s| s.is_closed()) { @@ -364,6 +366,33 @@ impl MFDecoder { break; } + // Serve requests that arrive during readahead between samples and slide the + // window with them; otherwise a sequential reader waits out the whole batch + // and then finds the decoder idle while it consumes the cached frames. + if sender.is_none() && deferred_requests.is_empty() { + match rx.try_recv() { + Ok(VideoDecoderMessage::GetFrame(time, distance, reply)) => { + let frame = (time * fps as f32).floor() as u32; + if let Some(cached) = + cached_frame_for(&cache, last_decoded_frame, frame) + { + let _ = reply.send(cached.to_decoded_frame()); + readahead_target = + readahead_target.max(frame + readahead_frames); + cache_min = frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); + cache_max = frame + FRAME_CACHE_SIZE as u32 / 2; + batch_start = Instant::now(); + continue; + } + carried = + Some(VideoDecoderMessage::GetFrame(time, distance, reply)); + break; + } + Err(mpsc::TryRecvError::Empty) => {} + Err(mpsc::TryRecvError::Disconnected) => break, + } + } + let decode_start = Instant::now(); match decoder.read_sample() { Ok(Some(mf_frame)) => { @@ -448,18 +477,14 @@ impl MFDecoder { last_valid_frame = Some(cached); } - if frame_number >= requested_frame { - let frame_to_send = if frame_number == requested_frame { - cache.get(&requested_frame) - } else { - last_valid_frame.as_ref().or_else(|| { - cache - .range(..=requested_frame) - .next_back() - .map(|(_, f)| f) - .or_else(|| cache.get(&frame_number)) - }) - }; + if frame_number > requested_frame { + let frame_to_send = last_valid_frame.as_ref().or_else(|| { + cache + .range(..=requested_frame) + .next_back() + .map(|(_, f)| f) + .or_else(|| cache.get(&frame_number)) + }); if let Some(frame) = frame_to_send && let Some(s) = sender.take() @@ -468,7 +493,6 @@ impl MFDecoder { } } - let readahead_target = requested_frame + readahead_frames; if frame_number >= readahead_target || frame_number > cache_max { break; } @@ -544,21 +568,22 @@ fn pts_100ns_to_frame(pts_100ns: i64, fps: u32) -> u32 { super::pts_to_frame(pts_100ns, ::ffmpeg::Rational::new(1, 10_000_000), fps) } -// A frame the decoder has not reached yet has to be decoded; answering with -// the cached frame before it shows a stale image at every readahead boundary. -// Once the decoder is past a missing frame, the stream has no frame there, and -// the latest earlier one is the hold the video really shows. +// A frame the decoder has not moved past yet has to be decoded: answering with +// the cached frame before it shows a stale image at every readahead boundary, +// and a VFR stream can still have a later sample rounding to the same frame +// (which replaces it in the cache), so answering early would make the choice +// depend on request timing. Once the decoder is past a missing frame, the +// stream has no frame there, and the latest earlier one is the hold the video +// really shows. fn cached_frame_for( cache: &BTreeMap, last_decoded: Option, frame: u32, ) -> Option<&T> { - cache.get(&frame).or_else(|| { - last_decoded - .is_some_and(|last| last > frame) - .then(|| cache.range(..=frame).next_back().map(|(_, cached)| cached)) - .flatten() - }) + last_decoded + .is_some_and(|last| last > frame) + .then(|| cache.range(..=frame).next_back().map(|(_, cached)| cached)) + .flatten() } #[cfg(test)] @@ -576,11 +601,18 @@ mod tests { #[test] fn an_undecoded_frame_is_not_answered_with_the_one_before_it() { let cache = BTreeMap::from([(14, 'a'), (15, 'b')]); - assert_eq!(cached_frame_for(&cache, Some(15), 15), Some(&'b')); assert_eq!(cached_frame_for(&cache, Some(15), 16), None); assert_eq!(cached_frame_for(&cache, Some(15), 17), None); } + #[test] + fn the_newest_decoded_frame_waits_for_a_later_sample() { + let cache = BTreeMap::from([(14, 'a'), (15, 'b')]); + assert_eq!(cached_frame_for(&cache, Some(15), 14), Some(&'a')); + assert_eq!(cached_frame_for(&cache, Some(15), 15), None); + assert_eq!(cached_frame_for(&cache, Some(16), 15), Some(&'b')); + } + #[test] fn a_frame_the_stream_skipped_holds_the_previous_one() { let cache = BTreeMap::from([(14, 'a'), (17, 'b')]); From 6535881fa7e2f9de3a8f9113cd2a79810c70c5bc Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:19:02 -0600 Subject: [PATCH 06/19] chore(export): log NV12 export loop stage timings under CAP_RENDER_LOOP_STATS The renderer already accumulates per-stage time (decode wait, render, next-frame decode, join, send) behind CAP_RENDER_LOOP_STATS=1, but only the render farm read it. MP4 exports now log it at completion, together with the time the forwarder spent blocked on the encoder, which is what showed the 1080p export to be decode-bound and 4K to be NVENC-bound. --- crates/export/src/mp4.rs | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs index 99efd61730c..2370b65645c 100644 --- a/crates/export/src/mp4.rs +++ b/crates/export/src/mp4.rs @@ -1445,6 +1445,7 @@ async fn export_render_to_channel( let mut first_frame_data: Option = None; let mut frame_count = 0u32; let mut consecutive_timeouts = 0u32; + let mut encoder_blocked = Duration::ZERO; loop { let timeout_secs = if frame_count == 0 { @@ -1507,7 +1508,10 @@ async fn export_render_to_channel( }); } - if sender.send(export_frame).is_err() { + let send_started = std::time::Instant::now(); + let sent = sender.send(export_frame); + encoder_blocked += send_started.elapsed(); + if sent.is_err() { warn!("Encoder dropped, stopping render forwarding"); if stop_on_encoder_drop { return Err(cap_rendering::RenderingError::ImageLoadError( @@ -1536,13 +1540,29 @@ async fn export_render_to_channel( }); } - Ok::<_, cap_rendering::RenderingError>(()) + Ok::<_, cap_rendering::RenderingError>(encoder_blocked) }; + cap_rendering::loop_stats::take(); tokio::try_join!(render_future, forward_future) }; - render_result?; + let (_, encoder_blocked) = render_result?; + + if cap_rendering::loop_stats::enabled() { + let [decode, render, prefetch, join, render_send, frames] = + cap_rendering::loop_stats::take(); + info!( + frames, + decode_ms = decode / 1000, + render_ms = render / 1000, + prefetch_ms = prefetch / 1000, + join_ms = join / 1000, + render_send_ms = render_send / 1000, + encoder_send_ms = encoder_blocked.as_millis() as u64, + "NV12 export loop stats" + ); + } Ok(()) } From a4dae84fb775d9ba39d3db6bd38f187f36b75924 Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:24:59 -0600 Subject: [PATCH 07/19] perf(recording): pool the CPU frames Windows screen capture copies into Every captured frame was copied into a freshly allocated FFmpeg frame (8 MB at 1080p, 33 MB at 4K). Allocations that large come straight from new OS pages on Windows, so each frame also paid the page faults and zeroing, measured at ~2 ms per 1080p frame (~9 ms at 4K) against ~0.45 ms for the copy itself. Take the buffers from an AVBufferPool instead; they return to the pool when the encoder and the duplicate-frame holder drop them. capbench, 10 s recordings of a 60 fps test pattern, 5 interleaved reps (with the cursor change that follows; instant mode has no cursor thread, so its number is this change alone): studio 60 fps 0.309 -> 0.248 cores (-20%), delivered fps unchanged instant 30 fps 0.205 -> 0.167 cores (-18%) peak commit +7 MB (the pooled buffers) --- crates/scap-ffmpeg/src/direct3d.rs | 99 +++++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 10 deletions(-) diff --git a/crates/scap-ffmpeg/src/direct3d.rs b/crates/scap-ffmpeg/src/direct3d.rs index fcfdd34fa6c..762af9c58b9 100644 --- a/crates/scap-ffmpeg/src/direct3d.rs +++ b/crates/scap-ffmpeg/src/direct3d.rs @@ -46,6 +46,65 @@ fn copy_frame_data( } } +struct FramePool { + key: (Pixel, u32, u32), + linesize: usize, + pool: *mut ffmpeg::ffi::AVBufferPool, +} + +impl Drop for FramePool { + fn drop(&mut self) { + // Frees the pool once every buffer still referenced by an in-flight frame is returned. + unsafe { ffmpeg::ffi::av_buffer_pool_uninit(&mut self.pool) }; + } +} + +thread_local! { + static FRAME_POOL: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; +} + +// A fresh multi-megabyte allocation per frame is served by fresh OS pages on Windows, so every +// capture paid page-fault and zeroing costs several times the copy itself (~2ms per 1080p frame). +fn pooled_frame(format: Pixel, width: u32, height: u32) -> ffmpeg::frame::Video { + debug_assert!(matches!(format, Pixel::RGBA | Pixel::BGRA)); + FRAME_POOL.with_borrow_mut(|slot| { + if slot + .as_ref() + .is_none_or(|pool| pool.key != (format, width, height)) + { + *slot = None; + let linesize = (width as usize).next_multiple_of(32) * 4; + let size = + linesize * height as usize + ffmpeg::ffi::AV_INPUT_BUFFER_PADDING_SIZE as usize; + let pool = unsafe { ffmpeg::ffi::av_buffer_pool_init(size, None) }; + if pool.is_null() { + return ffmpeg::frame::Video::new(format, width, height); + } + *slot = Some(FramePool { + key: (format, width, height), + linesize, + pool, + }); + } + let pool = slot.as_ref().expect("frame pool initialized above"); + unsafe { + let buffer = ffmpeg::ffi::av_buffer_pool_get(pool.pool); + if buffer.is_null() { + return ffmpeg::frame::Video::new(format, width, height); + } + let mut frame = ffmpeg::frame::Video::empty(); + frame.set_format(format); + frame.set_width(width); + frame.set_height(height); + let raw = frame.as_mut_ptr(); + (*raw).buf[0] = buffer; + (*raw).data[0] = (*buffer).data; + (*raw).linesize[0] = pool.linesize as i32; + frame + } + }) +} + impl super::AsFFmpeg for scap_direct3d::Frame { fn as_ffmpeg(&self) -> Result { let buffer = self.as_buffer()?; @@ -59,11 +118,7 @@ impl super::AsFFmpeg for scap_direct3d::Frame { match self.pixel_format() { PixelFormat::R8G8B8A8Unorm => { - let mut ff_frame = ffmpeg::frame::Video::new( - ffmpeg::format::Pixel::RGBA, - self.width(), - self.height(), - ); + let mut ff_frame = pooled_frame(Pixel::RGBA, self.width(), self.height()); let dest_stride = ff_frame.stride(0); let dest_bytes = ff_frame.data_mut(0); @@ -80,11 +135,7 @@ impl super::AsFFmpeg for scap_direct3d::Frame { Ok(ff_frame) } PixelFormat::B8G8R8A8Unorm => { - let mut ff_frame = ffmpeg::frame::Video::new( - ffmpeg::format::Pixel::BGRA, - self.width(), - self.height(), - ); + let mut ff_frame = pooled_frame(Pixel::BGRA, self.width(), self.height()); let dest_stride = ff_frame.stride(0); let dest_bytes = ff_frame.data_mut(0); @@ -116,3 +167,31 @@ impl PixelFormatExt for PixelFormat { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pooled_frames_recycle_released_buffers() { + let first = pooled_frame(Pixel::BGRA, 1001, 3); + assert_eq!( + (first.format(), first.width(), first.height()), + (Pixel::BGRA, 1001, 3) + ); + assert!(first.stride(0) >= 1001 * 4); + assert!(first.data(0).len() >= first.stride(0) * 3); + let held = first.data(0).as_ptr(); + + let second = pooled_frame(Pixel::BGRA, 1001, 3); + assert_ne!(second.data(0).as_ptr(), held); + + drop(first); + let third = pooled_frame(Pixel::BGRA, 1001, 3); + assert_eq!(third.data(0).as_ptr(), held); + + let resized = pooled_frame(Pixel::BGRA, 640, 2); + assert_eq!((resized.width(), resized.height()), (640, 2)); + drop((second, third)); + } +} From 75da5484c57d8bb3866a90f928361f1cadc889c7 Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:25:00 -0600 Subject: [PATCH 08/19] perf(recording): reuse the Windows cursor image while its handle is unchanged The cursor thread rendered the cursor into a DIB, trimmed it, PNG-encoded it and hashed the PNG on every tick (~32 Hz) only to find the same id. That costs ~250 us per tick (~0.8% of a core for the whole recording) against ~0.5 us for returning the previous result when GetCursorInfo reports the same HCURSOR. --- crates/recording/src/cursor.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/crates/recording/src/cursor.rs b/crates/recording/src/cursor.rs index 8dba976650e..edfd17639aa 100644 --- a/crates/recording/src/cursor.rs +++ b/crates/recording/src/cursor.rs @@ -573,7 +573,7 @@ pub fn spawn_cursor_recorder( } } -#[derive(Debug)] +#[derive(Debug, Clone)] struct CursorData { image: Vec, hotspot: XY, @@ -798,6 +798,23 @@ fn get_cursor_data() -> Option { return None; } + // Rendering, trimming and PNG-encoding the cursor every tick only to hash it is wasted + // while the shape is unchanged. Keyed on the handle, so a cursor scheme change that keeps + // the system cursor handles shows the old image until the shape changes; key on the + // bitmap contents instead if that ever matters. + thread_local! { + static LAST_CURSOR: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; + } + let handle = cursor_info.hCursor.0 as usize; + if let Some(cached) = LAST_CURSOR.with_borrow(|last| { + last.as_ref() + .filter(|(last_handle, _)| *last_handle == handle) + .map(|(_, data)| data.clone()) + }) { + return Some(cached); + } + // Get icon info let mut icon_info = ICONINFO::default(); if GetIconInfo(cursor_info.hCursor.into(), &mut icon_info).is_err() { @@ -1071,11 +1088,13 @@ fn get_cursor_data() -> Option { ) .ok()?; - Some(CursorData { + let data = CursorData { image: png_data, hotspot: XY::new(hotspot_x, hotspot_y), shape: CursorShape::try_from(&cursor_info.hCursor).ok(), - }) + }; + LAST_CURSOR.set(Some((handle, data.clone()))); + Some(data) } } From b3a08c51569340d49fb6a46988e44002229547ba Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:22:35 -0600 Subject: [PATCH 09/19] perf(desktop-gpui): drop the export timer-resolution hold Windows exports held a 1 ms timer resolution because the NV12 readback wait slept in 1 ms steps. Readbacks now block on the GPU fence and nothing on the export's per-frame path sleeps: CLI exports (1080p60, 5 interleaved reps) take 8.62 s without the hold and 8.70 s with it, so the process no longer raises its timer resolution for the length of an export. --- apps/desktop-gpui/Cargo.toml | 1 - apps/desktop-gpui/src/editor_export.rs | 23 ----------------------- 2 files changed, 24 deletions(-) diff --git a/apps/desktop-gpui/Cargo.toml b/apps/desktop-gpui/Cargo.toml index f0823efa081..1826ab22ab7 100644 --- a/apps/desktop-gpui/Cargo.toml +++ b/apps/desktop-gpui/Cargo.toml @@ -179,7 +179,6 @@ raw-window-handle = "0.6" tray-icon = { version = "0.21.1", default-features = false } windows-sys = { version = "0.59", features = [ "Win32_Foundation", - "Win32_Media", "Win32_Security", "Win32_System_Registry", "Win32_System_SystemInformation", diff --git a/apps/desktop-gpui/src/editor_export.rs b/apps/desktop-gpui/src/editor_export.rs index abbf6d98e15..9d9ee19a795 100644 --- a/apps/desktop-gpui/src/editor_export.rs +++ b/apps/desktop-gpui/src/editor_export.rs @@ -2863,26 +2863,6 @@ fn matches_compression(current: ExportCompression, expected: ExportCompression) std::mem::discriminant(¤t) == std::mem::discriminant(&expected) } -// The NV12 readback wait sleeps in 1 ms steps (`frame_pipeline.rs`), which the -// default 15.6 ms Windows timer resolution stretches to most of a frame each. -#[cfg(windows)] -struct TimerResolution; - -#[cfg(windows)] -impl TimerResolution { - fn hold_one_millisecond() -> Self { - unsafe { windows_sys::Win32::Media::timeBeginPeriod(1) }; - Self - } -} - -#[cfg(windows)] -impl Drop for TimerResolution { - fn drop(&mut self) { - unsafe { windows_sys::Win32::Media::timeEndPeriod(1) }; - } -} - #[allow(clippy::too_many_arguments)] async fn run_export( project_path: PathBuf, @@ -2903,9 +2883,6 @@ async fn run_export( if cancel.load(Ordering::Relaxed) { return Err("Export cancelled".into()); } - #[cfg(windows)] - let _timer_resolution = TimerResolution::hold_one_millisecond(); - let mut builder = ExporterBase::builder(project_path).with_force_ffmpeg_decoder(force); if cursor_only { builder = builder.with_config(make_cursor_only_project(project)); From bad975f00f92988041cd936f1f4603227dd3c6a0 Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:45:42 -0600 Subject: [PATCH 10/19] fix(capbench): fail runs whose recording does not validate and keep failure logs `cap-test validate` exits 0 for an invalid recording, so read its `valid` and `errors` and count such runs as failures instead of comparing them. Failure logs were written under the work directory that is deleted after the run; copy them to /failures/ and record that path. --- scripts/capbench.py | 37 ++++++++++++++++++++++++++++--------- scripts/test-capbench.py | 9 +++++++++ 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/scripts/capbench.py b/scripts/capbench.py index 96d210f813d..600fa3ae123 100644 --- a/scripts/capbench.py +++ b/scripts/capbench.py @@ -347,6 +347,20 @@ def argv(ctx, out_dir): return {"argv": argv, "parse": parse, "timeout": timeout, "env": env or {}, "better": better} +class InvalidRun(Exception): + pass + + +def validated_video(stdout, returncode): + try: + report = json.loads(stdout) + except json.JSONDecodeError: + raise InvalidRun(f"cap-test validate exited {returncode} without a JSON report") + if returncode != 0 or not report.get("valid"): + raise InvalidRun("invalid recording: " + "; ".join(report.get("errors") or ["no errors reported"])) + return report.get("video_info") or {} + + def record_bench(mode, fps, seconds): def argv(ctx, out_dir): return [ @@ -360,10 +374,7 @@ def post(ctx, out_dir, metrics): [str(ctx.bin("cap-test")), "validate", str(project), "--json"], capture_output=True, text=True, ) - try: - video = json.loads(validation.stdout).get("video_info") or {} - except json.JSONDecodeError: - video = {} + video = validated_video(validation.stdout, validation.returncode) frames = video.get("frame_count") captured = video.get("duration_secs") if frames and captured: @@ -497,11 +508,16 @@ def run_once(name, spec, ctx, out_dir, keep_log): metrics = {key: value for key, value in measured.items() if key != "exit"} metrics.update(spec["parse"](lines)) artifact = None - if measured["exit"] == 0 and spec.get("post"): - artifact = spec["post"](ctx, out_dir, metrics) - if keep_log or measured["exit"] != 0: + code = measured["exit"] + if code == 0 and spec.get("post"): + try: + artifact = spec["post"](ctx, out_dir, metrics) + except InvalidRun as error: + code = "invalid" + lines.append((measured["wall_s"], f"capbench: {error}")) + if keep_log or code != 0: (out_dir / "output.log").write_text("\n".join(line for _, line in lines), encoding="utf-8") - return measured["exit"], metrics, artifact + return code, metrics, artifact def build_info(bin_dir): @@ -600,7 +616,10 @@ def cmd_run(args): if phase == "warmup": continue if code != 0: - results["benches"][name]["failures"].append({"rep": rep, "exit": code, "log": str(out_dir / "output.log")}) + log = out_root / "failures" / label / name / f"{phase}{rep}.log" + log.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(out_dir / "output.log", log) + results["benches"][name]["failures"].append({"rep": rep, "exit": code, "log": str(log)}) continue results["benches"][name]["runs"].append(metrics) if artifact and args.keep_outputs: diff --git a/scripts/test-capbench.py b/scripts/test-capbench.py index 4c450054bde..9eefd98ab85 100644 --- a/scripts/test-capbench.py +++ b/scripts/test-capbench.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import importlib.util +import json from pathlib import Path import sys import unittest @@ -117,6 +118,14 @@ def results(values): noise = capbench.compare(results([10.0, 12.0, 8.0, 11.0, 9.0]), results([10.5, 8.5, 11.5, 9.5, 12.5])) self.assertEqual(noise[0]["verdict"], "same") + def test_invalid_recordings_fail_the_run(self): + video = {"frame_count": 300, "duration_secs": 10.0} + self.assertEqual(capbench.validated_video(json.dumps({"valid": True, "video_info": video}), 0), video) + with self.assertRaisesRegex(capbench.InvalidRun, "Missing content directory"): + capbench.validated_video(json.dumps({"valid": False, "errors": ["Missing content directory"]}), 0) + with self.assertRaises(capbench.InvalidRun): + capbench.validated_video("thread 'main' panicked", 101) + def test_variant_env_is_scoped_per_label(self): env = capbench.parse_variant_env(["base", "t4"], ["t4:CAP_EXP_DECODE_THREADS=4", "t4:X=a=b"]) self.assertEqual(env, {"base": {}, "t4": {"CAP_EXP_DECODE_THREADS": "4", "X": "a=b"}}) From 610ba3971f38bfda02597ad725648976c6b05254 Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:06:11 -0600 Subject: [PATCH 11/19] docs(export): note that the loop stats are process-wide --- crates/export/src/mp4.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs index 2370b65645c..e042a5b7ccb 100644 --- a/crates/export/src/mp4.rs +++ b/crates/export/src/mp4.rs @@ -1549,6 +1549,9 @@ async fn export_render_to_channel( let (_, encoder_blocked) = render_result?; + // The loop_stats counters are process-wide, so an export that overlaps another one reports + // a mix of both. That is fine for profiling one export at a time; per-export numbers need + // the counters passed through the render loop instead of kept in statics. if cap_rendering::loop_stats::enabled() { let [decode, render, prefetch, join, render_send, frames] = cap_rendering::loop_stats::take(); From 470f90dc0c5e8e56acaf8c3bec417f5ce498affd Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:04:42 -0600 Subject: [PATCH 12/19] fix(rendering): tag readahead Media Foundation frames with the source colour matrix --- crates/rendering/src/decoder/media_foundation.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/rendering/src/decoder/media_foundation.rs b/crates/rendering/src/decoder/media_foundation.rs index 36430827277..664fb43ad22 100644 --- a/crates/rendering/src/decoder/media_foundation.rs +++ b/crates/rendering/src/decoder/media_foundation.rs @@ -379,7 +379,8 @@ impl MFDecoder { if let Some(cached) = cached_frame_for(&cache, last_decoded_frame, frame) { - let _ = reply.send(cached.to_decoded_frame()); + let _ = + reply.send(cached.to_decoded_frame().with_matrix(matrix)); readahead_target = readahead_target.max(frame + readahead_frames); cache_min = frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); From 9611675195a255bc3cce527885d6b76c67e0b8f1 Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:14:19 -0600 Subject: [PATCH 13/19] fix(scripts): reject benchmark runs that report no measurements A benchmark that exits 0 without its metrics (decode-benchmark does when it cannot create a decoder) was kept with a warning, and its short wall time could make a broken build look faster. Such runs are now invalid runs, like failed ffprobe or recording validation. The job's CPU and memory figures stay optional because they exist only on Windows, and instant recordings no longer list delivered_fps and captured_ratio, which cap-test validate never reports for them. --- scripts/capbench.py | 29 ++++++++++++++++++++++------- scripts/test-capbench.py | 11 +++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/scripts/capbench.py b/scripts/capbench.py index 600fa3ae123..94e6769dcd3 100644 --- a/scripts/capbench.py +++ b/scripts/capbench.py @@ -368,6 +368,14 @@ def argv(ctx, out_dir): "--fps", str(fps), "--duration", str(seconds), "--path", str(out_dir / "rec.cap"), ] + better = { + "cpu_per_rec_s": "lower", "peak_commit_mb": "lower", "delivered_fps": "higher", + "captured_ratio": "higher", "overhead_s": "lower", + } + if mode == "instant": + # cap-test validate reports no frame count or duration for instant recordings. + del better["delivered_fps"], better["captured_ratio"] + def post(ctx, out_dir, metrics): project = out_dir / "rec.cap" validation = subprocess.run( @@ -394,10 +402,7 @@ def post(ctx, out_dir, metrics): "timeout": seconds + 120, "env": {}, "needs_screen": True, - "better": { - "cpu_per_rec_s": "lower", "peak_commit_mb": "lower", "delivered_fps": "higher", - "captured_ratio": "higher", "overhead_s": "lower", - }, + "better": better, } @@ -495,6 +500,15 @@ def start_animation(bounds): return process +# Job accounting, and the metrics derived from it, exist only where the job object does (Windows). +JOB_METRICS = {"cpu_s", "peak_commit_mb", "cpu_per_rec_s"} + + +def missing_metrics(better, metrics): + optional = set() if "cpu_s" in metrics else JOB_METRICS + return sorted(key for key in better if key not in metrics and key not in optional) + + def run_once(name, spec, ctx, out_dir, keep_log): out_dir.mkdir(parents=True, exist_ok=True) argv = spec["argv"](ctx, out_dir) @@ -515,6 +529,10 @@ def run_once(name, spec, ctx, out_dir, keep_log): except InvalidRun as error: code = "invalid" lines.append((measured["wall_s"], f"capbench: {error}")) + missing = missing_metrics(spec["better"], metrics) + if code == 0 and missing: + code = "invalid" + lines.append((measured["wall_s"], f"capbench: no {', '.join(missing)} in this run's output")) if keep_log or code != 0: (out_dir / "output.log").write_text("\n".join(line for _, line in lines), encoding="utf-8") return code, metrics, artifact @@ -610,9 +628,6 @@ def cmd_run(args): status = "ok" if code == 0 else f"exit={code}" summary = ", ".join(f"{key}={value:.3f}" for key, value in sorted(metrics.items()) if key in spec["better"]) print(f"[{phase} {rep}] {label} {name}: {status} {summary}", flush=True) - missing = sorted(key for key in spec["better"] if key not in metrics) - if code == 0 and missing: - print(f" warning: no {', '.join(missing)} in this run's output", flush=True) if phase == "warmup": continue if code != 0: diff --git a/scripts/test-capbench.py b/scripts/test-capbench.py index 9eefd98ab85..0351938266d 100644 --- a/scripts/test-capbench.py +++ b/scripts/test-capbench.py @@ -126,6 +126,17 @@ def test_invalid_recordings_fail_the_run(self): with self.assertRaises(capbench.InvalidRun): capbench.validated_video("thread 'main' panicked", 101) + def test_runs_missing_a_metric_are_invalid(self): + better = capbench.BENCHES["decode-display"]["better"] + complete = {key: 1.0 for key in better} + self.assertEqual(capbench.missing_metrics(better, complete), []) + crashed = {"wall_s": 0.4, "cpu_s": 0.1, "peak_commit_mb": 40.0} + self.assertIn("sequential_fps", capbench.missing_metrics(better, crashed)) + no_job = {key: 1.0 for key in better if key not in capbench.JOB_METRICS} + self.assertEqual(capbench.missing_metrics(better, no_job), []) + self.assertNotIn("delivered_fps", capbench.BENCHES["record-instant-30"]["better"]) + self.assertIn("delivered_fps", capbench.BENCHES["record-studio-60"]["better"]) + def test_variant_env_is_scoped_per_label(self): env = capbench.parse_variant_env(["base", "t4"], ["t4:CAP_EXP_DECODE_THREADS=4", "t4:X=a=b"]) self.assertEqual(env, {"base": {}, "t4": {"CAP_EXP_DECODE_THREADS": "4", "X": "a=b"}}) From 0505abc52ad7704ac4aaebe0f1b90b317c6beec1 Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:01:14 -0600 Subject: [PATCH 14/19] fix(scripts): check benchmark helpers up front and reject decode runs with failed reads A missing cap-test or ffprobe raised out of the run loop after earlier benchmarks had finished, so nothing was saved; each bench now names the helpers it needs and the startup check finds them before any run. decode-benchmark exits 0 and times only the reads that succeeded, so failed sequential reads, seeks or random reads now make the run invalid. --- scripts/capbench.py | 30 ++++++++++++++++++++++++------ scripts/test-capbench.py | 13 +++++++++++++ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/scripts/capbench.py b/scripts/capbench.py index 94e6769dcd3..12f072b1a21 100644 --- a/scripts/capbench.py +++ b/scripts/capbench.py @@ -209,6 +209,13 @@ def section_floats(lines, start_pattern, pattern, group=1): return None +def on_path(name): + path = shutil.which(name) + if path is None: + raise FileNotFoundError(f"missing {name}; put it on PATH") + return path + + def ffprobe_video(path): result = subprocess.run( [ @@ -273,6 +280,7 @@ def post(ctx, out_dir, metrics): "argv": argv, "parse": parse_export, "post": post, + "requires": lambda ctx: on_path("ffprobe"), "timeout": 900, "env": {"CAP_DISABLE_ENCODER_SELF_TEST": None}, "better": { @@ -329,6 +337,14 @@ def parse_decode(lines): "random_avg_ms": float_after(lines, r"Avg access time:\s*([\d.]+)ms"), "random_p95_ms": section_floats(lines, r"RANDOM ACCESS PERFORMANCE", r"P95:\s*([\d.]+)ms"), } + # Timings cover only the reads that succeeded, so a run with failed reads looks faster. + failed = sum( + int(match.group(2)) + for _, line in lines + if (match := re.match(r"(Frames failed|Seek failures|Failures):\s*(\d+)", line.strip())) + ) + if failed: + raise InvalidRun(f"decode-benchmark reported {failed} failed reads") return {key: value for key, value in metrics.items() if value is not None} @@ -399,6 +415,7 @@ def post(ctx, out_dir, metrics): "argv": argv, "parse": lambda lines: {}, "post": post, + "requires": lambda ctx: ctx.bin("cap-test"), "timeout": seconds + 120, "env": {}, "needs_screen": True, @@ -520,15 +537,15 @@ def run_once(name, spec, ctx, out_dir, keep_log): animation.kill() animation.wait() metrics = {key: value for key, value in measured.items() if key != "exit"} - metrics.update(spec["parse"](lines)) artifact = None code = measured["exit"] - if code == 0 and spec.get("post"): - try: + try: + metrics.update(spec["parse"](lines)) + if code == 0 and spec.get("post"): artifact = spec["post"](ctx, out_dir, metrics) - except InvalidRun as error: - code = "invalid" - lines.append((measured["wall_s"], f"capbench: {error}")) + except InvalidRun as error: + code = "invalid" if code == 0 else code + lines.append((measured["wall_s"], f"capbench: {error}")) missing = missing_metrics(spec["better"], metrics) if code == 0 and missing: code = "invalid" @@ -590,6 +607,7 @@ def cmd_run(args): try: for name in names: BENCHES[name]["argv"](ctx, work) + BENCHES[name].get("requires", lambda ctx: None)(ctx) except FileNotFoundError as error: sys.exit(str(error)) if work.exists(): diff --git a/scripts/test-capbench.py b/scripts/test-capbench.py index 0351938266d..d38d5fa8b92 100644 --- a/scripts/test-capbench.py +++ b/scripts/test-capbench.py @@ -3,6 +3,7 @@ import json from pathlib import Path import sys +from types import SimpleNamespace import unittest @@ -90,6 +91,18 @@ def test_decode_metrics(self): "random_avg_ms": 107.38, "random_p95_ms": 150.12, }) + def test_decode_runs_with_failed_reads_are_invalid(self): + failed = DECODE_OUTPUT + "\nRANDOM ACCESS PERFORMANCE\n Failures: 50\n" + with self.assertRaisesRegex(capbench.InvalidRun, "50 failed reads"): + capbench.parse_decode(timed(failed)) + clean = DECODE_OUTPUT + "\n Frames failed: 0\n" + self.assertIn("sequential_fps", capbench.parse_decode(timed(clean))) + + def test_startup_check_finds_missing_helpers(self): + args = SimpleNamespace(fixture=".", bin_dir=str(Path(__file__).parent), screen=None, animate=False, affinity=None) + with self.assertRaisesRegex(FileNotFoundError, "cap-test"): + capbench.BENCHES["record-studio-60"]["requires"](capbench.Context(args)) + def test_startup_metrics(self): lines = timed("Startup: avg=1159.3ms p50=671.2ms p95=2229.8ms\nFirst preview: avg=37.0ms p50=44.4ms p95=45.9ms") self.assertEqual(capbench.parse_startup(lines), {"startup_p50_ms": 671.2, "first_preview_p50_ms": 44.4}) From 2858f64ee314aeb9027c70e942bda3fbc29ab4d9 Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:28:45 -0600 Subject: [PATCH 15/19] fix(scripts): refuse a benchmark fixture that overlaps the work directory cmd_run deletes /work before copying the fixture into it, so a fixture reused from an earlier --keep-work run under the same --out was deleted before it was copied. Overlapping fixture and work paths now stop the run before anything is deleted. --- scripts/capbench.py | 2 ++ scripts/test-capbench.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/scripts/capbench.py b/scripts/capbench.py index 12f072b1a21..1decfcd0647 100644 --- a/scripts/capbench.py +++ b/scripts/capbench.py @@ -602,6 +602,8 @@ def cmd_run(args): out_root = Path(args.out).resolve() work = out_root / "work" + if ctx.source_fixture.is_relative_to(work) or work.is_relative_to(ctx.source_fixture): + sys.exit(f"--fixture {ctx.source_fixture} and the work directory {work} overlap; the work directory is deleted before each run") for _, bin_dir in variants: ctx.bin_dir = Path(bin_dir).resolve() try: diff --git a/scripts/test-capbench.py b/scripts/test-capbench.py index d38d5fa8b92..d6b60c01cb9 100644 --- a/scripts/test-capbench.py +++ b/scripts/test-capbench.py @@ -3,6 +3,7 @@ import json from pathlib import Path import sys +import tempfile from types import SimpleNamespace import unittest @@ -98,6 +99,19 @@ def test_decode_runs_with_failed_reads_are_invalid(self): clean = DECODE_OUTPUT + "\n Frames failed: 0\n" self.assertIn("sequential_fps", capbench.parse_decode(timed(clean))) + def test_fixture_inside_the_work_directory_is_not_deleted(self): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "out" + fixture = out / "work" / "fixture" / "run" / "reference.cap" + fixture.mkdir(parents=True) + args = SimpleNamespace( + bench=["decode-display"], out=str(out), label="run", fixture=str(fixture), bin_dir=tmp, + variant=[], env=[], screen=None, animate=False, affinity=None, + ) + with self.assertRaisesRegex(SystemExit, "overlap"): + capbench.cmd_run(args) + self.assertTrue(fixture.exists()) + def test_startup_check_finds_missing_helpers(self): args = SimpleNamespace(fixture=".", bin_dir=str(Path(__file__).parent), screen=None, animate=False, affinity=None) with self.assertRaisesRegex(FileNotFoundError, "cap-test"): From 03c72d976350fd2c172105d999628191441ad9fe Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:30:05 -0600 Subject: [PATCH 16/19] fix(scripts): keep successful-run logs when --keep-logs is set run_once writes each log inside the work directory, which is deleted at the end unless --keep-work, so --keep-logs alone kept nothing. Kept logs are now copied to /logs, like failure logs to /failures. --- scripts/capbench.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/capbench.py b/scripts/capbench.py index 1decfcd0647..5bdf2374caa 100644 --- a/scripts/capbench.py +++ b/scripts/capbench.py @@ -634,6 +634,13 @@ def cmd_run(args): "benches": {name: {"runs": [], "failures": [], "better": BENCHES[name]["better"]} for name in names}, } + # Logs are written inside work, which is deleted at the end unless --keep-work. + def save_log(kind, label, name, run, out_dir): + log = out_root / kind / label / name / f"{run}.log" + log.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(out_dir / "output.log", log) + return log + schedule = [("warmup", rep) for rep in range(args.warmup)] + [("rep", rep) for rep in range(args.reps)] for phase, rep in schedule: for name in names: @@ -651,12 +658,12 @@ def cmd_run(args): if phase == "warmup": continue if code != 0: - log = out_root / "failures" / label / name / f"{phase}{rep}.log" - log.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(out_dir / "output.log", log) + log = save_log("failures", label, name, f"{phase}{rep}", out_dir) results["benches"][name]["failures"].append({"rep": rep, "exit": code, "log": str(log)}) continue results["benches"][name]["runs"].append(metrics) + if args.keep_logs: + save_log("logs", label, name, f"{phase}{rep}", out_dir) if artifact and args.keep_outputs: kept = out_root / "artifacts" / label / f"{name}{artifact.suffix}" kept.parent.mkdir(parents=True, exist_ok=True) From e26f981f3cc6e7dc237778a5c31eca90b4a1d322 Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:11:23 -0600 Subject: [PATCH 17/19] fix(rendering): pass the submission index in the macOS surface completion test after_submit takes the submission index on this branch; the macOS-only test main added still called it with the queue alone, so cap-rendering's tests did not compile on macOS. --- crates/rendering/src/frame_pipeline.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rendering/src/frame_pipeline.rs b/crates/rendering/src/frame_pipeline.rs index d8561066468..249d222df96 100644 --- a/crates/rendering/src/frame_pipeline.rs +++ b/crates/rendering/src/frame_pipeline.rs @@ -2494,8 +2494,8 @@ mod surface_output_tests { .await .expect("surface conversion") ); - queue.submit([encoder.finish()]); - converter.after_submit(&queue); + let submission = queue.submit([encoder.finish()]); + converter.after_submit(&queue, &submission); let Some(PendingNv12Output::Surface(mut pending)) = converter.take_pending() else { panic!("expected surface output"); }; From e00c11ee31ed87b0e9562651c92c136acead8d1a Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:11:23 -0600 Subject: [PATCH 18/19] fix(scripts): refuse binaries inside the work directory and playback runs with failed frames A --bin-dir inside /work passed the binary checks and was then deleted with the work directory. playback-pipeline-benchmark exits 0 and times only the frames that decoded and rendered, so runs reporting decode or render failures are now invalid, like decode-benchmark runs with failed reads. --- scripts/capbench.py | 10 ++++++++++ scripts/test-capbench.py | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/scripts/capbench.py b/scripts/capbench.py index 5bdf2374caa..bda164fed80 100644 --- a/scripts/capbench.py +++ b/scripts/capbench.py @@ -327,6 +327,13 @@ def parse_playback_pipeline(lines): "scrub_p50_ms": scrub.get("Total (decode+render)", {}).get("p50"), "scrub_p95_ms": scrub.get("Total (decode+render)", {}).get("p95"), } + failed = sum( + int(match.group(1)) + for _, line in lines + if (match := re.match(r"(?:Decode|Render) failures:\s*(\d+)", line.strip())) + ) + if failed: + raise InvalidRun(f"playback-pipeline-benchmark reported {failed} failed frames") return {key: value for key, value in metrics.items() if value is not None} @@ -604,6 +611,9 @@ def cmd_run(args): work = out_root / "work" if ctx.source_fixture.is_relative_to(work) or work.is_relative_to(ctx.source_fixture): sys.exit(f"--fixture {ctx.source_fixture} and the work directory {work} overlap; the work directory is deleted before each run") + for label, bin_dir in variants: + if Path(bin_dir).resolve().is_relative_to(work): + sys.exit(f"binaries for {label} ({bin_dir}) are inside the work directory {work}, which is deleted before each run") for _, bin_dir in variants: ctx.bin_dir = Path(bin_dir).resolve() try: diff --git a/scripts/test-capbench.py b/scripts/test-capbench.py index d6b60c01cb9..38c32b185e5 100644 --- a/scripts/test-capbench.py +++ b/scripts/test-capbench.py @@ -85,6 +85,11 @@ def test_playback_picks_full_and_scrub_totals(self): self.assertEqual(metrics["scrub_p50_ms"], 93.93) self.assertEqual(metrics["scrub_p95_ms"], 141.57) + def test_playback_runs_with_failed_frames_are_invalid(self): + failed = PLAYBACK_OUTPUT + "\n Decode failures: 3\n Render failures: 1\n" + with self.assertRaisesRegex(capbench.InvalidRun, "4 failed frames"): + capbench.parse_playback_pipeline(timed(failed)) + def test_decode_metrics(self): metrics = capbench.parse_decode(timed(DECODE_OUTPUT)) self.assertEqual(metrics, { @@ -112,6 +117,21 @@ def test_fixture_inside_the_work_directory_is_not_deleted(self): capbench.cmd_run(args) self.assertTrue(fixture.exists()) + def test_binaries_inside_the_work_directory_are_not_deleted(self): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "out" + bins = out / "work" / "build" + bins.mkdir(parents=True) + fixture = Path(tmp) / "reference.cap" + fixture.mkdir() + args = SimpleNamespace( + bench=["decode-display"], out=str(out), label="run", fixture=str(fixture), bin_dir=str(bins), + variant=[], env=[], screen=None, animate=False, affinity=None, + ) + with self.assertRaisesRegex(SystemExit, "inside the work directory"): + capbench.cmd_run(args) + self.assertTrue(bins.exists()) + def test_startup_check_finds_missing_helpers(self): args = SimpleNamespace(fixture=".", bin_dir=str(Path(__file__).parent), screen=None, animate=False, affinity=None) with self.assertRaisesRegex(FileNotFoundError, "cap-test"): From b9c3f08b45e9b774d4e1cbbf798641cedf1b1d47 Mon Sep 17 00:00:00 2001 From: Marcos Daniel Rivera Gonzalez <139647784+LordOfGrillos@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:33:15 -0600 Subject: [PATCH 19/19] fix(capbench): reject labels that are not plain file names --- scripts/capbench.py | 3 +++ scripts/test-capbench.py | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/scripts/capbench.py b/scripts/capbench.py index bda164fed80..f270e14b661 100644 --- a/scripts/capbench.py +++ b/scripts/capbench.py @@ -605,6 +605,9 @@ def cmd_run(args): variants = [tuple(item.split("=", 1)) for item in args.variant] or [(args.label, args.bin_dir)] if len({label for label, _ in variants}) != len(variants): sys.exit("variant labels must be unique") + for label, _ in variants: + if label in ("", ".", "..") or Path(label).name != label: + sys.exit(f"label {label!r} must be a plain file name; results are written to /