perf(windows): fence-waited readbacks, overlapped MF export decode, cheaper screen capture

.github/workflows/sync-tests.yml

@@ -33,6 +33,8 @@ on:
       - "scripts/recording-reliability.py"
       - "scripts/recording-reliability-owned-process.cs"
       - "scripts/test-recording-reliability.py"
+      - "scripts/capbench.py"
+      - "scripts/test-capbench.py"
       - ".github/workflows/sync-tests.yml"
 
 concurrency:
@@ -99,6 +101,7 @@ jobs:
         run: |
           cargo test --locked -p cap-project --lib
           python -B scripts/test-recording-reliability.py
+          python -B scripts/test-capbench.py
 
       - name: Compile recording process supervisor (Windows)
         if: runner.os == 'Windows'

apps/desktop-gpui/Cargo.toml

@@ -184,7 +184,6 @@ raw-window-handle = "0.6"
 tray-icon = { version = "0.21.1", default-features = false }
 windows-sys = { version = "0.59", features = [
     "Win32_Foundation",
-    "Win32_Media",
     "Win32_Security",
     "Win32_System_Registry",
     "Win32_System_SystemInformation",

apps/desktop-gpui/src/editor_export.rs

@@ -2863,26 +2863,6 @@ fn matches_compression(current: ExportCompression, expected: ExportCompression)
     std::mem::discriminant(&current) == std::mem::discriminant(&expected)
 }
 
-// The NV12 readback wait sleeps in 1 ms steps (`frame_pipeline.rs`), which the
-// default 15.6 ms Windows timer resolution stretches to most of a frame each.
-#[cfg(windows)]
-struct TimerResolution;
-
-#[cfg(windows)]
-impl TimerResolution {
-    fn hold_one_millisecond() -> Self {
-        unsafe { windows_sys::Win32::Media::timeBeginPeriod(1) };
-        Self
-    }
-}
-
-#[cfg(windows)]
-impl Drop for TimerResolution {
-    fn drop(&mut self) {
-        unsafe { windows_sys::Win32::Media::timeEndPeriod(1) };
-    }
-}
-
 #[allow(clippy::too_many_arguments)]
 async fn run_export(
     project_path: PathBuf,
@@ -2903,9 +2883,6 @@ async fn run_export(
     if cancel.load(Ordering::Relaxed) {
         return Err("Export cancelled".into());
     }
-    #[cfg(windows)]
-    let _timer_resolution = TimerResolution::hold_one_millisecond();
-
     let mut builder = ExporterBase::builder(project_path).with_force_ffmpeg_decoder(force);
     if cursor_only {
         builder = builder.with_config(make_cursor_only_project(project));

crates/editor/examples/playback-pipeline-benchmark.rs

@@ -849,6 +849,12 @@ async fn main() {
         .and_then(|s| s.parse().ok())
         .unwrap_or(300);
     let force_ffmpeg = args.iter().any(|arg| arg == "--force-ffmpeg");
+    // The editor holds a 1ms timer resolution while playing; this measures under the same condition.
+    let timer_1ms = args.iter().any(|arg| arg == "--timer-1ms");
+    #[cfg(target_os = "windows")]
+    if timer_1ms {
+        unsafe { windows::Win32::Media::timeBeginPeriod(1) };
+    }
 
     println!("{}", "=".repeat(60));
     println!("  CAP PLAYBACK PIPELINE BENCHMARK");
@@ -858,6 +864,7 @@ async fn main() {
     println!("Target FPS: {fps}");
     println!("Max frames: {frame_count}");
     println!("Force FFmpeg: {force_ffmpeg}");
+    println!("1ms timer: {timer_1ms}");
     println!();
 
     let (recording_meta, meta, project, recordings) = match load_recording(&recording_path).await {

crates/export/src/mp4.rs

@@ -1477,6 +1477,7 @@ async fn export_render_to_channel(
             let mut first_frame_data: Option<FirstFrameNv12> = None;
             let mut frame_count = 0u32;
             let mut consecutive_timeouts = 0u32;
+            let mut encoder_blocked = Duration::ZERO;
 
             loop {
                 let timeout_secs = if frame_count == 0 {
@@ -1539,7 +1540,10 @@ async fn export_render_to_channel(
                     });
                 }
 
-                if sender.send(export_frame).is_err() {
+                let send_started = std::time::Instant::now();
+                let sent = sender.send(export_frame);
+                encoder_blocked += send_started.elapsed();
+                if sent.is_err() {
                     warn!("Encoder dropped, stopping render forwarding");
                     if stop_on_encoder_drop {
                         return Err(cap_rendering::RenderingError::ImageLoadError(
@@ -1568,13 +1572,32 @@ async fn export_render_to_channel(
                 });
             }
 
-            Ok::<_, cap_rendering::RenderingError>(())
+            Ok::<_, cap_rendering::RenderingError>(encoder_blocked)
         };
 
+        cap_rendering::loop_stats::take();
         tokio::try_join!(render_future, forward_future)
     };
 
-    render_result?;
+    let (_, encoder_blocked) = render_result?;
+
+    // The loop_stats counters are process-wide, so an export that overlaps another one reports
+    // a mix of both. That is fine for profiling one export at a time; per-export numbers need
+    // the counters passed through the render loop instead of kept in statics.
+    if cap_rendering::loop_stats::enabled() {
+        let [decode, render, prefetch, join, render_send, frames] =
+            cap_rendering::loop_stats::take();
+        info!(
+            frames,
+            decode_ms = decode / 1000,
+            render_ms = render / 1000,
+            prefetch_ms = prefetch / 1000,
+            join_ms = join / 1000,
+            render_send_ms = render_send / 1000,
+            encoder_send_ms = encoder_blocked.as_millis() as u64,
+            "NV12 export loop stats"
+        );
+    }
 
     Ok(())
 }

crates/recording/src/cursor.rs

@@ -770,7 +770,7 @@ fn cursor_id_for_image(
     cursor_id
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 struct CursorData {
     image: Vec<u8>,
     hotspot: XY<f64>,
@@ -992,6 +992,23 @@ fn get_cursor_data() -> Option<CursorData> {
             return None;
         }
 
+        // Rendering, trimming and PNG-encoding the cursor every tick only to hash it is wasted
+        // while the shape is unchanged. Keyed on the handle, so a cursor scheme change that keeps
+        // the system cursor handles shows the old image until the shape changes; key on the
+        // bitmap contents instead if that ever matters.
+        thread_local! {
+            static LAST_CURSOR: std::cell::RefCell<Option<(usize, CursorData)>> =
+                const { std::cell::RefCell::new(None) };
+        }
+        let handle = cursor_info.hCursor.0 as usize;
+        if let Some(cached) = LAST_CURSOR.with_borrow(|last| {
+            last.as_ref()
+                .filter(|(last_handle, _)| *last_handle == handle)
+                .map(|(_, data)| data.clone())
+        }) {
+            return Some(cached);
+        }
+
         // Get icon info
         let mut icon_info = ICONINFO::default();
         if GetIconInfo(cursor_info.hCursor.into(), &mut icon_info).is_err() {
@@ -1265,11 +1282,13 @@ fn get_cursor_data() -> Option<CursorData> {
             )
             .ok()?;
 
-        Some(CursorData {
+        let data = CursorData {
             image: png_data,
             hotspot: XY::new(hotspot_x, hotspot_y),
             shape: CursorShape::try_from(&cursor_info.hCursor).ok(),
-        })
+        };
+        LAST_CURSOR.set(Some((handle, data.clone())));
+        Some(data)
     }
 }
 

crates/rendering/src/decoder/media_foundation.rs

@@ -249,7 +249,8 @@ impl MFDecoder {
                 sender: oneshot::Sender<DecodedFrame>,
             }
 
-            while let Ok(r) = rx.recv() {
+            let mut carried: Option<VideoDecoderMessage> = None;
+            while let Some(r) = carried.take().or_else(|| rx.recv().ok()) {
                 let mut pending_requests: Vec<PendingRequest> = Vec::with_capacity(8);
 
                 let mut push_request =
@@ -315,8 +316,8 @@ impl MFDecoder {
                     );
                 }
 
-                let cache_min = requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2);
-                let cache_max = requested_frame + FRAME_CACHE_SIZE as u32 / 2;
+                let mut cache_min = requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2);
+                let mut cache_max = requested_frame + FRAME_CACHE_SIZE as u32 / 2;
 
                 let needs_seek = last_decoded_frame
                     .map(|last| {
@@ -355,7 +356,8 @@ impl MFDecoder {
                 } else {
                     Duration::from_millis(800)
                 };
-                let batch_start = Instant::now();
+                let mut batch_start = Instant::now();
+                let mut readahead_target = requested_frame + readahead_frames;
 
                 loop {
                     if sender.as_ref().is_some_and(|s| s.is_closed()) {
@@ -367,6 +369,34 @@ impl MFDecoder {
                         break;
                     }
 
+                    // Serve requests that arrive during readahead between samples and slide the
+                    // window with them; otherwise a sequential reader waits out the whole batch
+                    // and then finds the decoder idle while it consumes the cached frames.
+                    if sender.is_none() && deferred_requests.is_empty() {
+                        match rx.try_recv() {
+                            Ok(VideoDecoderMessage::GetFrame(time, distance, reply)) => {
+                                let frame = super::request_frame(time, fps);
+                                if let Some(cached) =
+                                    cached_frame_for(&cache, last_decoded_frame, frame)
+                                {
+                                    let _ =
+                                        reply.send(cached.to_decoded_frame().with_matrix(matrix));
+                                    readahead_target =
+                                        readahead_target.max(frame + readahead_frames);
+                                    cache_min = frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2);
+                                    cache_max = frame + FRAME_CACHE_SIZE as u32 / 2;
+                                    batch_start = Instant::now();
+                                    continue;
+                                }
+                                carried =
+                                    Some(VideoDecoderMessage::GetFrame(time, distance, reply));
+                                break;
+                            }
+                            Err(mpsc::TryRecvError::Empty) => {}
+                            Err(mpsc::TryRecvError::Disconnected) => break,
+                        }
+                    }
+
                     let decode_start = Instant::now();
                     match decoder.read_sample() {
                         Ok(Some(mf_frame)) => {
@@ -451,18 +481,14 @@ impl MFDecoder {
                                 last_valid_frame = Some(cached);
                             }
 
-                            if frame_number >= requested_frame {
-                                let frame_to_send = if frame_number == requested_frame {
-                                    cache.get(&requested_frame)
-                                } else {
-                                    last_valid_frame.as_ref().or_else(|| {
-                                        cache
-                                            .range(..=requested_frame)
-                                            .next_back()
-                                            .map(|(_, f)| f)
-                                            .or_else(|| cache.get(&frame_number))
-                                    })
-                                };
+                            if frame_number > requested_frame {
+                                let frame_to_send = last_valid_frame.as_ref().or_else(|| {
+                                    cache
+                                        .range(..=requested_frame)
+                                        .next_back()
+                                        .map(|(_, f)| f)
+                                        .or_else(|| cache.get(&frame_number))
+                                });
 
                                 if let Some(frame) = frame_to_send
                                     && let Some(s) = sender.take()
@@ -471,7 +497,6 @@ impl MFDecoder {
                                 }
                             }
 
-                            let readahead_target = requested_frame + readahead_frames;
                             if frame_number >= readahead_target || frame_number > cache_max {
                                 break;
                             }
@@ -549,21 +574,22 @@ fn pts_100ns_to_frame(pts_100ns: i64, fps: u32) -> u32 {
     super::pts_to_frame(pts_100ns, ::ffmpeg::Rational::new(1, 10_000_000), fps)
 }
 
-// A frame the decoder has not reached yet has to be decoded; answering with
-// the cached frame before it shows a stale image at every readahead boundary.
-// Once the decoder is past a missing frame, the stream has no frame there, and
-// the latest earlier one is the hold the video really shows.
+// A frame the decoder has not moved past yet has to be decoded: answering with
+// the cached frame before it shows a stale image at every readahead boundary,
+// and a VFR stream can still have a later sample rounding to the same frame
+// (which replaces it in the cache), so answering early would make the choice
+// depend on request timing. Once the decoder is past a missing frame, the
+// stream has no frame there, and the latest earlier one is the hold the video
+// really shows.
 fn cached_frame_for<T>(
     cache: &BTreeMap<u32, T>,
     last_decoded: Option<u32>,
     frame: u32,
 ) -> Option<&T> {
-    cache.get(&frame).or_else(|| {
-        last_decoded
-            .is_some_and(|last| last > frame)
-            .then(|| cache.range(..=frame).next_back().map(|(_, cached)| cached))
-            .flatten()
-    })
+    last_decoded
+        .is_some_and(|last| last > frame)
+        .then(|| cache.range(..=frame).next_back().map(|(_, cached)| cached))
+        .flatten()
 }
 
 #[cfg(test)]
@@ -581,11 +607,18 @@ mod tests {
     #[test]
     fn an_undecoded_frame_is_not_answered_with_the_one_before_it() {
         let cache = BTreeMap::from([(14, 'a'), (15, 'b')]);
-        assert_eq!(cached_frame_for(&cache, Some(15), 15), Some(&'b'));
         assert_eq!(cached_frame_for(&cache, Some(15), 16), None);
         assert_eq!(cached_frame_for(&cache, Some(15), 17), None);
     }
 
+    #[test]
+    fn the_newest_decoded_frame_waits_for_a_later_sample() {
+        let cache = BTreeMap::from([(14, 'a'), (15, 'b')]);
+        assert_eq!(cached_frame_for(&cache, Some(15), 14), Some(&'a'));
+        assert_eq!(cached_frame_for(&cache, Some(15), 15), None);
+        assert_eq!(cached_frame_for(&cache, Some(16), 15), Some(&'b'));
+    }
+
     #[test]
     fn a_frame_the_stream_skipped_holds_the_previous_one() {
         let cache = BTreeMap::from([(14, 'a'), (17, 'b')]);