diff --git a/Cargo.lock b/Cargo.lock index 64f83487158..e0841a3dd64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1695,6 +1695,7 @@ dependencies = [ "cap-frame-converter", "cap-media-info", "ffmpeg-next", + "libc", "serde", "serde_json", "sysinfo 0.32.1", diff --git a/crates/enc-ffmpeg/Cargo.toml b/crates/enc-ffmpeg/Cargo.toml index 878134e909c..3cadf132da2 100644 --- a/crates/enc-ffmpeg/Cargo.toml +++ b/crates/enc-ffmpeg/Cargo.toml @@ -21,5 +21,8 @@ windows = { workspace = true, features = ["Win32_Storage_FileSystem"] } [dev-dependencies] tempfile = "3.20.0" +[target.'cfg(target_os = "macos")'.dev-dependencies] +libc = "0.2" + [lints] workspace = true diff --git a/crates/enc-ffmpeg/examples/recording-audio-benchmark.rs b/crates/enc-ffmpeg/examples/recording-audio-benchmark.rs new file mode 100644 index 00000000000..40473a05f8c --- /dev/null +++ b/crates/enc-ffmpeg/examples/recording-audio-benchmark.rs @@ -0,0 +1,216 @@ +#[cfg(target_os = "macos")] +mod benchmark { + use std::{ + path::{Path, PathBuf}, + time::{Duration, Instant}, + }; + + use cap_enc_ffmpeg::fragmented_audio::FragmentedAudioFile; + use cap_media_info::AudioInfo; + use ffmpeg::{ + ChannelLayout, + codec::context, + format::{self, Sample, sample::Type}, + frame, media, software, + }; + + const RATE: u32 = 48_000; + const FRAME_SAMPLES: usize = 960; + + fn cpu_secs() -> f64 { + let mut usage = std::mem::MaybeUninit::::zeroed(); + unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) }; + let usage = unsafe { usage.assume_init() }; + let secs = |time: libc::timeval| time.tv_sec as f64 + time.tv_usec as f64 / 1_000_000.0; + secs(usage.ru_utime) + secs(usage.ru_stime) + } + + fn decode_to_mono(path: &Path) -> Vec { + let mut input = format::input(path).unwrap(); + let stream = input.streams().best(media::Type::Audio).unwrap(); + let stream_index = stream.index(); + let mut decoder = context::Context::from_parameters(stream.parameters()) + .unwrap() + .decoder() + .audio() + .unwrap(); + let input_layout = if decoder.channel_layout().is_empty() { + AudioInfo::new_raw(decoder.format(), decoder.rate(), decoder.channels()) + .channel_layout() + } else { + decoder.channel_layout() + }; + let mut resampler = software::resampler( + (decoder.format(), input_layout, decoder.rate()), + (Sample::F32(Type::Packed), ChannelLayout::MONO, RATE), + ) + .unwrap(); + + let mut samples = Vec::new(); + let mut decoded = frame::Audio::empty(); + let mut resampled = frame::Audio::empty(); + for (stream, packet) in input.packets() { + if stream.index() != stream_index { + continue; + } + decoder.send_packet(&packet).unwrap(); + while decoder.receive_frame(&mut decoded).is_ok() { + decoded.set_channel_layout(input_layout); + resampler.run(&decoded, &mut resampled).unwrap(); + samples.extend_from_slice(resampled.plane::(0)); + } + } + decoder.send_eof().unwrap(); + while decoder.receive_frame(&mut decoded).is_ok() { + decoded.set_channel_layout(input_layout); + resampler.run(&decoded, &mut resampled).unwrap(); + samples.extend_from_slice(resampled.plane::(0)); + } + while resampler.delay().is_some() { + resampler.flush(&mut resampled).unwrap(); + if resampled.samples() == 0 { + break; + } + samples.extend_from_slice(resampled.plane::(0)); + } + samples + } + + fn synthetic(seconds: usize) -> Vec { + let mut state = 0x9e37_79b9_u32; + let mut low = 0.0_f32; + (0..seconds * RATE as usize) + .map(|index| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + let white = state as f32 / u32::MAX as f32 - 0.5; + low = 0.98f32.mul_add(low, 0.02 * white); + let time = index as f32 / RATE as f32; + 0.15 * (std::f32::consts::TAU * 220.0 * time).sin() + 0.4 * low + }) + .collect() + } + + fn encode_run( + path: &Path, + channels: u16, + mono: &[f32], + prefer_audiotoolbox: bool, + ) -> (f64, f64, u64) { + let info = AudioInfo::new_raw(Sample::F32(Type::Packed), RATE, channels); + let cpu_started = cpu_secs(); + let wall_started = Instant::now(); + let mut file = FragmentedAudioFile::init_with_encoder_preference( + path.into(), + info, + prefer_audiotoolbox, + ) + .unwrap(); + let expected = if prefer_audiotoolbox { "aac_at" } else { "aac" }; + assert_eq!( + file.encoder().codec_name().as_deref(), + Some(expected), + "requested {expected} but the encoder opened {:?} (aac_at init fell back)", + file.encoder().codec_name() + ); + for (index, chunk) in mono.chunks(FRAME_SAMPLES).enumerate() { + let mut frame = frame::Audio::new( + Sample::F32(Type::Packed), + chunk.len(), + info.channel_layout(), + ); + frame.set_rate(RATE); + for (i, bytes) in frame.data_mut(0)[..chunk.len() * usize::from(channels) * 4] + .chunks_exact_mut(4) + .enumerate() + { + bytes.copy_from_slice(&chunk[i / usize::from(channels)].to_le_bytes()); + } + file.queue_frame( + frame, + Duration::from_secs_f64((index * FRAME_SAMPLES) as f64 / f64::from(RATE)), + ) + .unwrap(); + } + file.finish().unwrap().unwrap(); + let wall = wall_started.elapsed().as_secs_f64(); + let cpu = cpu_secs() - cpu_started; + let bytes = std::fs::metadata(path).map(|meta| meta.len()).unwrap_or(0); + (cpu, wall, bytes) + } + + fn median(values: &[f64]) -> f64 { + let mut sorted = values.to_vec(); + sorted.sort_by(f64::total_cmp); + sorted[sorted.len() / 2] + } + + pub fn run() { + let mut arguments = std::env::args().skip(1); + let input = arguments.next().map(PathBuf::from); + let rounds = arguments + .next() + .and_then(|value| value.parse::().ok()) + .unwrap_or(5) + .max(1); + + let mono = input + .as_deref() + .map(decode_to_mono) + .unwrap_or_else(|| synthetic(60)); + let audio_secs = mono.len() as f64 / f64::from(RATE); + println!( + "input: {} ({audio_secs:.1}s of 48 kHz mono), {rounds} rounds", + input + .map(|p| p.display().to_string()) + .unwrap_or_else(|| "synthetic".to_string()) + ); + + let dir = std::env::temp_dir().join("cap-recording-audio-benchmark"); + std::fs::create_dir_all(&dir).unwrap(); + + let configs = [ + ("aac", 1u16, false), + ("aac_at", 1, true), + ("aac", 2, false), + ("aac_at", 2, true), + ]; + let mut runs = vec![(Vec::::new(), Vec::::new(), 0u64); configs.len()]; + for round in 0..rounds { + for (index, &(name, channels, prefer)) in configs.iter().enumerate() { + let path = dir.join(format!("{name}-{channels}ch-{round}.m4a")); + let _ = std::fs::remove_file(&path); + let (cpu, wall, bytes) = encode_run(&path, channels, &mono, prefer); + runs[index].0.push(cpu); + runs[index].1.push(wall); + runs[index].2 = bytes; + } + } + + println!("encoder ch cpu_ms/s wall_s kbps"); + for (index, &(name, channels, _)) in configs.iter().enumerate() { + let (cpus, walls, bytes) = &runs[index]; + let cpu_ms = median(cpus) * 1_000.0 / audio_secs; + let kbps = *bytes as f64 * 8.0 / audio_secs / 1_000.0; + println!( + "{name:<8} {channels} {cpu_ms:>8.1} {:>6.2} {kbps:>6.1}", + median(walls) + ); + } + for channels in [1usize, 2] { + let aac = median(&runs[channels * 2 - 2].0); + let at = median(&runs[channels * 2 - 1].0); + println!("cpu ratio aac_at/aac ({channels}ch): {:.2}", at / aac); + } + } +} + +fn main() { + #[cfg(target_os = "macos")] + benchmark::run(); + #[cfg(not(target_os = "macos"))] + eprintln!( + "recording-audio-benchmark compares FFmpeg aac with AudioToolbox aac_at and only runs on macOS" + ); +} diff --git a/crates/enc-ffmpeg/src/audio/aac.rs b/crates/enc-ffmpeg/src/audio/aac.rs index 89ce84fcd3a..ca58b26641e 100644 --- a/crates/enc-ffmpeg/src/audio/aac.rs +++ b/crates/enc-ffmpeg/src/audio/aac.rs @@ -33,6 +33,7 @@ pub struct AACEncoder { impl AACEncoder { const OUTPUT_BITRATE: usize = 320 * 1000; // 128k const SAMPLE_FORMAT: Sample = Sample::F32(Type::Planar); + const FFMPEG_AAC_PRIMING: i32 = 1_024; pub fn factory( input_config: AudioInfo, @@ -44,7 +45,57 @@ impl AACEncoder { input_config: AudioInfo, output: &mut format::context::Output, ) -> Result { - let codec = encoder::find_by_name("aac").ok_or(AACEncoderError::CodecNotFound)?; + Self::init_with_codec( + "aac", + ffmpeg::Dictionary::new(), + Self::OUTPUT_BITRATE, + input_config, + output, + ) + } + + pub fn init_for_recording( + input_config: AudioInfo, + output: &mut format::context::Output, + ) -> Result { + let prefer_audiotoolbox = cfg!(target_os = "macos") + && std::env::var_os("CAP_RECORDING_FFMPEG_AAC").is_none_or(|value| value.is_empty()); + Self::init_for_recording_with(prefer_audiotoolbox, input_config, output) + } + + pub(crate) fn init_for_recording_with( + prefer_audiotoolbox: bool, + input_config: AudioInfo, + output: &mut format::context::Output, + ) -> Result { + if prefer_audiotoolbox { + let mut options = ffmpeg::Dictionary::new(); + options.set("aac_at_mode", "abr"); + let bit_rate = if input_config.channels == 1 { + 192_000 + } else { + 256_000 + }; + match Self::init_with_codec("aac_at", options, bit_rate, input_config, output) { + Ok(encoder) => return Ok(encoder), + Err(error) => { + tracing::warn!(%error, "AudioToolbox AAC encoder unavailable, using FFmpeg's"); + } + } + } + + Self::init(input_config, output) + } + + fn init_with_codec( + name: &str, + options: ffmpeg::Dictionary, + bit_rate: usize, + input_config: AudioInfo, + output: &mut format::context::Output, + ) -> Result { + let codec = encoder::find_by_name(name).ok_or(AACEncoderError::CodecNotFound)?; + let audio_codec = codec.audio()?; let mut encoder_ctx = context::Context::new_with_codec(codec); let thread_count = thread::available_parallelism() .map(|v| v.get()) @@ -53,39 +104,62 @@ impl AACEncoder { let mut encoder = encoder_ctx.encoder().audio()?; let rate = { - let mut rates = codec - .audio() - .unwrap() + let mut rates = audio_codec .rates() .into_iter() .flatten() .collect::>(); rates.sort(); - let Some(&rate) = rates - .iter() - .find(|r| **r >= input_config.rate()) - .or(rates.first()) - else { - return Err(AACEncoderError::RateNotSupported(input_config.rate())); - }; - rate + if rates.is_empty() { + input_config.rate() + } else { + let Some(&rate) = rates + .iter() + .find(|r| **r >= input_config.rate()) + .or(rates.first()) + else { + return Err(AACEncoderError::RateNotSupported(input_config.rate())); + }; + rate + } + }; + + let sample_format = { + let formats = audio_codec + .formats() + .into_iter() + .flatten() + .collect::>(); + if formats.contains(&Self::SAMPLE_FORMAT) { + Self::SAMPLE_FORMAT + } else { + formats.first().copied().unwrap_or(Self::SAMPLE_FORMAT) + } }; let mut output_config = input_config; - output_config.sample_format = Self::SAMPLE_FORMAT; + output_config.sample_format = sample_format; output_config.sample_rate = rate as u32; let resampler = BufferedResampler::new(input_config, output_config) .map_err(AACEncoderError::Resampler)?; - encoder.set_bit_rate(Self::OUTPUT_BITRATE); + encoder.set_bit_rate(bit_rate); encoder.set_rate(rate); encoder.set_format(output_config.sample_format); encoder.set_channel_layout(output_config.channel_layout()); encoder.set_time_base(FFRational(1, output_config.rate())); - let encoder = encoder.open()?; + let encoder = encoder.open_with(options)?; + + // Fragmented outputs keep encoder priming in the decoded timeline; + // dropping the priming a codec adds beyond FFmpeg's aac (e.g. 2112 vs + // 1024 samples for aac_at) keeps every content sample where the native + // encoder put it. + let leading_trim = unsafe { (*encoder.as_ptr()).initial_padding } + .saturating_sub(Self::FFMPEG_AAC_PRIMING) + .max(0) as usize; let mut output_stream = output.add_stream(codec)?; output_stream.set_time_base(FFRational(1, output_config.rate())); @@ -93,7 +167,8 @@ impl AACEncoder { let stream_index = output_stream.index(); Ok(Self { - base: AudioEncoderBase::new(encoder, resampler, stream_index), + base: AudioEncoderBase::new(encoder, resampler, stream_index) + .with_leading_trim(leading_trim), stream_index, }) } @@ -102,6 +177,10 @@ impl AACEncoder { self.stream_index } + pub fn codec_name(&self) -> Option { + self.base.codec_name() + } + pub fn send_frame( &mut self, frame: frame::Audio, diff --git a/crates/enc-ffmpeg/src/audio/base.rs b/crates/enc-ffmpeg/src/audio/base.rs index cd5197a7c71..fbfdbb72fae 100644 --- a/crates/enc-ffmpeg/src/audio/base.rs +++ b/crates/enc-ffmpeg/src/audio/base.rs @@ -1,12 +1,18 @@ use super::buffered_resampler::BufferedResampler; use crate::base::{EncoderBase, PacketSink}; -use ffmpeg::{codec::encoder, frame}; +use ffmpeg::{ + Packet, Rational, Rescale, + codec::{Capabilities, encoder}, + frame, +}; use std::time::Duration; pub struct AudioEncoderBase { inner: EncoderBase, encoder: encoder::Audio, resampler: BufferedResampler, + leading_trim: usize, + input_end: Option, } impl AudioEncoderBase { @@ -15,9 +21,20 @@ impl AudioEncoderBase { inner: EncoderBase::new(stream_index), encoder, resampler, + leading_trim: 0, + input_end: None, } } + pub fn with_leading_trim(mut self, samples: usize) -> Self { + self.leading_trim = samples; + self + } + + pub fn codec_name(&self) -> Option { + self.encoder.codec().map(|codec| codec.name().to_owned()) + } + pub fn send_frame( &mut self, mut frame: frame::Audio, @@ -32,7 +49,15 @@ impl AudioEncoderBase { self.resampler.add_frame(frame); + if self.leading_trim > 0 { + match self.resampler.get_frame(self.leading_trim) { + Some(_) => self.leading_trim = 0, + None => return Ok(()), + } + } + while let Some(frame) = self.resampler.get_frame(self.encoder.frame_size() as usize) { + self.track_input_end(&frame); self.inner.send_frame(&frame, output, &mut self.encoder)?; } @@ -40,10 +65,72 @@ impl AudioEncoderBase { } pub fn flush(&mut self, output: &mut S) -> Result<(), ffmpeg::Error> { + while self.leading_trim > 0 { + match self.resampler.flush(self.leading_trim) { + Some(frame) => self.leading_trim -= frame.samples(), + None => break, + } + } + + let mut frames = Vec::new(); while let Some(frame) = self.resampler.flush(self.encoder.frame_size() as usize) { - self.inner.send_frame(&frame, output, &mut self.encoder)?; + self.track_input_end(&frame); + frames.push(frame); + } + + // Encoders that only take full frames (AudioToolbox) get the last one + // padded by libavcodec, and their packet durations count that padding, + // so the track would end up to a frame after the input did. + let pads_final_frame = self.encoder.codec().is_some_and(|codec| { + !codec + .capabilities() + .intersects(Capabilities::SMALL_LAST_FRAME | Capabilities::VARIABLE_FRAME_SIZE) + }); + let mut output = EndTrimmedSink { + output, + end: self.input_end.filter(|_| pads_final_frame), + time_base: self.encoder.time_base(), + }; + for frame in &frames { + self.inner + .send_frame(frame, &mut output, &mut self.encoder)?; } - self.inner.process_eof(output, &mut self.encoder) + self.inner.process_eof(&mut output, &mut self.encoder) + } + + fn track_input_end(&mut self, frame: &frame::Audio) { + if let Some(pts) = frame.pts() { + let end = pts + frame.samples() as i64; + self.input_end = Some(self.input_end.map_or(end, |current| current.max(end))); + } + } +} + +struct EndTrimmedSink<'a, S: PacketSink + ?Sized> { + output: &'a mut S, + end: Option, + time_base: Rational, +} + +impl PacketSink for EndTrimmedSink<'_, S> { + fn stream_time_base(&self, stream_index: usize) -> Rational { + self.output.stream_time_base(stream_index) + } + + fn write_packet(&mut self, mut packet: Packet) -> Result<(), ffmpeg::Error> { + if let (Some(end), Some(pts)) = (self.end, packet.pts()) { + let end = end.rescale( + self.time_base, + self.output.stream_time_base(packet.stream()), + ); + if pts >= end { + return Ok(()); + } + if pts + packet.duration() > end { + packet.set_duration(end - pts); + } + } + self.output.write_packet(packet) } } diff --git a/crates/enc-ffmpeg/src/mux/fragmented_audio.rs b/crates/enc-ffmpeg/src/mux/fragmented_audio.rs index 1203ad72ce4..cf865c77c56 100644 --- a/crates/enc-ffmpeg/src/mux/fragmented_audio.rs +++ b/crates/enc-ffmpeg/src/mux/fragmented_audio.rs @@ -64,7 +64,29 @@ fn configure_fragmentation(output: &mut format::context::Output) -> Result<(), f } impl FragmentedAudioFile { - pub fn init(mut output_path: PathBuf, audio_config: AudioInfo) -> Result { + pub fn init(output_path: PathBuf, audio_config: AudioInfo) -> Result { + Self::init_inner(output_path, audio_config, AACEncoder::init_for_recording) + } + + #[doc(hidden)] + pub fn init_with_encoder_preference( + output_path: PathBuf, + audio_config: AudioInfo, + prefer_audiotoolbox: bool, + ) -> Result { + Self::init_inner(output_path, audio_config, |config, output| { + AACEncoder::init_for_recording_with(prefer_audiotoolbox, config, output) + }) + } + + fn init_inner( + mut output_path: PathBuf, + audio_config: AudioInfo, + encoder_init: impl FnOnce( + AudioInfo, + &mut format::context::Output, + ) -> Result, + ) -> Result { output_path.set_extension("m4a"); if let Some(parent) = output_path.parent() { @@ -75,7 +97,7 @@ impl FragmentedAudioFile { configure_fragmentation(&mut output)?; - let encoder = AACEncoder::init(audio_config, &mut output)?; + let encoder = encoder_init(audio_config, &mut output)?; output.write_header()?; @@ -324,3 +346,300 @@ mod publication_tests { assert!(path.exists()); } } + +#[cfg(all(test, target_os = "macos"))] +mod recording_encoder_tests { + use super::*; + use ffmpeg::{ + Rescale, + codec::context, + format::{Sample, sample::Type}, + }; + use std::path::Path; + + const RATE: u32 = 48_000; + const FRAME_SAMPLES: usize = 960; + const CLICK_SAMPLES: usize = 1; + const FFMPEG_AAC_PRIMING: usize = 1_024; + + fn timeline_with_clicks(total: usize, clicks: &[usize]) -> Vec { + let mut timeline = vec![0.0_f32; total]; + for &click in clicks { + for sample in timeline + .iter_mut() + .take((click + CLICK_SAMPLES).min(total)) + .skip(click) + { + *sample = 0.9; + } + } + timeline + } + + fn encode_recording( + prefer_audiotoolbox: bool, + channels: u16, + timeline: &[f32], + send_ranges: &[(usize, usize)], + ) -> (Vec, i64, u64) { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("audio.m4a"); + let info = AudioInfo::new_raw(Sample::F32(Type::Packed), RATE, channels); + let mut file = FragmentedAudioFile::init_with_encoder_preference( + path.clone(), + info, + prefer_audiotoolbox, + ) + .unwrap(); + assert_eq!( + file.encoder().codec_name().as_deref(), + Some(if prefer_audiotoolbox { "aac_at" } else { "aac" }), + "requested AudioToolbox={prefer_audiotoolbox}, but opened {:?}", + file.encoder().codec_name() + ); + + for &(start, len) in send_ranges { + let mut position = start; + while position < start + len { + let samples = FRAME_SAMPLES.min(start + len - position); + let mut frame = + frame::Audio::new(Sample::F32(Type::Packed), samples, info.channel_layout()); + frame.set_rate(RATE); + for (index, bytes) in frame.data_mut(0) + [..samples * usize::from(channels) * size_of::()] + .chunks_exact_mut(size_of::()) + .enumerate() + { + bytes.copy_from_slice( + &timeline[position + index / usize::from(channels)].to_le_bytes(), + ); + } + file.queue_frame( + frame, + Duration::from_secs_f64(position as f64 / f64::from(RATE)), + ) + .unwrap(); + position += samples; + } + } + + file.finish().unwrap().unwrap(); + let bytes = std::fs::metadata(&path).map(|meta| meta.len()).unwrap_or(0); + let (samples, track_end) = decode_channel_zero(&path); + (samples, track_end, bytes) + } + + fn encode_recording_variable( + prefer_audiotoolbox: bool, + channels: u16, + timeline: &[f32], + frame_sizes: &[usize], + ) -> (Vec, i64, u64) { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("audio.m4a"); + let info = AudioInfo::new_raw(Sample::F32(Type::Packed), RATE, channels); + let mut file = FragmentedAudioFile::init_with_encoder_preference( + path.clone(), + info, + prefer_audiotoolbox, + ) + .unwrap(); + assert_eq!( + file.encoder().codec_name().as_deref(), + Some(if prefer_audiotoolbox { "aac_at" } else { "aac" }), + "requested AudioToolbox={prefer_audiotoolbox}, but opened {:?}", + file.encoder().codec_name() + ); + + let mut position = 0; + let mut frame_index = 0; + while position < timeline.len() { + let samples = + frame_sizes[frame_index % frame_sizes.len()].min(timeline.len() - position); + let mut frame = + frame::Audio::new(Sample::F32(Type::Packed), samples, info.channel_layout()); + frame.set_rate(RATE); + for (index, bytes) in frame.data_mut(0) + [..samples * usize::from(channels) * size_of::()] + .chunks_exact_mut(size_of::()) + .enumerate() + { + bytes.copy_from_slice( + &timeline[position + index / usize::from(channels)].to_le_bytes(), + ); + } + file.queue_frame( + frame, + Duration::from_nanos(position as u64 * 1_000_000_000 / u64::from(RATE)), + ) + .unwrap(); + position += samples; + frame_index += 1; + } + + file.finish().unwrap().unwrap(); + let bytes = std::fs::metadata(&path).map(|meta| meta.len()).unwrap_or(0); + let (samples, track_end) = decode_channel_zero(&path); + (samples, track_end, bytes) + } + + fn decode_channel_zero(path: &Path) -> (Vec, i64) { + let mut input = format::input(path).unwrap(); + let stream = input.streams().best(ffmpeg::media::Type::Audio).unwrap(); + let stream_index = stream.index(); + let time_base = stream.time_base(); + let mut decoder = context::Context::from_parameters(stream.parameters()) + .unwrap() + .decoder() + .audio() + .unwrap(); + let mut decoded = frame::Audio::empty(); + let mut samples = Vec::new(); + let mut track_end = i64::MIN; + + for (stream, packet) in input.packets() { + if stream.index() != stream_index { + continue; + } + if let Some(pts) = packet.pts() { + track_end = + track_end.max((pts + packet.duration()).rescale(time_base, (1, 48_000))); + } + decoder.send_packet(&packet).unwrap(); + while decoder.receive_frame(&mut decoded).is_ok() { + assert_eq!(decoded.format(), Sample::F32(Type::Planar)); + samples.extend_from_slice(decoded.plane::(0)); + } + } + decoder.send_eof().unwrap(); + while decoder.receive_frame(&mut decoded).is_ok() { + assert_eq!(decoded.format(), Sample::F32(Type::Planar)); + samples.extend_from_slice(decoded.plane::(0)); + } + (samples, track_end) + } + + fn click_index(samples: &[f32]) -> usize { + samples + .iter() + .enumerate() + .max_by(|(_, a), (_, b)| a.abs().partial_cmp(&b.abs()).unwrap()) + .map(|(index, _)| index) + .expect("no decoded samples") + } + + fn click_positions(samples: &[f32]) -> Vec { + let mut positions = Vec::new(); + let mut index = 0; + while index < samples.len() { + if samples[index].abs() > 0.2 { + let window_end = (index + 512).min(samples.len()); + positions.push(index + click_index(&samples[index..window_end])); + index = window_end; + } else { + index += 1; + } + } + positions + } + + #[test] + fn recording_audio_click_decodes_at_the_same_sample_with_both_encoders() { + for channels in [1u16, 2] { + let timeline = timeline_with_clicks(96_000, &[24_000]); + let ranges = [(0usize, 96_000usize)]; + + let (ffmpeg_samples, _, _) = encode_recording(false, channels, &timeline, &ranges); + let ffmpeg_click = click_index(&ffmpeg_samples); + assert_eq!( + ffmpeg_click, + 24_000 + FFMPEG_AAC_PRIMING, + "FFmpeg aac decoded the click at {ffmpeg_click} for {channels}ch" + ); + + let (at_samples, _, _) = encode_recording(true, channels, &timeline, &ranges); + let at_click = click_index(&at_samples); + assert!( + at_click.abs_diff(ffmpeg_click) <= 4, + "AudioToolbox decoded the click at {at_click}, FFmpeg at {ffmpeg_click} ({channels}ch)" + ); + } + } + + #[test] + fn recording_audio_silence_filled_gap_decodes_the_same_with_both_encoders() { + let frame_sizes = [441, 960, 1000, 512, 4800]; + for channels in [1u16, 2] { + let timeline = timeline_with_clicks(168_000, &[24_000, 120_000]); + + let (ffmpeg_samples, ffmpeg_end, _) = + encode_recording_variable(false, channels, &timeline, &frame_sizes); + let (at_samples, at_end, _) = + encode_recording_variable(true, channels, &timeline, &frame_sizes); + + let ffmpeg_clicks = click_positions(&ffmpeg_samples); + let at_clicks = click_positions(&at_samples); + assert_eq!(ffmpeg_clicks.len(), 2, "FFmpeg clicks: {ffmpeg_clicks:?}"); + assert_eq!(at_clicks.len(), 2, "AudioToolbox clicks: {at_clicks:?}"); + for (index, (&ffmpeg_click, &at_click)) in + ffmpeg_clicks.iter().zip(&at_clicks).enumerate() + { + let expected = [24_000usize, 120_000][index] + FFMPEG_AAC_PRIMING; + assert_eq!( + ffmpeg_click, expected, + "FFmpeg aac decoded click {index} at {ffmpeg_click}, expected {expected} ({channels}ch)" + ); + assert!( + at_click.abs_diff(ffmpeg_click) <= 4, + "click {index}: AudioToolbox {at_click} vs FFmpeg {ffmpeg_click} ({channels}ch)" + ); + } + assert!( + at_samples.len().abs_diff(ffmpeg_samples.len()) <= 4, + "decoded {} vs {} samples ({channels}ch)", + at_samples.len(), + ffmpeg_samples.len() + ); + assert!( + at_end.abs_diff(ffmpeg_end) <= 1, + "track end {at_end} vs {ffmpeg_end} ({channels}ch)" + ); + } + } + + #[test] + fn recording_audio_track_ends_where_the_ffmpeg_track_ends() { + let timeline = vec![0.0_f32; 47_123]; + let ranges = [(0usize, 47_123usize)]; + + let (_, ffmpeg_end, _) = encode_recording(false, 1, &timeline, &ranges); + let (_, at_end, _) = encode_recording(true, 1, &timeline, &ranges); + + assert!( + at_end.abs_diff(ffmpeg_end) <= 1, + "track end {at_end} vs {ffmpeg_end}" + ); + } + + #[test] + fn recording_audio_uses_audiotoolbox_on_macos() { + if std::env::var_os("CAP_RECORDING_FFMPEG_AAC").is_some_and(|v| !v.is_empty()) { + return; + } + let directory = tempfile::tempdir().unwrap(); + let info = AudioInfo::new_raw(Sample::F32(Type::Packed), RATE, 1); + let file = FragmentedAudioFile::init(directory.path().join("audio.m4a"), info).unwrap(); + assert_eq!(file.encoder().codec_name().as_deref(), Some("aac_at")); + } + + #[test] + fn recording_audio_silence_stays_small() { + let timeline = vec![0.0_f32; 480_000]; + let ranges = [(0usize, 480_000usize)]; + let (_, _, bytes) = encode_recording(true, 2, &timeline, &ranges); + assert!( + bytes < 64 * 1024, + "10s of stereo silence took {bytes} bytes" + ); + } +}