diff --git a/Cargo.lock b/Cargo.lock
index 312b8df..68fb70c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8,6 +8,8 @@ version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3b7f7f85a7e5f68090000ed7622545829afd484d210358702ae4cb97dd0c320"
dependencies = [
+ "enumn",
+ "serde",
"uuid",
]
@@ -26,6 +28,7 @@ dependencies = [
"cfg-if",
"getrandom 0.3.4",
"once_cell",
+ "serde",
"version_check",
"zerocopy",
]
@@ -606,6 +609,9 @@ name = "bitflags"
version = "2.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
+dependencies = [
+ "serde_core",
+]
[[package]]
name = "bitstream-io"
@@ -1843,6 +1849,7 @@ checksum = "0656a7530b9e2841ee7a5f983965bdd9840c32f4012f759cfd44b912cced6b34"
dependencies = [
"bytemuck",
"emath",
+ "serde",
]
[[package]]
@@ -1871,6 +1878,8 @@ dependencies = [
"pollster 1.0.1",
"profiling",
"raw-window-handle",
+ "ron",
+ "serde",
"static_assertions",
"wasm-bindgen",
"web-sys",
@@ -1895,6 +1904,8 @@ dependencies = [
"log",
"nohash-hasher",
"profiling",
+ "ron",
+ "serde",
"smallvec",
"unicode-segmentation",
"web-sys",
@@ -1935,6 +1946,7 @@ dependencies = [
"objc2-ui-kit 0.3.2",
"profiling",
"raw-window-handle",
+ "serde",
"smithay-clipboard",
"web-time",
"winit",
@@ -1979,6 +1991,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52c8d4141cf0f60fb2f7aefaf352d3ffa5171ef0ab09731c64391801e95e48f1"
dependencies = [
"bytemuck",
+ "serde",
]
[[package]]
@@ -2040,6 +2053,17 @@ dependencies = [
"syn 2.0.119",
]
+[[package]]
+name = "enumn"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f9ed6b3789237c8a0c1c505af1c7eb2c560df6186f01b098c3a1064ea532f38"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
[[package]]
name = "epaint"
version = "0.36.2"
@@ -2058,6 +2082,7 @@ dependencies = [
"parking_lot",
"profiling",
"self_cell",
+ "serde",
"skrifa",
"smallvec",
"unicode-general-category",
@@ -2226,6 +2251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "835ae9d781464a79026cde387f9cbc650c9a4909d7cf7e658b127e3c35d2f35d"
dependencies = [
"bytemuck",
+ "serde",
]
[[package]]
@@ -6228,6 +6254,20 @@ dependencies = [
"portable-atomic-util",
]
+[[package]]
+name = "ron"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81116b9531d61eabc41aeb228e4b6b2435bcca3233b98cf3b3077d4e6e9debb3"
+dependencies = [
+ "bitflags 2.13.2",
+ "once_cell",
+ "serde",
+ "serde_derive",
+ "typeid",
+ "unicode-ident",
+]
+
[[package]]
name = "roxmltree"
version = "0.21.1"
@@ -6861,6 +6901,9 @@ name = "smallvec"
version = "1.16.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b"
+dependencies = [
+ "serde",
+]
[[package]]
name = "smithay-client-toolkit"
@@ -7850,6 +7893,12 @@ dependencies = [
"rustc-hash 2.1.3",
]
+[[package]]
+name = "typeid"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c"
+
[[package]]
name = "typenum"
version = "1.20.1"
diff --git a/README.md b/README.md
index 3584705..a94fa40 100644
--- a/README.md
+++ b/README.md
@@ -30,8 +30,8 @@ cargo run --release -p moq-studio --features capture,openh264 -- \
the first relay only. The stream's tracks claim their bitrate on the first
relay's link alone, so only that link is probed toward the stream's rate;
no encoder adapts to either estimate.
-- `capture` adds camera, display, window, microphone, and system-audio sources.
- On Linux it links ALSA; add `pipewire` for PipeWire cameras. `openh264`
+- `capture` adds camera, display, window, microphone, and system-audio
+ sources, and listening to a mix. On Linux it links ALSA; add `pipewire` for PipeWire cameras. `openh264`
is the software H.264 fallback for a machine without a hardware encoder.
- `cef` adds browser sources, on Linux only for now. The build takes CEF from
`CEF_PATH`, which the dev shell sets, or else downloads it, and the binary
@@ -98,7 +98,7 @@ Every panel is a tab you can drag, split, or stack.
| Devices | List this machine's cameras, displays, windows, and audio inputs, and add one as a source. |
| Guests | Invite a guest, show their join link as a QR code, see who joined, and take a guest out of every scene. |
| Filters | Crop the selected source or item, add, order, and set its filters, set an item's blend mode, and copy or paste filters. |
-| Audio | Set each audio source's gain and mute, and watch its meter and the program's. |
+| Audio | Add, rename (double-click), and remove mixes, pick the one on program and the one to listen to and on which output, and set each audio source's gain, mute, delay, and the mixes it feeds, with a meter on every mix and source. |
| Output | Start and stop recording, set the canvas size and frame rate, and see the broadcast path, whether the program is encoding, and the encoder's work. |
| Health | See how the stream reaches viewers, relay by relay and track by track, and the cause of each problem. |
| Replay | Replay the last moments of the program or a source in a replay source, slowed down if you like, or save them as a clip. |
@@ -151,12 +151,34 @@ broadcast through the first relay while that relay has a route to it, and
through the failover otherwise, so a publisher restarted behind only the
failover replaces the first relay's copy once that copy ends. It plays the
best video and audio its catalog lists, switches when the catalog changes, and
-comes back on its own when the publisher restarts. Audio from every source is mixed into the program whether
-or not its scene is live; mute a source to leave it out.
+comes back on its own when the publisher restarts.
+
+### Audio mixes
+
+Audio from every source is mixed whether or not its scene is live. A
+collection has one or more **mixes**, each the sum of the sources it is fed:
+for example, a stream mix that leaves out music you may not broadcast, and a
+monitor mix for your headphones. A source feeds every mix, a new one included,
+until you untick that mix in its row of the Audio panel; mute a source to take
+it out of every mix. **On program** picks the mix the broadcast carries, and
+switching it, or editing a mix, never drops or repeats audio. A recording
+holds the mix on program. The mix on program cannot be removed.
+
+**Listen** plays a mix to an audio output, from the `capture` feature's
+build: the system default, or the one picked under **Listen on**, which the
+studio remembers for you rather than in the scene file. Use headphones: a
+microphone that hears the speakers feeds the mix back into itself.
+
+A source's **delay** holds it back in every mix, up to 2 s, so a microphone
+lines up with a camera that lags it.
+
+Each mix is advanced on the program clock, and each output reads its mix on
+its own, so an output that falls behind skips ahead in its own audio rather
+than holding up the mixes or another output.
A browser source renders a web page, such as an overlay, alert box, or chat
widget, at the canvas frame rate (CEF caps it at 60) over a transparent
-background, and its sound joins the program audio. Every page shares one
+background, and its sound joins the mixes. Every page shares one
browser profile, kept in `$XDG_DATA_HOME/moq-studio/browser`
(`~/.local/share/moq-studio/browser` by default) rather than beside the scene
file, so cookies and logins, such as an alert service's, survive a reload and
@@ -334,8 +356,14 @@ html = '''
Ada Lovelace
'''
+[mixes]
+names = ["Stream", "Headphones"]
+program = "Stream" # the mix the broadcast carries
+
[audio.guest]
gain = 0.5
+exclude = ["Headphones"] # feeds every other mix
+delay = "80ms"
[sources.camera-faded]
kind = "clone"
@@ -383,6 +411,11 @@ file, which no item places, but which grades what a `lut` filter names it in),
and `replay`, where the Replay panel plays replays. The Devices panel fills in
ids. `[[top]]` items make up the top layer.
+`[mixes]` names every mix, in the order the Audio panel lists them, and the
+one on `program`; without it, the collection has one mix, `Program`.
+`[audio.]` sets a source's linear `gain`, `muted`, the mixes it is
+left out of (`exclude`), and its `delay`.
+
`[video.]` sets a source's `crop` and `filters`, wherever it is shown,
and an item's `crop`, `filters`, and `blend` its own. A `crop` trims a fraction
(0 to 1) of the picture off its `left`, `top`, `right`, and `bottom`, opposite
@@ -442,7 +475,7 @@ before sharing.
## Output
The program publishes as one H.264 rendition at the canvas size and frame rate,
-with Opus stereo audio, under the `--broadcast` path. Timestamps come from the
+with the mix on program as Opus stereo audio, under the `--broadcast` path. Timestamps come from the
studio's own clock, so a source that stalls repeats its last picture instead of
holding up the stream.
@@ -586,8 +619,9 @@ The Health panel grades the stream every second on the thresholds moq's
`hang::health` shares with the rest of the stack, and lists each problem
with its cause in plain words: a relay is unreachable, the uplink is
congested or losing packets, the relay is far away, the encoder is behind or
-stuck, a source stalled (a camera or broadcast the program draws, or an
-unmuted microphone or system audio), or viewers are late, stalling, or cannot
+stuck, a source stalled (a camera or broadcast the program draws, or a
+microphone or system audio the mix on program plays: unmuted and not left
+out of it), or viewers are late, stalling, or cannot
decode a track. A relay lost while the other serves is
degraded, since viewers have nothing left to fall back to; with none serving
it is unhealthy. Below that it shows:
diff --git a/quest/m0/README.md b/quest/m0/README.md
index 24aca04..4eea230 100644
--- a/quest/m0/README.md
+++ b/quest/m0/README.md
@@ -100,7 +100,8 @@ repository.
- [Multiple canvases](/quest/m0/multi-canvas.md) - landscape and portrait canvases composite at once from shared sources
- [The join page is live](/quest/m0/join-page-live.md) - `join.studio.moq.dev` serves the join page `--join-page` defaults to
- [Guest return feed](/quest/m0/guest-return.md) - guests see the program and hear a mix-minus
-- [Audio mixes](/quest/m0/mixes.md) - per-output mixes, meters, monitoring, and sync offsets
+- [A recording picks its mix](/quest/m0/record-mix.md) - a recording keeps what the stream's mix leaves out
+- [Clap-to-measure sync delay](/quest/m0/sync-clap.md) - a clap sets a microphone's delay against its camera
- [Audio filters](/quest/m0/audio-filters.md) - per-source noise suppression, gate, compressor, limiter, and plugins, configured from the studio
- [Browser sources on macOS and Windows](/quest/m0/browser-source.md) - the out-of-process CEF pages Linux has, with shared GPU textures on the other desktops
- [Published independently](/quest/m0/publish-independently.md) - a source opts out of the program and publishes its own track for services that place it themselves
diff --git a/quest/m0/broadcast-sync.md b/quest/m0/broadcast-sync.md
index 5f74d9f..8c1646b 100644
--- a/quest/m0/broadcast-sync.md
+++ b/quest/m0/broadcast-sync.md
@@ -12,7 +12,8 @@ can leave audio and video apart.
Decided 2026-10-26 (maintainer):
- **Scope:** broadcast sources only. Local devices keep the manual
- per-source offsets in [Audio mixes](/quest/m0/mixes.md).
+ per-source `delay`, which [Clap-to-measure](/quest/m0/sync-clap.md) sets
+ from a clap.
- **Delay:** reuse `moq play`'s timing, the `--delay auto` target from
moq-audio's jitter estimate (`rs/moq-audio/src/jitter.rs`,
`doc/concept/audio-jitter.md`), rather than a second algorithm or a manual
diff --git a/quest/m0/guest-return.md b/quest/m0/guest-return.md
index 74088b2..c963c1e 100644
--- a/quest/m0/guest-return.md
+++ b/quest/m0/guest-return.md
@@ -10,7 +10,7 @@ The join page plays it.
## Plan
Decided 2026-11-02 (maintainer): guests get a mix-minus, not the full program,
-so this waits on audio mixes; video and audio land together. Publish into the
+and video and audio land together. Publish into the
room the studio already owns, under names the room's token scope allows, and
keep it free of the studio's own credentials. A guest's token subscribes only
to its own renewals (`.renew//`), so it must also grant the feed's
@@ -19,7 +19,7 @@ with the guest count, so bound it by design (the guest roster is already
bounded to guests the studio signed for). Raised on
[#10](https://github.com/moq-dev/studio/pull/10).
-## Required
-
-- [Audio mixes](/quest/m0/mixes.md) - the per-output mixes a mix-minus is
- built from
+Audio mixes landed in [#32](https://github.com/moq-dev/studio/pull/32): build
+each guest's feed with moq-audio's `bus::Mix::minus` from the mix on program
+(`Sources::mix`), taking the guest's own strip out, which costs one
+subtraction per block rather than a mix per guest.
diff --git a/quest/m0/mixes.md b/quest/m0/mixes.md
deleted file mode 100644
index 9737d14..0000000
--- a/quest/m0/mixes.md
+++ /dev/null
@@ -1,26 +0,0 @@
-# [M] Audio mixes
-
-## Goal
-
-The program bus in moq-audio feeds several mixes. For example, a stream mix
-excludes the creator's music, a recording mix keeps it, and a monitor mix
-plays to headphones. Each source routes to any subset of mixes, and each
-output or recording picks a mix. Every source and mix shows a level meter.
-Per-source sync offsets fix cameras that lag their microphones, and a clap
-in front of the camera, seen and heard, measures the offset instead of
-guessing it.
-Each guest hears a mix-minus: the program without their own voice.
-
-## Plan
-
-Decided 2026-11-02 (maintainer): routing, mix-minus, and sync offsets live in
-moq-audio as a multi-mix router rather than one `Bus` per mix in the studio:
-`bus::Mix`, which the program audio already is. The studio adds the routing
-UI, meters, the headphone monitor, clap-to-measure offsets, and a mix pick per
-output and recording.
-
-The mixer runs on the audio clock and never blocks on a slow output. An
-output that falls behind drops its own audio, not the bus's.
-
-Benchmark source count against mix count, including sparse routing. A mix
-must cost only what is routed into it, never a scan of every source.
diff --git a/quest/m0/record-mix.md b/quest/m0/record-mix.md
new file mode 100644
index 0000000..038cdf3
--- /dev/null
+++ b/quest/m0/record-mix.md
@@ -0,0 +1,31 @@
+# [S] A recording picks its mix
+
+## Goal
+
+A recording in `moq-studio` picks any of the collection's mixes, so it can
+keep what the stream leaves out, such as the creator's music, while the
+broadcast carries the mix on program. A recording on the same mix as the
+program records as it does today.
+
+## Plan
+
+Split from [Audio mixes](https://github.com/moq-dev/studio/pull/32), which
+landed the mixes, routing, meters, delay, listening, and the program's pick
+(`[mixes] program`).
+
+A recording reads the studio's broadcast the way a viewer does
+(`rs/moq-studio/src/record.rs`), so a mix other than the program's needs an
+Opus track of its own. Keep that track off the relay: a mix the creator left
+off the stream must not be fetchable from it, as source tracks stay local
+after [Live encode control](/quest/m0/encode-control.md). Encode it only while
+a recording reads it, and record the program's video beside it in one file.
+The pick lives in the scene file beside `[mixes] program`, and the Output
+panel shows it. Update `README.md`.
+
+## Required
+
+- [Live encode control](/quest/m0/encode-control.md) - the local broadcast a mix track stays off the relay in
+
+## Related
+
+- [Destinations](/quest/m0/destinations.md) - each destination picks a mix the same way
diff --git a/quest/m0/sync-clap.md b/quest/m0/sync-clap.md
new file mode 100644
index 0000000..915f631
--- /dev/null
+++ b/quest/m0/sync-clap.md
@@ -0,0 +1,28 @@
+# [S] Clap-to-measure sync delay
+
+## Goal
+
+A clap in front of a camera, seen and heard, measures how far a microphone
+leads that camera, and sets the microphone's `delay` to it, instead of the
+creator guessing the number.
+
+## Plan
+
+Split from [Audio mixes](https://github.com/moq-dev/studio/pull/32), which
+landed the per-source `delay` the measurement sets.
+
+Measure both onsets on the program clock: the clap's sound in the
+microphone's audio before it is delayed, and the frame where the hands meet in
+the camera's pictures. The difference, bounded by moq-audio's
+`bus::MAX_DELAY`, becomes the delay, as one undoable edit. A clap that is not
+found, or found in only one of the two, fails with the reason rather than
+setting a guess.
+
+The studio sees a source's audio only through its strip's level meter, read
+once a frame, which is too coarse to time a clap. Timing it needs the
+source's samples, or an onset, with their position on the bus clock, which is
+moq-audio work: land it there first and pin it.
+
+## Related
+
+- [Broadcast sources play in sync](/quest/m0/broadcast-sync.md) - broadcast sources align on the publisher's timestamps instead
diff --git a/rs/moq-scene/src/clip.rs b/rs/moq-scene/src/clip.rs
index 7abee57..3c6398d 100644
--- a/rs/moq-scene/src/clip.rs
+++ b/rs/moq-scene/src/clip.rs
@@ -1,13 +1,13 @@
//! Items and filters copied as text, to paste into another scene or another
//! collection.
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use serde::{Deserialize, Serialize};
use crate::Error;
-use crate::collection::{Audio, Collection, Filter, Item, Layer, Source, Video, validate_filters, versioned};
+use crate::collection::{Audio, Collection, Filter, Item, Layer, Mixes, Source, Video, validate_filters, versioned};
use crate::edit::Draft;
/// What a copy holds: items, or a filter chain.
@@ -77,7 +77,7 @@ impl Clip {
clip.sources.insert(name.clone(), source);
names.push(collection.capture(&name).to_string());
if let Some(audio) = collection.audio.get(&name) {
- clip.audio.insert(name.clone(), *audio);
+ clip.audio.insert(name.clone(), audio.clone());
}
if let Some(video) = collection.video.get(&name) {
names.extend(video.filters.iter().filter_map(read));
@@ -99,16 +99,26 @@ impl Clip {
let clip = Self::deserialize(table).map_err(|err| Error::Format(err.to_string()))?;
match &clip {
// The rules a collection holding just the clip would break, such as
- // an item showing a source the clip lacks.
- Self::Items(items) => Collection {
- sources: items.sources.clone(),
- audio: items.audio.clone(),
- video: items.video.clone(),
- scenes: Vec::new(),
- top: items.items.clone(),
- ..Collection::new()
+ // an item showing a source the clip lacks. It has whatever mixes the
+ // clip's sources are left out of, which pasting checks.
+ Self::Items(items) => {
+ let excluded = items.audio.values().flat_map(|audio| &audio.exclude);
+ let mut names: BTreeSet = excluded.cloned().collect();
+ names.insert(Mixes::DEFAULT.into());
+ Collection {
+ sources: items.sources.clone(),
+ mixes: Mixes {
+ names: names.into_iter().collect(),
+ program: Mixes::DEFAULT.into(),
+ },
+ audio: items.audio.clone(),
+ video: items.video.clone(),
+ scenes: Vec::new(),
+ top: items.items.clone(),
+ ..Collection::new()
+ }
+ .validate()?
}
- .validate()?,
Self::Filters(filters) => validate_filters(&filters.filters, "the copied filters")?,
}
Ok(clip)
@@ -138,7 +148,8 @@ impl Draft {
/// one shown, so items pasted within a collection are linked to the ones
/// copied. A source whose name is taken by a different one is added under
/// a new name, which clones and filters naming it follow, and an added
- /// source brings its audio and video settings. An image or LUT under `dir`
+ /// source brings its audio and video settings, left out of only the mixes
+ /// both collections have. An image or LUT under `dir`
/// is named relative to it, as the scene file would.
pub fn paste(&mut self, layer: &Layer, items: &Items, dir: &Path) -> Result<(), Error> {
let mut names = BTreeMap::new();
@@ -190,7 +201,11 @@ impl Draft {
};
for (name, renamed) in added {
if let Some(audio) = items.audio.get(name) {
- self.set_audio(&renamed, Some(*audio))?;
+ // Left out only of the mixes this collection has too.
+ let mut audio = audio.clone();
+ let mixes = &self.collection().mixes.names;
+ audio.exclude.retain(|mix| mixes.contains(mix));
+ self.set_audio(&renamed, Some(audio))?;
}
if let Some(video) = items.video.get(name) {
let filters = rename(&video.filters);
@@ -297,6 +312,40 @@ mod tests {
assert_eq!(pasted.scenes[0].items, source.scenes[0].items);
}
+ /// A pasted source keeps its sound's settings, left out of only the mixes
+ /// the collection it lands in has.
+ #[test]
+ fn pastes_audio_into_the_mixes_there() {
+ let mut source = Collection::new();
+ source.mixes.names.push("Stream".into());
+ source.sources.insert(
+ "guest".into(),
+ Source::Broadcast {
+ path: "room/guest".into(),
+ },
+ );
+ let audio = crate::Audio {
+ gain: 0.5,
+ exclude: ["Program".to_string(), "Stream".to_string()].into(),
+ ..Default::default()
+ };
+ source.audio.insert("guest".into(), audio.clone());
+ source.scenes[0]
+ .items
+ .push(Item::new("guest", Rect::fill(&source.canvas)));
+ source.validate().unwrap();
+ let text = Clip::items(&source, source.scenes[0].items.clone(), Path::new(DIR))
+ .to_toml()
+ .unwrap();
+
+ let pasted = paste(&Collection::new(), &Layer::Top, &text, DIR);
+ let expected = crate::Audio {
+ exclude: ["Program".to_string()].into(),
+ ..audio
+ };
+ assert_eq!(pasted.audio["guest"], expected);
+ }
+
/// A relative image path names the same file wherever the clip is pasted,
/// and stays relative, and linked, within its own collection.
#[test]
diff --git a/rs/moq-scene/src/collection.rs b/rs/moq-scene/src/collection.rs
index b197919..9c74226 100644
--- a/rs/moq-scene/src/collection.rs
+++ b/rs/moq-scene/src/collection.rs
@@ -29,8 +29,11 @@ pub struct Collection {
/// so one camera can appear in many scenes while being opened once.
#[serde(default)]
pub sources: BTreeMap,
- /// Gain and mute of each audio-carrying source, by source name. A source
- /// without an entry plays at unity.
+ /// The audio mixes, and which one the program carries.
+ #[serde(default, skip_serializing_if = "Mixes::is_default")]
+ pub mixes: Mixes,
+ /// Gain, mute, routing, and sync delay of each audio-carrying source, by
+ /// source name. A source without an entry plays at unity in every mix.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub audio: BTreeMap,
/// The filters on each picture-carrying source, by source name, applied
@@ -60,6 +63,7 @@ impl Collection {
canvas: Canvas::default(),
transition: Transition::default(),
sources: BTreeMap::new(),
+ mixes: Mixes::default(),
audio: BTreeMap::new(),
video: BTreeMap::new(),
scenes: vec![Scene::new("Scene")],
@@ -215,6 +219,7 @@ impl Collection {
self.validate_chain(&video.filters, &on)?;
}
+ self.mixes.validate()?;
for (name, audio) in &self.audio {
if !self.sources.contains_key(name) {
return Err(Error::Invalid(format!(
@@ -224,6 +229,18 @@ impl Collection {
if !audio.gain.is_finite() || audio.gain < 0.0 {
return Err(Error::Invalid(format!("`{name}` has gain {}", audio.gain)));
}
+ if let Some(mix) = audio.exclude.iter().find(|mix| !self.mixes.names.contains(mix)) {
+ return Err(Error::Invalid(format!(
+ "`{name}` is left out of `{mix}`, which is not a mix"
+ )));
+ }
+ if audio.delay > moq_audio::bus::MAX_DELAY {
+ return Err(Error::Invalid(format!(
+ "`{name}` is held back {:?}, past the longest, {:?}",
+ audio.delay,
+ moq_audio::bus::MAX_DELAY
+ )));
+ }
}
Ok(())
@@ -1108,15 +1125,72 @@ impl Source {
}
}
-/// The gain and mute of one source in the program audio.
-#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
+/// The audio mixes every source with sound feeds, and which one the program
+/// carries.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct Mixes {
+ /// Each mix's unique name, in the order the studio lists them.
+ pub names: Vec,
+ /// The mix the program broadcast carries.
+ pub program: String,
+}
+
+impl Mixes {
+ /// The one mix a new collection has, which the program carries.
+ pub const DEFAULT: &str = "Program";
+
+ fn is_default(&self) -> bool {
+ *self == Self::default()
+ }
+
+ fn validate(&self) -> Result<(), Error> {
+ let mut names = BTreeSet::new();
+ for name in &self.names {
+ if name.is_empty() {
+ return Err(Error::Invalid("a mix has an empty name".into()));
+ }
+ if !names.insert(name.as_str()) {
+ return Err(Error::Invalid(format!("two mixes are named `{name}`")));
+ }
+ }
+ if !names.contains(self.program.as_str()) {
+ return Err(Error::Invalid(format!(
+ "the program carries `{}`, which is not a mix",
+ self.program
+ )));
+ }
+ Ok(())
+ }
+}
+
+impl Default for Mixes {
+ fn default() -> Self {
+ Self {
+ names: vec![Self::DEFAULT.into()],
+ program: Self::DEFAULT.into(),
+ }
+ }
+}
+
+/// The gain, mute, routing, and sync delay of one source with sound.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Audio {
/// Linear gain: 1.0 is unity.
pub gain: f32,
- /// Whether the source is silenced, keeping its gain.
+ /// Whether the source is silenced in every mix, keeping its gain.
#[serde(default)]
pub muted: bool,
+ /// The mixes the source is left out of. It feeds every other one,
+ /// including a mix added later.
+ #[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
+ pub exclude: BTreeSet,
+ /// How long the source is held back in every mix, so a microphone lines up
+ /// with a camera that lags it. At most the bus's
+ /// [`MAX_DELAY`](moq_audio::bus::MAX_DELAY).
+ #[serde(default, with = "humantime_serde", skip_serializing_if = "Duration::is_zero")]
+ pub delay: Duration,
}
impl Default for Audio {
@@ -1124,6 +1198,8 @@ impl Default for Audio {
Self {
gain: 1.0,
muted: false,
+ exclude: BTreeSet::new(),
+ delay: Duration::ZERO,
}
}
}
@@ -1200,7 +1276,19 @@ mod tests {
height: 800,
},
);
- collection.audio.insert("mic".into(), Audio { gain: 0.5, muted: true });
+ collection.mixes = Mixes {
+ names: vec!["Stream".into(), "Recording".into()],
+ program: "Stream".into(),
+ };
+ collection.audio.insert(
+ "mic".into(),
+ Audio {
+ gain: 0.5,
+ muted: true,
+ exclude: ["Recording".to_string()].into(),
+ delay: Duration::from_millis(80),
+ },
+ );
collection.sources.insert(
"camera faded".into(),
Source::Clone {
@@ -1268,8 +1356,14 @@ kind = "camera"
kind = "color"
color = "#ff000080"
+[mixes]
+names = ["Stream", "Recording"]
+program = "Stream"
+
[audio.cam2]
gain = 2.0
+exclude = ["Stream"]
+delay = "120ms"
[sources.cam2]
kind = "broadcast"
@@ -1324,9 +1418,12 @@ rect = { x = 0, y = 680, width = 1280, height = 40 }
collection.audio["cam2"],
Audio {
gain: 2.0,
- muted: false
+ muted: false,
+ exclude: ["Stream".to_string()].into(),
+ delay: Duration::from_millis(120),
}
);
+ assert_eq!(collection.mixes.program, "Stream");
let main = collection.scene("Main").unwrap();
assert!(main.items[0].visible);
assert!(!main.items[1].visible);
@@ -1394,6 +1491,32 @@ rect = { x = 0, y = 680, width = 1280, height = 40 }
assert!(matches!(collection.validate(), Err(Error::Invalid(_))));
}
+ /// The program carries one of the mixes, each named once, and a source is
+ /// left out of mixes that exist, held back no longer than the bus can.
+ #[test]
+ fn refuses_a_mix_that_does_not_add_up() {
+ let refused = |edit: fn(&mut Collection)| {
+ let mut collection = example();
+ edit(&mut collection);
+ matches!(collection.validate(), Err(Error::Invalid(_)))
+ };
+ assert!(refused(|collection| collection.mixes.program = "Monitor".into()));
+ assert!(refused(|collection| collection.mixes.names.push("Stream".into())));
+ assert!(refused(|collection| collection.mixes.names.push(String::new())));
+ assert!(refused(|collection| {
+ let mic = collection.audio.get_mut("mic").unwrap();
+ mic.exclude.insert("Monitor".into());
+ }));
+ assert!(refused(|collection| {
+ let mic = collection.audio.get_mut("mic").unwrap();
+ mic.delay = moq_audio::bus::MAX_DELAY + Duration::from_millis(1);
+ }));
+ assert!(!refused(|collection| {
+ let mic = collection.audio.get_mut("mic").unwrap();
+ mic.delay = moq_audio::bus::MAX_DELAY;
+ }));
+ }
+
/// A clone shows another source's capture, so it must name one with a
/// picture that is not itself a clone, and only a picture is filtered.
#[test]
diff --git a/rs/moq-scene/src/edit.rs b/rs/moq-scene/src/edit.rs
index e7db3a4..b9ab777 100644
--- a/rs/moq-scene/src/edit.rs
+++ b/rs/moq-scene/src/edit.rs
@@ -14,7 +14,7 @@ use std::collections::{BTreeMap, VecDeque};
use serde::{Deserialize, Serialize};
use crate::Error;
-use crate::collection::{Audio, Canvas, Collection, Filter, Item, Layer, Scene, Source, Transition, Video};
+use crate::collection::{Audio, Canvas, Collection, Filter, Item, Layer, Mixes, Scene, Source, Transition, Video};
/// One part of a collection, before and after an edit.
///
@@ -70,6 +70,8 @@ pub enum Change {
Transition { before: Transition, after: Transition },
/// [`Collection::canvas`].
Canvas { before: Canvas, after: Canvas },
+ /// [`Collection::mixes`].
+ Mixes { before: Mixes, after: Mixes },
}
impl Change {
@@ -115,6 +117,10 @@ impl Change {
before: after,
after: before,
},
+ Self::Mixes { before, after } => Self::Mixes {
+ before: after,
+ after: before,
+ },
}
}
@@ -122,7 +128,11 @@ impl Change {
fn entry(&self) -> Option<&str> {
match self {
Self::Source { name, .. } | Self::Audio { name, .. } | Self::Video { name, .. } => Some(name),
- Self::Scene { .. } | Self::Item { .. } | Self::Transition { .. } | Self::Canvas { .. } => None,
+ Self::Scene { .. }
+ | Self::Item { .. }
+ | Self::Transition { .. }
+ | Self::Canvas { .. }
+ | Self::Mixes { .. } => None,
}
}
@@ -136,6 +146,7 @@ impl Change {
Self::Item { before, after, .. } => before == after,
Self::Transition { before, after } => before == after,
Self::Canvas { before, after } => before == after,
+ Self::Mixes { before, after } => before == after,
}
}
@@ -164,7 +175,7 @@ impl Change {
) if name == *n && after == *b => Self::Audio {
name,
before,
- after: *a,
+ after: a.clone(),
},
(
Self::Video { name, before, after },
@@ -221,6 +232,10 @@ impl Change {
(Self::Canvas { before, after }, Self::Canvas { before: b, after: a }) if after == *b => {
Self::Canvas { before, after: *a }
}
+ (Self::Mixes { before, after }, Self::Mixes { before: b, after: a }) if after == *b => Self::Mixes {
+ before,
+ after: a.clone(),
+ },
_ => return None,
};
Some(merged)
@@ -261,6 +276,13 @@ impl Change {
collection.canvas = *after;
Ok(())
}
+ Self::Mixes { before, after } => {
+ if collection.mixes != *before {
+ return Err(conflict("the mixes changed".into()));
+ }
+ collection.mixes = after.clone();
+ Ok(())
+ }
}
}
}
@@ -427,9 +449,10 @@ impl Draft {
})
}
- /// Set or clear the gain and mute of the source called `name`.
+ /// Set or clear the gain, mute, routing, and delay of the source called
+ /// `name`.
pub fn set_audio(&mut self, name: &str, audio: Option