diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte
index 5b296c62dced..12b66d18e57c 100644
--- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte
+++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte
@@ -64,7 +64,7 @@
parseClipboardContent,
uuid
} from '#lib/utils/index.ts';
- import { VoiceRecorder, warmUpVoiceCapture } from '#lib/voice/index.ts';
+ import { unlockVoicePlayback, VoiceRecorder, warmUpVoiceCapture } from '#lib/voice/index.ts';
import { onMount, tick, untrack } from 'svelte';
import type { ClassValue } from 'svelte/elements';
import { toast } from 'svelte-sonner';
@@ -99,6 +99,8 @@
/** Records voice input; tests and stories pass a fake, the app a VoiceRecorder. */
voiceRecorder?: VoiceRecorderLike;
+ /** Offers voice mode, a spoken conversation, as the main button of an empty message. */
+ voiceMode?: boolean;
}
let {
@@ -121,6 +123,7 @@
singleRow = false,
uploadedFiles = $bindable([]),
value = $bindable(''),
+ voiceMode = false,
voiceRecorder
}: Props = $props();
@@ -668,6 +671,16 @@
onSubmit?.();
}
+ // voice mode leads only when what is said can be transcribed: the chat model hears, or a
+ // transcription model is set or loaded; else the microphone leads, saying why it cannot
+ let voiceModeAvailable = $derived(voiceMode && !voice.unavailableReason);
+
+ // inside the click: a browser that plays sound only after a gesture lets the replies play later
+ function handleVoiceModeClick() {
+ unlockVoicePlayback();
+ uiStore.voiceModeOpen = true;
+ }
+
// Escape drops a recording, wherever the focus is
function handleWindowKeydown(event: KeyboardEvent) {
if (event.key !== KeyboardKey.ESCAPE || !voice.isActive || event.defaultPrevented) return;
@@ -862,6 +875,7 @@
{onStop}
onSystemPromptClick={() =>
onSystemPromptClick?.({ files: uploadedFiles, message: value })}
+ onVoiceModeClick={voiceModeAvailable ? handleVoiceModeClick : undefined}
onWorkingDirectoryClick={isWorkingDirectoryInAddMenu
? pickers.handleWorkingDirectoryOpen
: undefined}
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte
index 73069301474e..439e4f98cea3 100644
--- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte
+++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte
@@ -17,7 +17,7 @@
micWarning?: string | null;
/** Why no model can take the recording; disables the button. */
unavailableReason?: string | null;
- /** A quieter button, beside the main one (stop or send). */
+ /** A quieter button, beside the main one: stop, send or voice mode. */
secondary?: boolean;
/** With `secondary`: filled like the stop button it stands beside, not bare. */
filled?: boolean;
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionVoiceMode.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionVoiceMode.svelte
new file mode 100644
index 000000000000..67bba80b724c
--- /dev/null
+++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionVoiceMode.svelte
@@ -0,0 +1,46 @@
+
+
+
+
+
+
+
+ {VOICE_CONVERSATION_MESSAGE.open}
+
+
+
+
+
+
+ {unavailableReason ?? VOICE_CONVERSATION_MESSAGE.open}
+
+
+
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte
index a523300286a8..44368c396ca2 100644
--- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte
+++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte
@@ -2,10 +2,10 @@
import { Button } from '@ggml-org/llama-ui/ui/button';
import { LoaderCircle, SkipForward, Square } from '@lucide/svelte';
import {
- ChatFormActionReasoning,
ChatFormActionRecord,
ChatFormActionsAdd,
ChatFormActionSubmit,
+ ChatFormActionVoiceMode,
ChatFormContextGauge,
ModelsSelector
} from '#lib/components/app/index.ts';
@@ -50,6 +50,10 @@
workingDirectory?: string | null;
onFileUpload?: () => void;
onMicClick?: () => void;
+ /** Opens voice mode; left out, the microphone is the main button. */
+ onVoiceModeClick?: () => void;
+ /** Why voice mode cannot listen. */
+ voiceModeUnavailableReason?: string | null;
onStop?: () => void;
onSystemPromptClick?: () => void;
onWorkingDirectoryClick?: () => void;
@@ -69,11 +73,13 @@
onMicClick,
onStop,
onSystemPromptClick,
+ onVoiceModeClick,
onWorkingDirectoryClick,
showAddButton = true,
showModelSelector = true,
uploadedFiles = [],
voice,
+ voiceModeUnavailableReason = null,
workingDirectory = null
}: Props = $props();
@@ -171,12 +177,12 @@
{/if}
{#if showModelSelector}
-
-
+
{/if}
@@ -236,6 +242,7 @@
/>
{:else if shouldShowRecordButton}
+
+
+ {#if onVoiceModeClick}
+
+ {/if}
{:else}
{/if}
+{#if uiStore.voiceModeOpen}
+
+{/if}
+
diff --git a/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte
new file mode 100644
index 000000000000..c2db460fcdd9
--- /dev/null
+++ b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte
@@ -0,0 +1,348 @@
+
+
+
+
+
+
+
+
+
+
+ {#if conversation.turns.length === 0}
+
+
+
+
+ {VOICE_CONVERSATION_MESSAGE.greeting}
+
+
+ {:else}
+ {#each conversation.turns as turn (turn.id)}
+
+
+ {turn.user}
+
+
+ {#if turn.reply}
+
+
+
+ {:else if turn.id === lastTurnId && !turn.done}
+
+ {/if}
+
+ {/each}
+ {/if}
+
+
+
+
+
+
+
+
+ {#if askPermission}
+ {#key askPermission}
+
+
+
+ {/key}
+ {:else if askContinue}
+
+
+
+ {/if}
+
+
+ {#if conversation.liveText && (listening || conversation.state === 'transcribing')}
+
+ {:else}
+ {status}
+ {/if}
+
+
+
+
+
+
+ {#if conversation.micMuted}
+
+ {:else}
+
+ {/if}
+
+
+
+
+
+
+
+
+ conversation.toggleSpeaker()}
+ type="button"
+ variant="secondary"
+ >
+ {#if conversation.speakerMuted || !conversation.canSpeak}
+
+ {:else}
+
+ {/if}
+
+
+
+
+
+ {conversation.canSpeak
+ ? conversation.speakerMuted
+ ? VOICE_CONVERSATION_MESSAGE.speakerOn
+ : VOICE_CONVERSATION_MESSAGE.speakerOff
+ : VOICE_CONVERSATION_MESSAGE.noSpeechModel}
+
+
+
+
+
+
+
+
+
+
+
diff --git a/ui/apps/chat/src/lib/components/app/chat/index.ts b/ui/apps/chat/src/lib/components/app/chat/index.ts
index f7626c73a4f5..554d77333a0f 100644
--- a/ui/apps/chat/src/lib/components/app/chat/index.ts
+++ b/ui/apps/chat/src/lib/components/app/chat/index.ts
@@ -185,6 +185,16 @@ export { default as ChatFormActionsAdd } from './ChatForm/ChatFormActions/ChatFo
*/
export { default as ChatFormActionRecord } from './ChatForm/ChatFormActions/ChatFormActionRecord.svelte';
+/**
+ * **ChatFormActionVoiceMode** - Opens voice mode, the chat form's main button while the message is empty
+ */
+export { default as ChatFormActionVoiceMode } from './ChatForm/ChatFormActions/ChatFormActionVoiceMode.svelte';
+
+/**
+ * **VoiceModeScreen** - Full-screen spoken conversation with the chat model
+ */
+export { default as VoiceModeScreen } from './VoiceMode/VoiceModeScreen.svelte';
+
/**
* The actions row while the chat form records: cancel, the live waveform and the
* elapsed time, and stop, with a voice message's streaming text under the waveform
diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte
index 3b24b83a0472..7138a07f91cb 100644
--- a/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte
+++ b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte
@@ -1,8 +1,11 @@
+
+{#snippet header()}
+ {#if reasoning.isOff}
+
+ {:else}
+
+ {/if}
+
+ Reasoning
+
+ {current}
+{/snippet}
+
+{#snippet levelLabel(level: ReasoningEffortLevel)}
+ {@const tokenLabel = reasoning.tokenLabel(level)}
+
+ {#if reasoning.isSelected(level)}
+
+ {:else}
+
+ {/if}
+
+ {level.label}
+
+ {#if tokenLabel}
+ {tokenLabel}
+ {/if}
+{/snippet}
+
+{#if variant === 'menu'}
+
+
+
+
+ {#each reasoning.levels as level (level.value)}
+ reasoning.select(level)}>
+ {@render levelLabel(level)}
+
+ {/each}
+
+
+
+{:else}
+
+
+
+ {#each reasoning.levels as level (level.value)}
+ reasoning.select(level)}
+ type="button"
+ >
+ {@render levelLabel(level)}
+
+ {/each}
+
+
+
+{/if}
diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorReasoningBadge.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorReasoningBadge.svelte
new file mode 100644
index 000000000000..fabc65c5425c
--- /dev/null
+++ b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorReasoningBadge.svelte
@@ -0,0 +1,34 @@
+
+
+
+{#if reasoning.isOff}
+
+
+
+ Reasoning off
+
+{:else if reasoning.currentEffort !== ReasoningEffort.DEFAULT}
+
+ {REASONING_EFFORT_LABELS[reasoning.currentEffort]}
+
+ Reasoning {REASONING_EFFORT_LABELS[reasoning.currentEffort]}
+
+{/if}
diff --git a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts
index 7b067c468095..dcaed3da880a 100644
--- a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts
+++ b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts
@@ -94,6 +94,7 @@ export const SETTINGS_KEYS = {
VOICE_INPUT_MODE: 'voiceInputMode',
VOICE_MICROPHONE: 'voiceMicrophone',
VOICE_NOISE_SUPPRESSION: 'voiceNoiseSuppression',
+ VOICE_READING_SPEED: 'voiceReadingSpeed',
VOICE_TRANSCRIBE_MESSAGES: 'voiceTranscribeMessages',
VOICE_TRANSCRIPTION_LANGUAGE: 'voiceTranscriptionLanguage',
VOICE_TRANSCRIPTION_MODEL: 'voiceTranscriptionModel',
diff --git a/ui/apps/chat/src/lib/constants/settings.constants.ts b/ui/apps/chat/src/lib/constants/settings.constants.ts
index 4544a88df01f..ebf19c82715f 100644
--- a/ui/apps/chat/src/lib/constants/settings.constants.ts
+++ b/ui/apps/chat/src/lib/constants/settings.constants.ts
@@ -12,6 +12,7 @@ import {
Database,
File,
Funnel,
+ Gauge,
GlassWater,
ListRestart,
Mic,
@@ -481,6 +482,20 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [
label: 'Automatic gain',
type: SettingsFieldType.CHECKBOX
},
+ {
+ defaultValue: '1.25',
+ help: 'How fast voice mode reads replies aloud. The voice keeps its pitch.',
+ key: SETTINGS_KEYS.VOICE_READING_SPEED,
+ label: 'Reading speed',
+ options: [
+ { icon: Gauge, label: '1×', value: '1' },
+ { icon: Gauge, label: '1.25×', value: '1.25' },
+ { icon: Gauge, label: '1.5×', value: '1.5' },
+ { icon: Gauge, label: '1.75×', value: '1.75' },
+ { icon: Gauge, label: '2×', value: '2' }
+ ],
+ type: SettingsFieldType.SELECT
+ },
{
defaultValue: '',
help: 'The language you speak, as a two-letter code such as en or de, passed to the transcription. Leave it empty to let the model tell.',
diff --git a/ui/apps/chat/src/lib/constants/voice.constants.ts b/ui/apps/chat/src/lib/constants/voice.constants.ts
index 7c49dcfff478..b152017e11e1 100644
--- a/ui/apps/chat/src/lib/constants/voice.constants.ts
+++ b/ui/apps/chat/src/lib/constants/voice.constants.ts
@@ -16,6 +16,25 @@ export const VOICE_MODE_MESSAGE = {
voiceMessage: 'Send a voice message'
} as const;
+/** What voice mode says: its states, buttons and the reason it cannot read replies aloud. */
+export const VOICE_CONVERSATION_MESSAGE = {
+ close: 'Leave voice mode',
+ greeting: "Let's talk",
+ interrupt: 'Interrupt the reply',
+ listening: 'Listening...',
+ micOff: 'Microphone off',
+ mute: 'Turn the microphone off',
+ noSpeechModel:
+ 'No model that speaks is available, so replies are shown, not read aloud. Load a model that generates audio to hear them.',
+ open: 'Voice mode',
+ speakerOff: 'Stop reading replies aloud',
+ speakerOn: 'Read replies aloud',
+ starting: 'Starting...',
+ thinking: 'Thinking...',
+ transcribing: 'Transcribing...',
+ unmute: 'Turn the microphone on'
+} as const;
+
/** Where the chat form remembers that it showed the hint about a separate transcription model. */
export const VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.voiceSeparateModelHintSeen`;
diff --git a/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts
new file mode 100644
index 000000000000..a99eaa39640b
--- /dev/null
+++ b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts
@@ -0,0 +1,576 @@
+/**
+ * Voice mode: a spoken conversation with the chat model, hands-free.
+ *
+ * Each turn listens (echo cancellation on, as the reply plays from the same device),
+ * shows what is said as it is transcribed, and ends after a pause in speech: the text
+ * goes to the chat as a message. The reply streams on screen and, when a model that
+ * speaks is available, is read aloud sentence by sentence; then the next turn listens.
+ * Speaking while the reply plays stops it and starts a new turn.
+ */
+
+import { SETTINGS_KEYS } from '#lib/constants/index.ts';
+import { MessageRole, VoiceInputMode, VoiceMode } from '#lib/enums/index.ts';
+import {
+ chatStore,
+ conversationsStore,
+ modelsStore,
+ serverStore,
+ settingsStore
+} from '#lib/stores/index.ts';
+import { getAuthHeaders } from '#lib/utils/api-headers.ts';
+import { describeMicrophoneError } from '#lib/utils/audio-recording.ts';
+import { withBase } from '#lib/utils/base-path.ts';
+import { resolveVoiceMode } from '#lib/utils/voice-mode.ts';
+import {
+ PauseDetector,
+ ReplyReader,
+ resolveMicrophone,
+ synthesizeSpeech,
+ VOICE_LEVELS_INTERVAL_MS,
+ VOICE_MICROPHONE_AUTO,
+ VoiceCapture,
+ VoiceCaptureError,
+ VoiceRecorder,
+ type VoiceServerContext
+} from '#lib/voice/index.ts';
+import { onDestroy, untrack } from 'svelte';
+
+/** Where a voice conversation is. */
+export type VoiceConversationState =
+ | 'idle'
+ | 'starting'
+ | 'listening'
+ | 'transcribing'
+ | 'thinking'
+ | 'speaking'
+ | 'paused';
+
+/** One exchange: what was said, and the reply. */
+export interface VoiceTurn {
+ id: number;
+ user: string;
+ reply: string;
+ /** The message went to the chat, so its reply is followed */
+ sent: boolean;
+ done: boolean;
+}
+
+// voice mode talks to a device that also plays the reply: the browser's clean-up removes its echo
+const CONVERSATION_CONSTRAINTS: MediaTrackConstraints = {
+ autoGainControl: true,
+ echoCancellation: true,
+ noiseSuppression: true
+};
+/** Silence after speech that ends a turn. */
+const TURN_PAUSE_MS = 900;
+/** Sound over the reply that pauses it, to hear whether it is speech. */
+const BARGE_IN_MS = 300;
+/** How long a paused reply waits for speech before it goes on: no speech, it was a noise or its own echo. */
+const BARGE_IN_CONFIRM_MS = 1000;
+
+function serverContext(): { context: VoiceServerContext; realtime: boolean } {
+ const headers = getAuthHeaders();
+ const hasAuth = Object.keys(headers).some((key) => key.toLowerCase() === 'authorization');
+
+ return {
+ context: {
+ getHeaders: () => getAuthHeaders(),
+ resolveUrl: (path) => withBase(path.replace(/^\./, ''))
+ },
+ realtime: !hasAuth
+ };
+}
+
+export function useVoiceConversation() {
+ const config = $derived(settingsStore.config);
+ const isRouter = $derived(serverStore.isRouterMode);
+ const chatModel = $derived(modelsStore.activeModelId);
+ const hasAudio = (model: string | null) => {
+ void modelsStore.props.cacheVersion;
+
+ return model ? modelsStore.props.modelSupportsAudio(model) : false;
+ };
+ const transcriptionSetting = $derived(
+ String(config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL] ?? '')
+ );
+ // what is said always becomes text here, so it is resolved as dictation
+ const resolution = $derived.by(() => {
+ void modelsStore.props.cacheVersion;
+
+ const loadedAudioModel = isRouter
+ ? (modelsStore.models.find(
+ (option) => hasAudio(option.model) && modelsStore.isModelLoaded(option.model)
+ )?.model ?? null)
+ : null;
+
+ return resolveVoiceMode({
+ chatModel,
+ chatModelHasAudio: hasAudio(chatModel),
+ isRouter,
+ loadedAudioModel,
+ setting: VoiceInputMode.DICTATION,
+ transcriptionModel: transcriptionSetting,
+ transcriptionModelHasAudio: hasAudio(transcriptionSetting || null)
+ });
+ });
+ // the chat model reads its own replies when it can speak, else another model that can
+ const speechModel = $derived.by(() => {
+ const options = modelsStore.models;
+ const chat = options.find((option) => option.model === chatModel);
+
+ if (chat?.speaks) return chat.model;
+
+ return isRouter ? (options.find((option) => option.speaks)?.model ?? null) : null;
+ });
+
+ let state = $state('idle');
+ let micMuted = $state(false);
+ let speakerMuted = $state(false);
+ let liveText = $state('');
+ let levels = $state([]);
+ let error = $state(null);
+ let turns = $state([]);
+ let recorder: VoiceRecorder | null = null;
+ let monitor: VoiceCapture | null = null;
+ let generation = 0;
+ let turnId = 0;
+ // the current reply was seen loading: until then, not loading does not mean done
+ let replyStarted = false;
+ // the reply is paused over a sound, until speech confirms the interruption or a second passes
+ let tentative = false;
+ let tentativeTimer: ReturnType | undefined;
+ // a reply paused by an interruption: it goes on if the microphone is muted or nothing is
+ // said, and is dropped once the user's turn is sent
+ let heldReply = $state(false);
+
+ const reader = new ReplyReader({
+ onIdle: () => {
+ if (state === 'speaking' || state === 'thinking') void listen();
+ },
+ playbackRate: () => Number(config[SETTINGS_KEYS.VOICE_READING_SPEED]) || 1.25,
+ synthesize: (text, signal) => {
+ const { context } = serverContext();
+
+ return synthesizeSpeech(context, text, {
+ model: isRouter && speechModel ? speechModel : undefined,
+ signal
+ });
+ }
+ });
+ const currentTurn = $derived(turns[turns.length - 1]);
+
+ // the reply as it streams, then as stored once done; the conversation is read here, as a
+ // first message creates it while being sent
+ $effect(() => {
+ const turn = currentTurn;
+ const convId = conversationsStore.activeConversation?.id;
+
+ if (!turn || !turn.sent || turn.done || !convId) return;
+
+ const streaming = chatStore.getChatStreaming(convId);
+ const loading = chatStore.isChatLoading(convId);
+
+ untrack(() => {
+ if (loading) {
+ replyStarted = true;
+
+ if (!streaming) return;
+
+ turn.reply = streaming.response;
+ reader.update(streaming.response, false);
+
+ return;
+ }
+
+ if (replyStarted) finishReply(turn);
+ });
+ });
+
+ /** The reply is complete: as stored, read to its end. */
+ function finishReply(turn: VoiceTurn) {
+ if (turn.done) return;
+
+ const last = conversationsStore.activeMessages.at(-1);
+ const text = last?.role === MessageRole.ASSISTANT ? last.content : turn.reply;
+
+ turn.reply = text;
+ turn.done = true;
+ reader.update(text, true);
+ }
+
+ // the reply plays: speaking over it may interrupt it
+ $effect(() => {
+ if (reader.speaking && state === 'thinking') {
+ untrack(() => {
+ state = 'speaking';
+ void watchForBargeIn(generation);
+ });
+ }
+ });
+
+ async function microphone(): Promise {
+ return resolveMicrophone(
+ String(config[SETTINGS_KEYS.VOICE_MICROPHONE] ?? VOICE_MICROPHONE_AUTO)
+ );
+ }
+
+ function stopRecorder() {
+ recorder?.cancel();
+ recorder = null;
+ }
+
+ function stopMonitor() {
+ monitor?.cancel();
+ monitor = null;
+ }
+
+ /**
+ * Starts a turn: listens until the speaker pauses, then sends what was said. `floor` is
+ * the room's noise level when known; `tentative` listens over a paused reply, which
+ * goes on unless speech is heard within a second.
+ */
+ async function listen(options: { floor?: number; tentative?: boolean } = {}) {
+ stopMonitor();
+ stopRecorder();
+ liveText = '';
+ levels = [];
+
+ if (micMuted) {
+ state = 'paused';
+
+ return;
+ }
+
+ if (resolution.mode === VoiceMode.UNAVAILABLE) {
+ error = resolution.reason;
+ state = 'paused';
+
+ return;
+ }
+
+ const run = ++generation;
+ const model = resolution.transcriptionModel;
+ const { context, realtime } = serverContext();
+ const language = String(config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE] ?? '').trim();
+ const pauses = new PauseDetector({
+ floor: options.floor,
+ intervalMs: VOICE_LEVELS_INTERVAL_MS,
+ maxSegmentMs: Number.POSITIVE_INFINITY,
+ pauseMs: TURN_PAUSE_MS
+ });
+ const active = new VoiceRecorder();
+
+ recorder = active;
+ state = 'starting';
+
+ try {
+ const deviceId = await microphone();
+
+ await active.start({
+ capture: { constraints: CONVERSATION_CONSTRAINTS, ...(deviceId ? { deviceId } : {}) },
+ onLevels: (next) => {
+ levels = next as number[];
+
+ if (run !== generation) return;
+
+ const cut = pauses.push(next.at(-1) ?? 0);
+
+ // speech over the paused reply: the interruption was meant
+ if (tentative && pauses.heardSpeech) confirmInterrupt();
+
+ if (state === 'listening' && cut) void endTurn(run);
+ },
+ onText: (text) => {
+ if (run === generation) liveText = text;
+ },
+ realtime,
+ transcribe: {
+ context,
+ language: language || undefined,
+ model: isRouter && model ? model : undefined
+ }
+ });
+ } catch (cause) {
+ if (run !== generation) return;
+
+ error = cause instanceof VoiceCaptureError ? cause.message : describeMicrophoneError(cause);
+ state = 'paused';
+
+ return;
+ }
+
+ if (run !== generation) {
+ active.cancel();
+
+ return;
+ }
+
+ error = null;
+ state = 'listening';
+
+ if (options.tentative && tentative) {
+ clearTimeout(tentativeTimer);
+ tentativeTimer = setTimeout(() => {
+ if (run === generation && tentative && !pauses.heardSpeech) resumeReading();
+ }, BARGE_IN_CONFIRM_MS);
+ }
+ }
+
+ /** The speaker paused: what they said goes to the chat. */
+ async function endTurn(run: number) {
+ const active = recorder;
+
+ if (!active || run !== generation) return;
+
+ state = 'transcribing';
+ recorder = null;
+
+ let text = liveText;
+
+ try {
+ const result = await active.stop();
+
+ text = (await (result.transcript ?? Promise.resolve(text))) || result.liveText || text;
+ } catch (cause) {
+ console.warn('Voice mode: the turn could not be transcribed:', cause);
+ }
+
+ if (run !== generation) return;
+
+ text = text.trim();
+
+ // nothing understood: a reply held for this turn goes on, else listen again
+ if (!text) {
+ if (heldReply) resumeReading();
+ else void listen();
+
+ return;
+ }
+
+ // the user had something to say: the reply they interrupted gives way
+ dropHeldReply();
+
+ const turn: VoiceTurn = { done: false, id: ++turnId, reply: '', sent: false, user: text };
+
+ turns.push(turn);
+ liveText = '';
+ reader.reset();
+ reader.muted = speakerMuted || !speechModel;
+ replyStarted = false;
+ state = 'thinking';
+
+ const sending = chatStore.sendMessage(text);
+
+ // followed from here: the message is on its way
+ turns[turns.length - 1].sent = true;
+
+ try {
+ await sending;
+
+ // a reply that was never seen streaming (an error, a very short one) ends here
+ if (run === generation) finishReply(turns[turns.length - 1]);
+ } catch (cause) {
+ if (run !== generation) return;
+
+ error = cause instanceof Error ? cause.message : String(cause);
+ void listen();
+ }
+ }
+
+ /** While the reply plays, speech over it stops it and starts a new turn. */
+ async function watchForBargeIn(run: number) {
+ if (micMuted || speakerMuted || !speechModel || monitor) return;
+
+ const capture = new VoiceCapture({ constraints: CONVERSATION_CONSTRAINTS });
+
+ let speechMs = 0;
+ let floor = 1;
+
+ monitor = capture;
+ capture.onLevels((next) => {
+ const level = next.at(-1) ?? 0;
+
+ floor = Math.min(level, floor + 0.002);
+ speechMs = level > floor + 0.2 ? speechMs + VOICE_LEVELS_INTERVAL_MS : 0;
+
+ if (speechMs >= BARGE_IN_MS && run === generation && reader.speaking && !reader.paused) {
+ bargeIn(floor);
+ }
+ });
+
+ try {
+ await capture.start();
+ } catch {
+ // no barge-in without a microphone; the turn still follows the reply
+ }
+
+ if (run !== generation || monitor !== capture || state !== 'speaking') {
+ capture.cancel();
+
+ if (monitor === capture) monitor = null;
+ }
+ }
+
+ /** A sound over the reply: it pauses, and the next second tells whether it was speech. */
+ function bargeIn(floor: number) {
+ reader.pause();
+ heldReply = true;
+ tentative = true;
+ void listen({ floor, tentative: true });
+ }
+
+ /** No speech came, or the microphone was muted: the reply goes on from where it paused. */
+ function resumeReading() {
+ heldReply = false;
+ tentative = false;
+ clearTimeout(tentativeTimer);
+ generation++;
+ stopRecorder();
+ liveText = '';
+ levels = [];
+ state = 'speaking';
+ reader.resume();
+ void watchForBargeIn(generation);
+ }
+
+ /** Speech came: the turn that started goes on, over the reply held where it paused. */
+ function confirmInterrupt() {
+ tentative = false;
+ clearTimeout(tentativeTimer);
+ }
+
+ /** The interrupted reply is given up: its reading and its generation stop. */
+ function dropHeldReply() {
+ if (!heldReply) return;
+
+ heldReply = false;
+ reader.stop();
+
+ const convId = conversationsStore.activeConversation?.id;
+
+ if (convId && chatStore.isChatLoading(convId)) void chatStore.stopGeneration();
+
+ const turn = currentTurn;
+
+ if (turn) turn.done = true;
+ }
+
+ /** Holds the reply where it is, and listens; muting the microphone lets it go on. */
+ function interrupt() {
+ tentative = false;
+ clearTimeout(tentativeTimer);
+ reader.pause();
+ heldReply = true;
+ void listen();
+ }
+
+ /**
+ * Reads the reply on from `offset`, e.g. a word tapped while it is read; a reply held for
+ * an interruption goes on from there too, as the tap says the user wants to hear it.
+ */
+ function readFrom(offset: number) {
+ if (!reader.speaking && !heldReply) return;
+
+ if (heldReply) resumeReading();
+
+ reader.seek(offset);
+ }
+
+ /** Starts the conversation; the click that opened voice mode unlocked playback (unlockVoicePlayback). */
+ function open() {
+ error = null;
+ turns = [];
+ void listen();
+ }
+
+ function close() {
+ heldReply = false;
+ tentative = false;
+ clearTimeout(tentativeTimer);
+ generation++;
+ stopRecorder();
+ stopMonitor();
+ reader.stop();
+ state = 'idle';
+ liveText = '';
+ }
+
+ function toggleMic() {
+ micMuted = !micMuted;
+
+ if (micMuted) {
+ // an interrupted reply goes on from where it paused
+ if (heldReply) {
+ resumeReading();
+
+ return;
+ }
+
+ if (state === 'listening' || state === 'starting') {
+ generation++;
+ stopRecorder();
+ state = 'paused';
+ }
+
+ stopMonitor();
+
+ return;
+ }
+
+ if (state === 'paused') void listen();
+ }
+
+ function toggleSpeaker() {
+ speakerMuted = !speakerMuted;
+ reader.muted = speakerMuted || !speechModel;
+
+ if (speakerMuted) stopMonitor();
+ }
+
+ onDestroy(close);
+
+ return {
+ /** Whether replies can be read aloud: a model that speaks is available. */
+ get canSpeak() {
+ return !!speechModel;
+ },
+ close,
+ get error() {
+ return error;
+ },
+ interrupt,
+ get levels() {
+ return levels;
+ },
+ get liveText() {
+ return liveText;
+ },
+ get micMuted() {
+ return micMuted;
+ },
+ open,
+ get reader() {
+ return reader;
+ },
+ readFrom,
+ /** Whether the reply is being read, or held for an interruption: readFrom() can move it. */
+ get reading() {
+ return reader.speaking || heldReply;
+ },
+ get speakerMuted() {
+ return speakerMuted;
+ },
+ get state() {
+ return state;
+ },
+ toggleMic,
+ toggleSpeaker,
+ get turns() {
+ return turns;
+ },
+ /** Why voice mode cannot listen, or null. */
+ get unavailableReason() {
+ return resolution.mode === VoiceMode.UNAVAILABLE ? resolution.reason : null;
+ }
+ };
+}
diff --git a/ui/apps/chat/src/lib/stores/models/index.svelte.ts b/ui/apps/chat/src/lib/stores/models/index.svelte.ts
index 67518bf8bddf..7928defaad30 100644
--- a/ui/apps/chat/src/lib/stores/models/index.svelte.ts
+++ b/ui/apps/chat/src/lib/stores/models/index.svelte.ts
@@ -690,6 +690,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
model: modelId,
name: this.toDisplayName(displayNameSource),
parsedId: ModelsService.parseModelId(modelId),
+ speaks: item.architecture?.output_modalities?.includes('audio') ?? false,
tags: item.tags ?? []
};
})
diff --git a/ui/apps/chat/src/lib/stores/ui.svelte.ts b/ui/apps/chat/src/lib/stores/ui.svelte.ts
index 707648d06de7..1ba36014c014 100644
--- a/ui/apps/chat/src/lib/stores/ui.svelte.ts
+++ b/ui/apps/chat/src/lib/stores/ui.svelte.ts
@@ -107,6 +107,8 @@ class UiStore {
sidebarSectionOrder = $state(loadSidebarSectionOrder());
/** Expanded desktop sidebar width in px; null keeps the stylesheet default. */
sidebarWidth = $state(this.loadSidebarWidth());
+ /** Whether the full-screen voice mode is open: a spoken conversation with the chat model. */
+ voiceModeOpen = $state(false);
#isSidebarExpanded = $state(this.loadSidebarExpanded());
diff --git a/ui/apps/chat/src/lib/types/api.d.ts b/ui/apps/chat/src/lib/types/api.d.ts
index abdd1b3de36e..371913fc4c26 100644
--- a/ui/apps/chat/src/lib/types/api.d.ts
+++ b/ui/apps/chat/src/lib/types/api.d.ts
@@ -121,6 +121,8 @@ export interface ApiModelDataEntry {
export interface ApiModelArchitecture {
/** Accepted input modalities, always contains "text" */
input_modalities: string[];
+ /** Generated output modalities: "audio" when the model can speak */
+ output_modalities?: string[];
}
/**
diff --git a/ui/apps/chat/src/lib/types/models.d.ts b/ui/apps/chat/src/lib/types/models.d.ts
index 9d0594073406..9d4a7c88882b 100644
--- a/ui/apps/chat/src/lib/types/models.d.ts
+++ b/ui/apps/chat/src/lib/types/models.d.ts
@@ -23,6 +23,8 @@ export interface ModelOption {
/** Draft sidecars downloaded alongside the model, e.g. `mtp` or `dflash`. */
draftSidecars?: ModelSidecarBadge[];
modalities?: ModelModalities;
+ /** The model can generate speech, e.g. to read replies aloud. */
+ speaks?: boolean;
details?: ApiModelDetails['details'];
meta?: ApiModelDataEntry['meta'];
parsedId?: ParsedModelId;
diff --git a/ui/apps/chat/src/lib/voice/constants.ts b/ui/apps/chat/src/lib/voice/constants.ts
index 7004254429d8..d38d050300f6 100644
--- a/ui/apps/chat/src/lib/voice/constants.ts
+++ b/ui/apps/chat/src/lib/voice/constants.ts
@@ -73,6 +73,7 @@ export const VOICE_BASE64_SLICE = 0x8000;
/** Server routes, relative so a server under a subpath still works. */
export const VOICE_API_PATHS = {
REALTIME: './v1/realtime',
+ SPEECH: './v1/audio/speech',
TRANSCRIPTIONS: './v1/audio/transcriptions'
} as const;
diff --git a/ui/apps/chat/src/lib/voice/index.ts b/ui/apps/chat/src/lib/voice/index.ts
index ed54f2db83b6..28f157753798 100644
--- a/ui/apps/chat/src/lib/voice/index.ts
+++ b/ui/apps/chat/src/lib/voice/index.ts
@@ -33,7 +33,11 @@ export {
VOICE_MICROPHONE_DEFAULT
} from './microphone';
export { computePeaks, computePeaksFromAudio, decodeAudio, mixToMono } from './peaks';
+export { PauseDetector, type PauseDetectorOptions } from './pauses';
+export { type ReadAloudPiece, readAloudPieces, toSpeech } from './read-aloud';
export { RealtimeTranscriber, realtimeUrl } from './realtime';
+export { ReplyReader, type ReplyReaderOptions, unlockVoicePlayback } from './reply-reader.svelte';
+export { synthesizeSpeech, type SynthesizeSpeechOptions } from './speech';
export { startVoiceSession } from './session';
export { transcribeClip } from './transcribe';
export type * from './types';
diff --git a/ui/apps/chat/src/lib/voice/pauses.ts b/ui/apps/chat/src/lib/voice/pauses.ts
index 44083a7ee758..69815fe47c1b 100644
--- a/ui/apps/chat/src/lib/voice/pauses.ts
+++ b/ui/apps/chat/src/lib/voice/pauses.ts
@@ -17,6 +17,8 @@ export interface PauseDetectorOptions {
* is ever quiet
*/
maxSegmentMs?: number;
+ /** The room's noise level when it is known already, so speech from the first level counts */
+ floor?: number;
}
/** How far above the noise floor a level is speech, and below which it is quiet. */
@@ -47,6 +49,7 @@ export class PauseDetector {
this.pauseMs = options.pauseMs ?? 500;
this.minSpeechMs = options.minSpeechMs ?? 300;
this.maxSegmentMs = options.maxSegmentMs ?? 10_000;
+ this.floor = options.floor ?? 1;
}
/** Takes the newest level; true when this is a place to cut the recording. */
diff --git a/ui/apps/chat/src/lib/voice/read-aloud.ts b/ui/apps/chat/src/lib/voice/read-aloud.ts
new file mode 100644
index 000000000000..cf375edcfd65
--- /dev/null
+++ b/ui/apps/chat/src/lib/voice/read-aloud.ts
@@ -0,0 +1,79 @@
+/**
+ * Cuts a reply into pieces to read aloud while it streams: whole sentences (or lines),
+ * each read as soon as it is complete, and the rest once the reply is done. Markdown
+ * marks are left out of what is spoken; the offsets point into the reply as written,
+ * so the text on screen can follow the voice.
+ */
+
+export interface ReadAloudPiece {
+ /** Where the piece starts and ends in the reply as written */
+ start: number;
+ end: number;
+ /** What is spoken: the piece without markdown marks */
+ speech: string;
+}
+
+// a sentence ends at . ! ? (with closing quotes or brackets) before whitespace, or a line ends;
+// a list number such as "1." does not end one
+const PIECE_END = /(?|[-*+]|\d+[.)])\s+/gm, '')
+ .replace(/[*_~]{1,3}([^*_~]+)[*_~]{1,3}/g, '$1')
+ .replace(/\s+/g, ' ')
+ .trim();
+}
diff --git a/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts b/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts
new file mode 100644
index 000000000000..2e084b2b044b
--- /dev/null
+++ b/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts
@@ -0,0 +1,363 @@
+import { type ReadAloudPiece, readAloudPieces } from './read-aloud';
+import { trimSilence } from './trim-silence';
+
+// one element plays every reply: unlocked once inside a click, it may play later (iOS asks for that)
+let sharedAudio: HTMLAudioElement | null = null;
+
+function playbackElement(): HTMLAudioElement {
+ sharedAudio ??= new Audio();
+
+ return sharedAudio;
+}
+
+/**
+ * Plays nothing, inside a click: browsers that let a page play sound only after a gesture
+ * (iOS) then let replies play later. Call it in the handler of the click that opens voice mode.
+ */
+export function unlockVoicePlayback(): void {
+ if (typeof Audio === 'undefined') return;
+
+ const audio = playbackElement();
+
+ audio.muted = true;
+ audio.play().catch(() => {});
+ audio.pause();
+ audio.muted = false;
+}
+
+export interface ReplyReaderOptions {
+ /** Speaks a piece of text, e.g. with synthesizeSpeech(); the WAV or other audio to play */
+ synthesize: (text: string, signal: AbortSignal) => Promise;
+ /** Called once the reply is done and all of it has been read (or skipped while muted) */
+ onIdle?: () => void;
+ /** Speed of the voice, e.g. 1.25; the pitch is kept. Read before each piece. */
+ playbackRate?: () => number;
+}
+
+/**
+ * Reads a reply aloud while it streams: give it the reply as it grows with update(),
+ * and it speaks each complete sentence in order, fetching the next one while the
+ * current one plays. `spoken` follows the voice through the reply as written, for
+ * VoiceSpokenText, and seek() moves the voice to another place in it. Muted, it reads
+ * nothing and `spoken` keeps up with the text.
+ */
+export class ReplyReader {
+ /** Whether a piece is being fetched or played */
+ speaking = $state(false);
+ /** Characters of the reply read so far */
+ spoken = $state(0);
+
+ private audio: HTMLAudioElement | null = null;
+ private done = false;
+ // the reply's pieces and their audio, kept so seek() can go back; bookkeeping the UI never reads
+ // eslint-disable-next-line svelte/prefer-svelte-reactivity
+ private fetched = new Map>();
+ // the reply's speech requests, and the reading under way, which seek() starts anew
+ private fetches = new AbortController();
+ private frame: number | null = null;
+ private idle = true;
+ private isMuted = false;
+ private isPaused = false;
+ private readonly options: ReplyReaderOptions;
+ private pieces: ReadAloudPiece[] = [];
+ private playing = false;
+ private queue: ReadAloudPiece[] = [];
+ private queuedTo = 0;
+ private resumeWaiters: (() => void)[] = [];
+ private run = new AbortController();
+ // where in its piece the next one to play starts, after seek()
+ private startAt: { piece: ReadAloudPiece; share: number } | null = null;
+ private text = '';
+
+ get muted(): boolean {
+ return this.isMuted;
+ }
+
+ /** Whether reading is paused, see pause(). */
+ get paused(): boolean {
+ return this.isPaused;
+ }
+
+ set muted(value: boolean) {
+ this.isMuted = value;
+
+ if (value) this.silence();
+ }
+
+ constructor(options: ReplyReaderOptions) {
+ this.options = options;
+ }
+
+ /** Holds the voice where it is, e.g. to hear whether the user speaks; resume() goes on from there. */
+ pause(): void {
+ if (this.isPaused) return;
+
+ this.isPaused = true;
+
+ if (this.audio && !this.audio.paused) this.audio.pause();
+ }
+
+ /** Starts a new reply. */
+ reset(): void {
+ this.stop();
+ this.fetches = new AbortController();
+ this.run = new AbortController();
+ this.pieces = [];
+ this.text = '';
+ this.done = false;
+ this.queuedTo = 0;
+ this.spoken = 0;
+ this.idle = false;
+ }
+
+ /** Goes on reading from where pause() held it. */
+ resume(): void {
+ if (!this.isPaused) return;
+
+ this.isPaused = false;
+
+ const audio = this.audio;
+
+ if (this.playing && audio?.src && !audio.ended && audio.paused) audio.play().catch(() => {});
+
+ for (const wake of this.resumeWaiters.splice(0)) wake();
+ }
+
+ /**
+ * Reads on from `offset` in the reply as written, e.g. a word tapped on screen, before
+ * or after where the voice is; a piece already fetched plays at once, from about where
+ * the word falls in it. False when there is nothing to read there: the reply was read
+ * to its end, the voice is muted, or that sentence is not complete yet.
+ */
+ seek(offset: number): boolean {
+ if (this.idle || this.isMuted) return false;
+
+ const index = this.pieces.findIndex((piece) => offset < piece.end);
+
+ if (index === -1) return false;
+
+ const piece = this.pieces[index];
+ const from = Math.max(offset, piece.start);
+
+ this.run.abort();
+ this.run = new AbortController();
+
+ // a reading that waited on a pause or a fetch sees it was aborted and lets go
+ for (const wake of this.resumeWaiters.splice(0)) wake();
+
+ this.stopAudio();
+ this.queue = this.pieces.slice(index);
+ // a piece opens with the space after the one before: a word right after it starts the piece
+ this.startAt = this.text.slice(piece.start, from).trim()
+ ? { piece, share: (from - piece.start) / Math.max(piece.end - piece.start, 1) }
+ : null;
+ this.spoken = from;
+ this.playing = false;
+ void this.pump();
+
+ return true;
+ }
+
+ /** Stops reading: the current piece, and what was still to come. */
+ stop(): void {
+ this.isPaused = false;
+
+ for (const wake of this.resumeWaiters.splice(0)) wake();
+
+ this.fetches.abort();
+ this.run.abort();
+ this.queue = [];
+ this.startAt = null;
+ this.fetched.clear();
+ this.stopAudio();
+ this.playing = false;
+ this.speaking = false;
+ this.idle = true;
+ }
+
+ /** The reply so far; `done` once it is complete. */
+ update(text: string, done: boolean): void {
+ this.text = text;
+ this.done = done;
+
+ // the first piece may end at a clause, so the voice starts sooner
+ for (const piece of readAloudPieces(text, this.queuedTo, done, this.queuedTo === 0)) {
+ this.pieces.push(piece);
+ this.queue.push(piece);
+ this.queuedTo = piece.end;
+ }
+
+ void this.pump();
+ }
+
+ private element(): HTMLAudioElement {
+ this.audio ??= playbackElement();
+
+ return this.audio;
+ }
+
+ private fetchPiece(piece: ReadAloudPiece): Promise {
+ let pending = this.fetched.get(piece);
+
+ if (!pending) {
+ // the model's silence around each piece is cut short, so the voice runs on
+ pending = this.options
+ .synthesize(piece.speech, this.fetches.signal)
+ .then((blob) => trimSilence(blob));
+ // a failed fetch is handled where it is played, and fetched again next time
+ pending.catch(() => this.fetched.delete(piece));
+ this.fetched.set(piece, pending);
+ }
+
+ return pending;
+ }
+
+ private finishIfDone(): void {
+ if (this.playing || this.queue.length > 0 || !this.done || this.queuedTo < this.text.length) {
+ return;
+ }
+
+ this.speaking = false;
+ this.spoken = this.text.length;
+
+ if (!this.idle) {
+ this.idle = true;
+ this.options.onIdle?.();
+ }
+ }
+
+ private async play(
+ piece: ReadAloudPiece,
+ blob: Blob,
+ signal: AbortSignal,
+ startShare = 0
+ ): Promise {
+ const audio = this.element();
+ const url = URL.createObjectURL(blob);
+
+ try {
+ await new Promise((resolve) => {
+ const follow = () => {
+ // a piece left behind by seek() or stop() no longer moves the text
+ if (signal.aborted) return;
+
+ if (audio.duration > 0 && Number.isFinite(audio.duration)) {
+ const share = Math.min(audio.currentTime / audio.duration, 1);
+
+ this.spoken = Math.max(
+ this.spoken,
+ piece.start + Math.round((piece.end - piece.start) * share)
+ );
+ }
+
+ this.frame = requestAnimationFrame(follow);
+ };
+ const start = () => {
+ if (signal.aborted) return;
+
+ // after seek(): from about where the word falls in the piece
+ if (startShare > 0 && Number.isFinite(audio.duration)) {
+ audio.currentTime = audio.duration * startShare;
+ }
+
+ audio.play().then(
+ () => {
+ if (!signal.aborted) this.frame = requestAnimationFrame(follow);
+ },
+ () => end()
+ );
+ };
+ const end = () => {
+ audio.removeEventListener('loadedmetadata', start);
+ audio.onended = null;
+ audio.onerror = null;
+ signal.removeEventListener('abort', end);
+ resolve();
+ };
+
+ audio.onended = end;
+ audio.onerror = end;
+ signal.addEventListener('abort', end);
+ audio.src = url;
+ // set after src: loading a source resets the rate
+ audio.preservesPitch = true;
+ audio.playbackRate = this.options.playbackRate?.() ?? 1;
+ audio.defaultPlaybackRate = audio.playbackRate;
+
+ if (startShare > 0) {
+ audio.addEventListener('loadedmetadata', start, { once: true });
+ } else {
+ start();
+ }
+ });
+ } finally {
+ if (this.frame !== null) cancelAnimationFrame(this.frame);
+
+ this.frame = null;
+ URL.revokeObjectURL(url);
+ }
+ }
+
+ private async pump(): Promise {
+ if (this.playing) return;
+
+ const piece = this.queue.shift();
+
+ if (!piece) {
+ this.finishIfDone();
+
+ return;
+ }
+
+ const { signal } = this.run;
+ const startShare = this.startAt?.piece === piece ? this.startAt.share : 0;
+
+ this.startAt = null;
+ this.playing = true;
+ this.speaking = true;
+
+ if (piece.speech && !this.isMuted) {
+ // the next pieces are fetched while this one plays
+ for (const next of this.queue.slice(0, 2)) {
+ if (next.speech) this.fetchPiece(next);
+ }
+
+ try {
+ const blob = await this.fetchPiece(piece);
+
+ // paused while fetching: the piece waits to be played
+ while (this.isPaused && !signal.aborted) {
+ await new Promise((resolve) => this.resumeWaiters.push(resolve));
+ }
+
+ if (!signal.aborted && !this.isMuted) await this.play(piece, blob, signal, startShare);
+ } catch (error) {
+ if (!signal.aborted) console.warn('Reading the reply aloud failed:', error);
+ }
+ }
+
+ if (signal.aborted) return;
+
+ this.spoken = Math.max(this.spoken, piece.end);
+ this.playing = false;
+ void this.pump();
+ }
+
+ /** Muting mid-piece: the voice stops, the text keeps up. */
+ private silence(): void {
+ this.stopAudio();
+ this.spoken = Math.max(this.spoken, this.queuedTo);
+ }
+
+ private stopAudio(): void {
+ if (this.frame !== null) cancelAnimationFrame(this.frame);
+
+ this.frame = null;
+
+ const audio = this.audio;
+
+ if (audio && !audio.paused) audio.pause();
+
+ audio?.dispatchEvent(new Event('ended'));
+ }
+}
diff --git a/ui/apps/chat/src/lib/voice/speech.ts b/ui/apps/chat/src/lib/voice/speech.ts
new file mode 100644
index 000000000000..0cf584965550
--- /dev/null
+++ b/ui/apps/chat/src/lib/voice/speech.ts
@@ -0,0 +1,49 @@
+import { VOICE_API_PATHS } from './constants';
+import { VoiceTranscriptionError } from './errors';
+import type { VoiceServerContext } from './types';
+
+export interface SynthesizeSpeechOptions {
+ /** Model id, required by a router server */
+ model?: string;
+ /** One of the model's voices (`voices` of GET /props); left out, its default */
+ voice?: string;
+ signal?: AbortSignal;
+}
+
+/**
+ * Speaks `text` with llama-server's POST /v1/audio/speech and returns the WAV.
+ * @throws VoiceTranscriptionError with the server's message when it refuses
+ */
+export async function synthesizeSpeech(
+ context: VoiceServerContext,
+ text: string,
+ options: SynthesizeSpeechOptions = {}
+): Promise {
+ const response = await (context.fetch ?? fetch)(context.resolveUrl(VOICE_API_PATHS.SPEECH), {
+ body: JSON.stringify({
+ input: text,
+ response_format: 'wav',
+ ...(options.model ? { model: options.model } : {}),
+ ...(options.voice ? { voice: options.voice } : {})
+ }),
+ headers: { ...context.getHeaders?.(), 'Content-Type': 'application/json' },
+ method: 'POST',
+ signal: options.signal
+ });
+
+ if (!response.ok) {
+ let message = `Speech failed (${response.status})`;
+
+ try {
+ const data = await response.json();
+
+ if (typeof data?.error?.message === 'string') message = data.error.message;
+ } catch {
+ // not JSON: keep the status
+ }
+
+ throw new VoiceTranscriptionError(message, response.status);
+ }
+
+ return await response.blob();
+}
diff --git a/ui/apps/chat/src/lib/voice/trim-silence.ts b/ui/apps/chat/src/lib/voice/trim-silence.ts
new file mode 100644
index 000000000000..83bc7ed93647
--- /dev/null
+++ b/ui/apps/chat/src/lib/voice/trim-silence.ts
@@ -0,0 +1,83 @@
+import { encodeWav } from './wav';
+
+/** Silence kept before the voice starts and after it ends, so pieces still breathe. */
+const LEAD_MS = 40;
+const TAIL_MS = 90;
+/** A sample under this share of the clip's loudest one is silence. */
+const SILENCE_SHARE = 0.03;
+
+/**
+ * A WAV with the silence a speech model puts around what it says cut down to a short
+ * breath, so pieces read one after another (a sentence, then the paragraph after an
+ * empty line) follow without long gaps. Anything but 16-bit PCM is returned as it is.
+ */
+export async function trimSilence(blob: Blob): Promise {
+ const bytes = await blob.arrayBuffer();
+ const pcm = readPcm16(bytes);
+
+ if (!pcm || pcm.channels !== 1) return blob;
+
+ const { sampleRate, samples } = pcm;
+
+ let loudest = 0;
+
+ for (let i = 0; i < samples.length; i++) loudest = Math.max(loudest, Math.abs(samples[i]));
+
+ if (loudest === 0) return blob;
+
+ const threshold = loudest * SILENCE_SHARE;
+
+ let first = 0;
+ let last = samples.length - 1;
+
+ while (first < samples.length && Math.abs(samples[first]) < threshold) first++;
+ while (last > first && Math.abs(samples[last]) < threshold) last--;
+
+ const start = Math.max(0, first - Math.round((sampleRate * LEAD_MS) / 1000));
+ const end = Math.min(samples.length, last + 1 + Math.round((sampleRate * TAIL_MS) / 1000));
+
+ if (start === 0 && end === samples.length) return blob;
+
+ return encodeWav(samples.slice(start, end), sampleRate);
+}
+
+/** The samples of a 16-bit PCM WAV, or null for any other format. */
+function readPcm16(
+ bytes: ArrayBuffer
+): { channels: number; sampleRate: number; samples: Int16Array } | null {
+ const view = new DataView(bytes);
+ const tag = (offset: number) =>
+ String.fromCharCode(...new Uint8Array(bytes, offset, Math.min(4, bytes.byteLength - offset)));
+
+ if (bytes.byteLength < 12 || tag(0) !== 'RIFF' || tag(8) !== 'WAVE') return null;
+
+ let channels = 0;
+ let sampleRate = 0;
+ let bits = 0;
+ let offset = 12;
+
+ while (offset + 8 <= bytes.byteLength) {
+ const id = tag(offset);
+ // a streamed WAV may leave the data size unknown (0xFFFFFFFF): it runs to the end
+ const size = Math.min(view.getUint32(offset + 4, true), bytes.byteLength - offset - 8);
+ const body = offset + 8;
+
+ if (id === 'fmt ') {
+ if (view.getUint16(body, true) !== 1) return null;
+
+ channels = view.getUint16(body + 2, true);
+ sampleRate = view.getUint32(body + 4, true);
+ bits = view.getUint16(body + 14, true);
+ } else if (id === 'data') {
+ if (bits !== 16 || !sampleRate) return null;
+
+ const samples = new Int16Array(bytes.slice(body, body + size - (size % 2)));
+
+ return { channels, sampleRate, samples };
+ }
+
+ offset = body + size + (size % 2);
+ }
+
+ return null;
+}
diff --git a/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts b/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts
index 395bde627cc5..058c7e02ed98 100644
--- a/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts
+++ b/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts
@@ -2,11 +2,15 @@
// what the microphone offers per model, the recording row, dictated text written into the
// message around what the user types, and a voice message whose transcript comes later.
-import { modelServerProps, seedStores } from '../stories/chat/form/fixtures';
+import { modelServerProps, routerModels, seedStores } from '../stories/chat/form/fixtures';
import { deferred, FakeVoiceRecorder } from '../stories/voice/fake-voice-recorder.svelte';
import ChatFormVoiceTestWrapper from './components/ChatFormVoiceTestWrapper.svelte';
import { EditorView } from '@codemirror/view';
-import { SETTINGS_KEYS, VOICE_MODE_MESSAGE } from '#lib/constants/index.ts';
+import {
+ SETTINGS_KEYS,
+ VOICE_CONVERSATION_MESSAGE,
+ VOICE_MODE_MESSAGE
+} from '#lib/constants/index.ts';
import { ServerRole, VoiceInputMode } from '#lib/enums/index.ts';
import { voiceTranscriptsStore } from '#lib/stores/index.ts';
import { parseFilesToMessageExtras } from '#lib/utils/browser-only.ts';
@@ -39,10 +43,37 @@ function seed(
});
}
-async function renderForm({ isLoading = false } = {}) {
+/** A router whose chat model (Qwen3) can't hear while another (Gemma) can, none of them loaded. */
+function seedRouter(transcriptionModel = '') {
+ const [chat, hearing] = routerModels;
+
+ restore = seedStores({
+ config: {
+ autoMicOnEmpty: true,
+ [SETTINGS_KEYS.VOICE_INPUT_MODE]: VoiceInputMode.AUTO,
+ [SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL]: transcriptionModel
+ },
+ modelProps: {
+ [chat.model]: modelServerProps(),
+ [hearing.model]: modelServerProps({
+ modalities: { audio: true, video: false, vision: false }
+ })
+ },
+ models: routerModels,
+ selectedModelId: chat.id,
+ serverRole: ServerRole.ROUTER
+ });
+}
+
+async function renderForm(options: { isLoading?: boolean; voiceMode?: boolean } = {}) {
const recorder = new FakeVoiceRecorder();
const state = $state({ files: [] as ChatUploadedFile[], sent: 0, value: '' });
- const screen = render(ChatFormVoiceTestWrapper, { isLoading, state, voiceRecorder: recorder });
+ const screen = render(ChatFormVoiceTestWrapper, {
+ isLoading: options.isLoading,
+ state,
+ voiceMode: options.voiceMode,
+ voiceRecorder: recorder
+ });
await tick();
@@ -141,6 +172,40 @@ describe('the microphone', () => {
});
});
+describe('voice mode', () => {
+ it('is the main button when the model hears', async () => {
+ seed(true);
+
+ const { screen } = await renderForm({ voiceMode: true });
+
+ await expect
+ .element(screen.getByRole('button', { name: VOICE_CONVERSATION_MESSAGE.open }).last())
+ .toBeInTheDocument();
+ });
+
+ it('leaves the lead to the microphone when nothing can transcribe what is said', async () => {
+ seedRouter();
+
+ const { screen } = await renderForm({ voiceMode: true });
+
+ // the microphone stays, saying why it can't record
+ await expect
+ .element(screen.getByRole('button', { name: VOICE_MODE_MESSAGE.unavailable }).last())
+ .toBeInTheDocument();
+ expect(screen.container.innerHTML).not.toContain(VOICE_CONVERSATION_MESSAGE.open);
+ });
+
+ it('is the main button again once a transcription model is chosen', async () => {
+ seedRouter(routerModels[1].model);
+
+ const { screen } = await renderForm({ voiceMode: true });
+
+ await expect
+ .element(screen.getByRole('button', { name: VOICE_CONVERSATION_MESSAGE.open }).last())
+ .toBeInTheDocument();
+ });
+});
+
describe('recording', () => {
it('asks for the raw microphone, and for the processing that is switched on', async () => {
seed(true, VoiceInputMode.DICTATION);
diff --git a/ui/apps/chat/tests/client/components/ChatFormVoiceTestWrapper.svelte b/ui/apps/chat/tests/client/components/ChatFormVoiceTestWrapper.svelte
index 34dfe4ce77ef..a286a23b6689 100644
--- a/ui/apps/chat/tests/client/components/ChatFormVoiceTestWrapper.svelte
+++ b/ui/apps/chat/tests/client/components/ChatFormVoiceTestWrapper.svelte
@@ -9,9 +9,11 @@
/** The form's message and files, kept in step with it for the test to read, and how often it sent. */
state: { files: ChatUploadedFile[]; sent?: number; value: string };
voiceRecorder: VoiceRecorderLike;
+ /** Offer voice mode, as the chat screen does. */
+ voiceMode?: boolean;
}
- let { isLoading = false, state, voiceRecorder }: Props = $props();
+ let { isLoading = false, state, voiceMode = false, voiceRecorder }: Props = $props();
@@ -20,6 +22,7 @@
bind:value={state.value}
{isLoading}
onSubmit={() => (state.sent = (state.sent ?? 0) + 1)}
+ {voiceMode}
{voiceRecorder}
/>
diff --git a/ui/apps/chat/tests/client/voice-reply-reader.svelte.test.ts b/ui/apps/chat/tests/client/voice-reply-reader.svelte.test.ts
new file mode 100644
index 000000000000..3149b5bbccd3
--- /dev/null
+++ b/ui/apps/chat/tests/client/voice-reply-reader.svelte.test.ts
@@ -0,0 +1,163 @@
+// Guards reading a reply aloud from a word the user picks: the voice moves there, back or
+// ahead, plays audio it already fetched without asking for it again, and starts inside a
+// sentence at about where the word falls in it.
+
+import { ReplyReader } from '#lib/voice/reply-reader.svelte.ts';
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
+
+const DURATION = 2;
+
+/** An audio element that "plays" a clip of DURATION seconds until finish() is called. */
+class FakeAudio extends EventTarget {
+ currentTime = 0;
+ defaultPlaybackRate = 1;
+ duration = Number.NaN;
+ static last: FakeAudio | null = null;
+ onended: (() => void) | null = null;
+ onerror: (() => void) | null = null;
+ paused = true;
+ playbackRate = 1;
+ preservesPitch = true;
+ /** What each play() started on: the clip and where in it */
+ starts: { src: string; time: number }[] = [];
+ private source = '';
+
+ get src() {
+ return this.source;
+ }
+
+ set src(value: string) {
+ this.source = value;
+ this.currentTime = 0;
+ this.duration = Number.NaN;
+ // metadata comes after the source is set, as in a browser
+ queueMicrotask(() => {
+ this.duration = DURATION;
+ this.dispatchEvent(new Event('loadedmetadata'));
+ });
+ }
+
+ constructor() {
+ super();
+ FakeAudio.last = this;
+ }
+
+ override dispatchEvent(event: Event): boolean {
+ if (event.type === 'ended') this.onended?.();
+
+ return super.dispatchEvent(event);
+ }
+
+ /** The clip plays to its end. */
+ finish() {
+ this.currentTime = this.duration;
+ this.paused = true;
+ this.onended?.();
+ }
+
+ pause() {
+ this.paused = true;
+ }
+
+ play() {
+ this.paused = false;
+ this.starts.push({ src: this.source, time: this.currentTime });
+
+ return Promise.resolve();
+ }
+}
+
+const REPLY = 'One two three. Four five six. Seven eight nine.';
+const at = (word: string) => REPLY.indexOf(word);
+const settle = () => new Promise((resolve) => setTimeout(resolve, 20));
+
+function reader() {
+ const synthesize = vi.fn(async (text: string) => new Blob([text], { type: 'audio/test' }));
+ const replyReader = new ReplyReader({ synthesize });
+
+ replyReader.reset();
+ replyReader.update(REPLY, true);
+
+ return { replyReader, synthesize };
+}
+
+/** The text of the clip that last started, and where in it. */
+function lastStart() {
+ const start = FakeAudio.last!.starts.at(-1)!;
+
+ return { text: clips.get(start.src), time: start.time };
+}
+
+const clips = new Map();
+
+beforeAll(() => {
+ vi.stubGlobal('Audio', FakeAudio);
+ vi.spyOn(URL, 'createObjectURL').mockImplementation((blob) => {
+ const url = `blob:test/${clips.size}`;
+
+ void (blob as Blob).text().then((text) => clips.set(url, text));
+
+ return url;
+ });
+ vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {});
+});
+
+afterAll(() => {
+ vi.unstubAllGlobals();
+ vi.restoreAllMocks();
+});
+
+describe('ReplyReader.seek', () => {
+ it('jumps ahead to a later sentence', async () => {
+ const { replyReader, synthesize } = reader();
+
+ await settle();
+ expect(lastStart().text).toBe('One two three.');
+
+ expect(replyReader.seek(at('Seven'))).toBe(true);
+ expect(replyReader.spoken).toBe(at('Seven'));
+ await settle();
+
+ expect(lastStart()).toEqual({ text: 'Seven eight nine.', time: 0 });
+ // the later sentences were fetched while the first played: nothing is asked for twice
+ expect(synthesize).toHaveBeenCalledTimes(3);
+ replyReader.stop();
+ });
+
+ it('goes back into a sentence already read, from about where the word is', async () => {
+ const { replyReader, synthesize } = reader();
+
+ await settle();
+ FakeAudio.last!.finish();
+ await settle();
+ expect(lastStart().text).toBe('Four five six.');
+
+ expect(replyReader.seek(at('three'))).toBe(true);
+ expect(replyReader.spoken).toBe(at('three'));
+ await settle();
+
+ const { text, time } = lastStart();
+
+ expect(text).toBe('One two three.');
+ expect(time).toBeGreaterThan(0.8);
+ expect(time).toBeLessThan(DURATION);
+ expect(synthesize).toHaveBeenCalledTimes(3);
+
+ // and reads on from there
+ FakeAudio.last!.finish();
+ await settle();
+ expect(lastStart()).toEqual({ text: 'Four five six.', time: 0 });
+ replyReader.stop();
+ });
+
+ it('has nowhere to go once muted or read to the end', async () => {
+ const { replyReader } = reader();
+
+ replyReader.muted = true;
+ expect(replyReader.seek(at('Four'))).toBe(false);
+
+ replyReader.muted = false;
+ replyReader.stop();
+ expect(replyReader.seek(at('Four'))).toBe(false);
+ });
+});
diff --git a/ui/apps/chat/tests/client/voice-spoken-text.svelte.test.ts b/ui/apps/chat/tests/client/voice-spoken-text.svelte.test.ts
new file mode 100644
index 000000000000..d78a69f1e35a
--- /dev/null
+++ b/ui/apps/chat/tests/client/voice-spoken-text.svelte.test.ts
@@ -0,0 +1,26 @@
+// Guards picking a word of a reply being read aloud: a click gives where the word starts,
+// and without a listener the words are plain text.
+
+import { VoiceSpokenText } from '@ggml-org/llama-ui/voice';
+import { describe, expect, it, vi } from 'vitest';
+import { page, userEvent } from 'vitest/browser';
+import { render } from 'vitest-browser-svelte';
+
+const TEXT = 'Read this. Then that.';
+
+describe('VoiceSpokenText word picks', () => {
+ it('gives where a clicked word starts', async () => {
+ const onWordSelect = vi.fn();
+
+ render(VoiceSpokenText, { onWordSelect, spoken: 4, text: TEXT });
+
+ await userEvent.click(page.getByText('Then'));
+ expect(onWordSelect).toHaveBeenCalledWith(TEXT.indexOf(' Then') + 1);
+ });
+
+ it('leaves the words alone without a listener', () => {
+ const { container } = render(VoiceSpokenText, { spoken: 4, text: TEXT });
+
+ expect(container.querySelector('[data-start]')).toBeNull();
+ });
+});
diff --git a/ui/apps/chat/tests/unit/voice-mode.test.ts b/ui/apps/chat/tests/unit/voice-mode.test.ts
index 7d2f09225f66..a5c39085f901 100644
--- a/ui/apps/chat/tests/unit/voice-mode.test.ts
+++ b/ui/apps/chat/tests/unit/voice-mode.test.ts
@@ -158,6 +158,7 @@ describe('voice settings', () => {
SETTINGS_KEYS.VOICE_NOISE_SUPPRESSION,
SETTINGS_KEYS.VOICE_ECHO_CANCELLATION,
SETTINGS_KEYS.VOICE_AUTO_GAIN,
+ SETTINGS_KEYS.VOICE_READING_SPEED,
SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE
]);
expect(section?.fields?.[0].type).toBe(SettingsFieldType.SELECT);
diff --git a/ui/apps/chat/tests/unit/voice-pauses.test.ts b/ui/apps/chat/tests/unit/voice-pauses.test.ts
index 9b2bfeda600b..188288958c6e 100644
--- a/ui/apps/chat/tests/unit/voice-pauses.test.ts
+++ b/ui/apps/chat/tests/unit/voice-pauses.test.ts
@@ -42,6 +42,19 @@ describe('PauseDetector', () => {
expect(hold(detector, 0.45, 600)).toBe(true);
});
+ it('hears speech from the first level when the room is known', () => {
+ // without a known floor, the first level is taken as the room's noise
+ const unknown = new PauseDetector({ intervalMs: INTERVAL });
+
+ unknown.push(0.6);
+ expect(unknown.heardSpeech).toBe(false);
+
+ const known = new PauseDetector({ floor: 0.1, intervalMs: INTERVAL });
+
+ known.push(0.6);
+ expect(known.heardSpeech).toBe(true);
+ });
+
it('cuts a long stretch of speech at its next quiet moment', () => {
const detector = new PauseDetector({ intervalMs: INTERVAL, maxSegmentMs: 2000 });
diff --git a/ui/apps/chat/tests/unit/voice-read-aloud.test.ts b/ui/apps/chat/tests/unit/voice-read-aloud.test.ts
new file mode 100644
index 000000000000..1df51a68a375
--- /dev/null
+++ b/ui/apps/chat/tests/unit/voice-read-aloud.test.ts
@@ -0,0 +1,59 @@
+// Guards how voice mode reads a streaming reply aloud: whole sentences as soon as they are
+// complete, the rest once the reply is done, without markdown marks, with offsets into the
+// reply as written so the text on screen can follow the voice.
+
+import { readAloudPieces, toSpeech } from '#lib/voice/index.ts';
+import { describe, expect, it } from 'vitest';
+
+describe('readAloudPieces', () => {
+ it('reads complete sentences while the reply streams', () => {
+ const text = 'Sure, here you go. The first one is';
+ const pieces = readAloudPieces(text, 0, false);
+
+ expect(pieces.map((piece) => piece.speech)).toEqual(['Sure, here you go.']);
+ expect(text.slice(pieces[0].start, pieces[0].end)).toBe('Sure, here you go.');
+ });
+
+ it('reads the rest once the reply is done, from where it left off', () => {
+ const text = 'Sure, here you go. The first one is short';
+ const [first] = readAloudPieces(text, 0, false);
+ const rest = readAloudPieces(text, first.end, true);
+
+ expect(rest.map((piece) => piece.speech)).toEqual(['The first one is short']);
+ expect(rest[0].end).toBe(text.length);
+ });
+
+ it('does not end a sentence at a list number, and reads lines on their own', () => {
+ const text = 'Jokes:\n1. Why are fish clever? They live in schools.\n2. Next';
+ // a bare line break is a piece with nothing to say: it only moves the reading on
+ const pieces = readAloudPieces(text, 0, true)
+ .map((piece) => piece.speech)
+ .filter(Boolean);
+
+ expect(pieces).toEqual(['Jokes:', 'Why are fish clever?', 'They live in schools.', 'Next']);
+ });
+});
+
+describe('the first piece', () => {
+ it('may end at a clause, so the voice starts before the sentence is done', () => {
+ const text = 'Quantum physics describes how very small things, like electrons and photons';
+ const [first] = readAloudPieces(text, 0, false, true);
+
+ expect(first.speech).toBe('Quantum physics describes how very small things,');
+ // later pieces wait for whole sentences
+ expect(readAloudPieces(text, first.end, false)).toEqual([]);
+ });
+
+ it('waits for the sentence when its first clause is short', () => {
+ expect(readAloudPieces('Sure, here are ten jokes about', 0, false, true)).toEqual([]);
+ });
+});
+
+describe('toSpeech', () => {
+ it('leaves out markdown marks', () => {
+ expect(toSpeech('## **Quantization.** Some `values`, see [the docs](https://x.y).')).toBe(
+ 'Quantization. Some values, see the docs.'
+ );
+ expect(toSpeech('```js\nconst a = 1;\n```')).toBe('');
+ });
+});
diff --git a/ui/apps/chat/tests/unit/voice-trim-silence.test.ts b/ui/apps/chat/tests/unit/voice-trim-silence.test.ts
new file mode 100644
index 000000000000..5cc4cb98cc13
--- /dev/null
+++ b/ui/apps/chat/tests/unit/voice-trim-silence.test.ts
@@ -0,0 +1,46 @@
+// Guards the silence cut from read-aloud clips: the speech model's quiet lead and tail
+// shrink to a short breath, so pieces follow each other without long gaps.
+
+import { encodeWav } from '#lib/voice/index.ts';
+import { trimSilence } from '#lib/voice/trim-silence.ts';
+import { describe, expect, it } from 'vitest';
+
+const RATE = 24_000;
+
+/** `silenceMs` of silence, `toneMs` of a tone, then `silenceMs` of silence again. */
+function clip(silenceMs: number, toneMs: number): Blob {
+ const silence = Math.round((RATE * silenceMs) / 1000);
+ const tone = Math.round((RATE * toneMs) / 1000);
+ const samples = new Int16Array(silence * 2 + tone);
+
+ for (let i = 0; i < tone; i++) {
+ samples[silence + i] = Math.round(Math.sin((2 * Math.PI * 440 * i) / RATE) * 12_000);
+ }
+
+ return encodeWav(samples, RATE);
+}
+
+/** Length of a 16-bit mono WAV in ms. */
+async function lengthMs(blob: Blob): Promise {
+ return ((blob.size - 44) / 2 / RATE) * 1000;
+}
+
+describe('trimSilence', () => {
+ it('cuts the silence around the voice down to a short breath', async () => {
+ const trimmed = await trimSilence(clip(300, 1000));
+
+ // 1000 ms of tone, with 40 ms kept before and 90 ms after it
+ expect(await lengthMs(trimmed)).toBeGreaterThan(1100);
+ expect(await lengthMs(trimmed)).toBeLessThan(1160);
+ });
+
+ it('leaves a clip without silence, a silent one and anything not WAV alone', async () => {
+ const tight = clip(0, 500);
+ const silent = encodeWav(new Int16Array(RATE), RATE);
+ const other = new Blob(['not a wav'], { type: 'audio/mpeg' });
+
+ expect(await trimSilence(tight)).toBe(tight);
+ expect(await trimSilence(silent)).toBe(silent);
+ expect(await trimSilence(other)).toBe(other);
+ });
+});
diff --git a/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte b/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte
index f9299cc1eac1..9e2aa8e31592 100644
--- a/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte
+++ b/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte
@@ -3,6 +3,7 @@
muted. Used for a transcript followed during playback (`spoken`, e.g. from spokenLength()), for a reply read aloud,
and for text that streams in while someone speaks (`live`: each new word brightens as it arrives).
Without `spoken` or `live` all of it reads as spoken. The words are plain text for screen readers.
+ With `onWordSelect`, a click or tap on a word gives where it starts, e.g. to read on from there.
-->
-
+
+{#each words as { start, word }, index (live ? index : start)}{word} {word} {/each}