+
diff --git a/ui/apps/chat/tests/stories/voice/fixtures.ts b/ui/apps/chat/tests/stories/voice/fixtures.ts
new file mode 100644
index 000000000000..ca9583245f86
--- /dev/null
+++ b/ui/apps/chat/tests/stories/voice/fixtures.ts
@@ -0,0 +1,100 @@
+/**
+ * Demo audio for the voice stories, made in the browser so no binary file sits in the repo: a WAV of a hummed tone
+ * that rises and falls like syllables, with its peaks.
+ */
+export interface DemoClip {
+ src: string;
+ durationMs: number;
+ peaks: number[];
+}
+
+const SAMPLE_RATE = 8000;
+const PEAK_COUNT = 64;
+
+/** Syllable-like loudness at time `t` (s): bursts of about 0.25 s with short gaps between words. */
+function envelope(t: number, seed: number): number {
+ const syllable = Math.max(Math.sin(t * Math.PI * 4 + seed), 0);
+ const word = Math.sin(t * Math.PI * 0.9 + seed * 2) > -0.6 ? 1 : 0.08;
+ const swell = 0.6 + 0.4 * Math.sin(t * 1.3 + seed);
+
+ return syllable * word * swell;
+}
+
+function wavBlob(samples: Float32Array): Blob {
+ const buffer = new ArrayBuffer(44 + samples.length * 2);
+ const view = new DataView(buffer);
+ const write = (offset: number, text: string) => {
+ for (let i = 0; i < text.length; i++) view.setUint8(offset + i, text.charCodeAt(i));
+ };
+
+ write(0, 'RIFF');
+ view.setUint32(4, 36 + samples.length * 2, true);
+ write(8, 'WAVE');
+ write(12, 'fmt ');
+ view.setUint32(16, 16, true);
+ view.setUint16(20, 1, true);
+ view.setUint16(22, 1, true);
+ view.setUint32(24, SAMPLE_RATE, true);
+ view.setUint32(28, SAMPLE_RATE * 2, true);
+ view.setUint16(32, 2, true);
+ view.setUint16(34, 16, true);
+ write(36, 'data');
+ view.setUint32(40, samples.length * 2, true);
+
+ for (let i = 0; i < samples.length; i++) {
+ view.setInt16(44 + i * 2, Math.max(-1, Math.min(1, samples[i])) * 0x7fff, true);
+ }
+
+ return new Blob([buffer], { type: 'audio/wav' });
+}
+
+/** The loudest sample of each of `count` buckets, scaled so the loudest bucket is 1. */
+export function peaksOf(samples: Float32Array, count = PEAK_COUNT): number[] {
+ const size = Math.max(Math.floor(samples.length / count), 1);
+ const peaks = Array.from({ length: count }, (_, bucket) => {
+ let peak = 0;
+
+ for (let i = bucket * size; i < Math.min((bucket + 1) * size, samples.length); i++) {
+ peak = Math.max(peak, Math.abs(samples[i]));
+ }
+
+ return peak;
+ });
+ const loudest = Math.max(...peaks, 1e-6);
+
+ return peaks.map((peak) => peak / loudest);
+}
+
+const cache = new Map();
+
+/** A clip of `seconds`, the same for the same arguments. */
+export function demoClip(seconds: number, seed = 1): DemoClip {
+ const key = `${seconds}:${seed}`;
+ const cached = cache.get(key);
+
+ if (cached) return cached;
+
+ const samples = new Float32Array(Math.round(seconds * SAMPLE_RATE));
+
+ for (let i = 0; i < samples.length; i++) {
+ const t = i / SAMPLE_RATE;
+ const pitch = 180 + 40 * Math.sin(t * 2 + seed);
+
+ samples[i] = 0.5 * envelope(t, seed) * Math.sin(2 * Math.PI * pitch * t);
+ }
+
+ const clip = {
+ durationMs: seconds * 1000,
+ peaks: peaksOf(samples),
+ src: URL.createObjectURL(wavBlob(samples))
+ };
+
+ cache.set(key, clip);
+
+ return clip;
+}
+
+/** One live input level (0..1) at time `t` (s), as a microphone meter would report it while someone speaks. */
+export function demoLevel(t: number, seed = 1): number {
+ return Math.min(envelope(t, seed) * (0.75 + 0.25 * Math.random()), 1);
+}
From a62b902dcfad0d008a14d7fe560ee7c1fcfbf2e8 Mon Sep 17 00:00:00 2001
From: Aleksander Grygier
Date: Sun, 15 Nov 2026 21:25:15 +0100
Subject: [PATCH 04/19] ui: store voice messages with their transcript and send
it to models that cannot hear
A voice message recorded in the chat form is an audio attachment like an
uploaded file, plus what the app needs to draw and read it. The audio
attachment gets four optional fields: voice (recorded in the app),
transcript, peaks and durationMs. Stored rows, export and import carry
them as they are, so no migration is needed and older messages keep
working.
The transcript is for reading. A model with audio input still gets the
audio alone; a model without it (the user switched models later) gets the
transcript as a labelled text part in place of the audio. When the
model's modalities are not known yet, or there is no transcript, the
audio goes as before.
A transcript often comes back after the message was sent. The new
voiceTranscriptsStore follows each pending transcript by the clip's name,
hands it to a send that happens after it arrived, and otherwise writes it
into the stored attachment of the message it went out with, in the open
chat or, if the user left it, in the database.
Copying a message now includes its voice transcript.
Assisted-by: Claude (Opus 5.5 Medium)
---
.../chat/src/lib/enums/attachment.enums.ts | 3 +-
ui/apps/chat/src/lib/services/chat.service.ts | 53 ++++++-
.../chat/src/lib/services/database.service.ts | 10 ++
.../src/lib/stores/agentic/index.svelte.ts | 6 +-
.../chat/src/lib/stores/chat/index.svelte.ts | 4 +-
ui/apps/chat/src/lib/stores/index.ts | 3 +
.../lib/stores/voice-transcripts.svelte.ts | 142 ++++++++++++++++++
ui/apps/chat/src/lib/types/chat.d.ts | 8 +
ui/apps/chat/src/lib/types/database.d.ts | 7 +
ui/apps/chat/src/lib/types/index.ts | 1 +
ui/apps/chat/src/lib/utils/clipboard.ts | 10 ++
.../src/lib/utils/convert-files-to-extra.ts | 11 +-
.../tests/unit/voice-message-send.test.ts | 119 +++++++++++++++
13 files changed, 364 insertions(+), 13 deletions(-)
create mode 100644 ui/apps/chat/src/lib/stores/voice-transcripts.svelte.ts
create mode 100644 ui/apps/chat/tests/unit/voice-message-send.test.ts
diff --git a/ui/apps/chat/src/lib/enums/attachment.enums.ts b/ui/apps/chat/src/lib/enums/attachment.enums.ts
index 5f096b974486..035fdac9cdd0 100644
--- a/ui/apps/chat/src/lib/enums/attachment.enums.ts
+++ b/ui/apps/chat/src/lib/enums/attachment.enums.ts
@@ -50,5 +50,6 @@ export enum AttachmentLabel {
FILE = 'File',
MCP_PROMPT = 'MCP Prompt',
MCP_RESOURCE = 'MCP Resource',
- PDF_FILE = 'PDF File'
+ PDF_FILE = 'PDF File',
+ VOICE_TRANSCRIPT = 'Voice message transcript'
}
diff --git a/ui/apps/chat/src/lib/services/chat.service.ts b/ui/apps/chat/src/lib/services/chat.service.ts
index 848bd06efb39..38c7eba144b1 100644
--- a/ui/apps/chat/src/lib/services/chat.service.ts
+++ b/ui/apps/chat/src/lib/services/chat.service.ts
@@ -50,6 +50,12 @@ import { getAuthHeaders, getJsonHeaders } from '#lib/utils/api-headers.ts';
import { formatAttachmentText } from '#lib/utils/formatters.ts';
import { streamIdentity } from '#lib/utils/stream-identity.ts';
+/** How stored messages are shaped for the model a request goes to. */
+export interface ApiMessageConversionOptions {
+ /** False when the model has no audio input: voice messages go as their transcript. */
+ audioInput?: boolean;
+}
+
interface ResumableStreamState {
bytesReceived: number;
updatedAt: number;
@@ -133,13 +139,24 @@ export class ChatService {
}
}
+ /**
+ * How messages are shaped for `model`: whether it takes audio, known only once its
+ * modalities are. Unknown modalities keep the audio, as before voice messages.
+ */
+ static conversionOptionsFor(model?: string | null): ApiMessageConversionOptions {
+ const modalities = modelsStore.props.getModelModalities(model ?? '');
+
+ return modalities ? { audioInput: modalities.audio } : {};
+ }
+
/**
* Converts a database message with attachments to API chat message format.
* Processes various attachment types (images, text files, PDFs) and formats them
* as content parts suitable for the chat completion API.
*/
static async convertDbMessageToApiChatMessageData(
- message: DatabaseMessage & { extra?: DatabaseMessageExtra[] }
+ message: DatabaseMessage & { extra?: DatabaseMessageExtra[] },
+ options: ApiMessageConversionOptions = {}
): Promise {
// Handle tool result messages (role: 'tool')
if (message.role === MessageRole.TOOL && message.toolCallId) {
@@ -231,6 +248,20 @@ export class ChatService {
);
for (const audio of audioFiles) {
+ // a model that cannot hear reads what was said instead; one that can gets the audio alone
+ if (options.audioInput === false && audio.transcript) {
+ contentParts.push({
+ text: formatAttachmentText(
+ AttachmentLabel.VOICE_TRANSCRIPT,
+ audio.name,
+ audio.transcript
+ ),
+ type: ContentPartType.TEXT
+ });
+
+ continue;
+ }
+
contentParts.push({
input_audio: {
data: audio.base64Data,
@@ -816,14 +847,18 @@ export class ChatService {
* messages. Shared by sendMessage, preEncode and the agentic flow.
*/
static async normalizeMessagesForApi(
- messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[]
+ messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[],
+ model?: string | null
): Promise {
+ const options = ChatService.conversionOptionsFor(model);
+
return (
await Promise.all(
messages.map((msg) => {
if ('id' in msg && 'convId' in msg && 'timestamp' in msg) {
return ChatService.convertDbMessageToApiChatMessageData(
- msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }
+ msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] },
+ options
);
}
@@ -855,8 +890,10 @@ export class ChatService {
signal?: AbortSignal,
tools?: OpenAIToolDefinition[]
): Promise {
- const normalizedMessages: ApiChatMessageData[] =
- await ChatService.normalizeMessagesForApi(messages);
+ const normalizedMessages: ApiChatMessageData[] = await ChatService.normalizeMessagesForApi(
+ messages,
+ model
+ );
const requestBody: Record = {
messages: normalizedMessages.map((msg: ApiChatMessageData) => {
const mapped: Record = {
@@ -1106,8 +1143,10 @@ export class ChatService {
xtc_probability,
xtc_threshold
} = options;
- const normalizedMessages: ApiChatMessageData[] =
- await ChatService.normalizeMessagesForApi(messages);
+ const normalizedMessages: ApiChatMessageData[] = await ChatService.normalizeMessagesForApi(
+ messages,
+ options.model
+ );
// Filter out image attachments if the model doesn't support vision
if (options.model && !modelsStore.props.modelSupportsVision(options.model)) {
diff --git a/ui/apps/chat/src/lib/services/database.service.ts b/ui/apps/chat/src/lib/services/database.service.ts
index de8ed4701e12..e77b97a964ca 100644
--- a/ui/apps/chat/src/lib/services/database.service.ts
+++ b/ui/apps/chat/src/lib/services/database.service.ts
@@ -572,6 +572,16 @@ export class DatabaseService {
return result;
}
+ /**
+ * Gets the messages written at or after `timestamp`, in any conversation.
+ *
+ * @param timestamp - Epoch ms
+ * @returns Messages, oldest first
+ */
+ static async getMessagesSince(timestamp: number): Promise {
+ return await db[IDXDB_TABLES.messages].where('timestamp').aboveOrEqual(timestamp).toArray();
+ }
+
/**
* Imports multiple conversations and their messages.
* Skips conversations that already exist.
diff --git a/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts b/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts
index b607a1373c56..2eb93ad3aa98 100644
--- a/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts
+++ b/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts
@@ -347,8 +347,10 @@ class AgenticStore {
console.log(`[AgenticStore] Starting agentic flow with ${tools.length} tools`);
- const normalizedMessages: ApiChatMessageData[] =
- await ChatService.normalizeMessagesForApi(messages);
+ const normalizedMessages: ApiChatMessageData[] = await ChatService.normalizeMessagesForApi(
+ messages,
+ options.model
+ );
this.updateSession(conversationId, {
currentTurn: 0,
diff --git a/ui/apps/chat/src/lib/stores/chat/index.svelte.ts b/ui/apps/chat/src/lib/stores/chat/index.svelte.ts
index ef8a1110035b..20c95be39867 100644
--- a/ui/apps/chat/src/lib/stores/chat/index.svelte.ts
+++ b/ui/apps/chat/src/lib/stores/chat/index.svelte.ts
@@ -1480,7 +1480,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost {
const allMessages = await conversationsStore.getConversationMessages(convId);
const branchPath = [...filterByLeafNodeId(allMessages, leafId, false)];
const request = await agenticStore.requestFor(
- await ChatService.normalizeMessagesForApi(sliceAtLastCompaction(branchPath)),
+ await ChatService.normalizeMessagesForApi(sliceAtLastCompaction(branchPath), model),
this.toolPolicy()
);
@@ -1646,7 +1646,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost {
// message shaping, cut before the preserved steps: the server
// reuses the prefix it cached and only the instruction is new.
const request = await agenticStore.requestFor(
- await ChatService.normalizeMessagesForApi(source.slice(0, cut)),
+ await ChatService.normalizeMessagesForApi(source.slice(0, cut), effectiveModel),
this.toolPolicy()
);
const configValue = settingsStore.config;
diff --git a/ui/apps/chat/src/lib/stores/index.ts b/ui/apps/chat/src/lib/stores/index.ts
index e98a48d06c45..345b5630afed 100644
--- a/ui/apps/chat/src/lib/stores/index.ts
+++ b/ui/apps/chat/src/lib/stores/index.ts
@@ -61,6 +61,9 @@ export { permissionsStore } from './permissions.svelte';
// TOOLS
export { toolsStore } from './tools.svelte';
+// VOICE (transcripts of voice messages on their way)
+export { voiceTranscriptsStore } from './voice-transcripts.svelte';
+
// ENVIRONMENT / META
export { versionStore } from './version.svelte';
diff --git a/ui/apps/chat/src/lib/stores/voice-transcripts.svelte.ts b/ui/apps/chat/src/lib/stores/voice-transcripts.svelte.ts
new file mode 100644
index 000000000000..d0395a4f123b
--- /dev/null
+++ b/ui/apps/chat/src/lib/stores/voice-transcripts.svelte.ts
@@ -0,0 +1,142 @@
+/**
+ * voiceTranscriptsStore - Transcripts of voice messages that are on their way
+ *
+ * A voice message is attached as soon as the recording stops, while its transcript
+ * can take a few seconds more. This keeps each pending transcript by the clip's name
+ * (unique per recording), so the chip in the form and the sent message show its
+ * state, a send picks it up once it is there, and a message sent before it arrived
+ * gets it written into its stored attachment afterwards. Kept in memory only.
+ */
+
+import { AttachmentType } from '#lib/enums/index.ts';
+import { DatabaseService } from '#lib/services/database.service.ts';
+// direct imports between stores, not via the barrel, to avoid circular deps
+import { conversationsStore } from '#lib/stores/conversations/index.svelte.ts';
+import type {
+ DatabaseMessage,
+ DatabaseMessageExtra,
+ DatabaseMessageExtraAudioFile
+} from '#lib/types/index.ts';
+import { SvelteMap } from 'svelte/reactivity';
+
+export type VoiceTranscriptState =
+ | { status: 'transcribing' }
+ | { status: 'done'; transcript: string }
+ | { status: 'error'; message: string };
+
+/** How long a transcript that came back keeps looking for the message it belongs to. */
+const APPLY_RETRY_DELAYS_MS = [500, 1500, 4000];
+
+function isPendingClip(extra: DatabaseMessageExtra, name: string): boolean {
+ return (
+ extra.type === AttachmentType.AUDIO &&
+ extra.name === name &&
+ extra.voice === true &&
+ extra.transcript === undefined
+ );
+}
+
+/** The extras with the clip's transcript filled in, or null when none of them is that clip. */
+export function withTranscript(
+ extras: DatabaseMessageExtra[] | undefined,
+ name: string,
+ transcript: string
+): DatabaseMessageExtra[] | null {
+ if (!extras?.some((extra) => isPendingClip(extra, name))) return null;
+
+ return extras.map((extra) =>
+ isPendingClip(extra, name)
+ ? ({ ...(extra as DatabaseMessageExtraAudioFile), transcript } as DatabaseMessageExtra)
+ : extra
+ );
+}
+
+class VoiceTranscriptsStore {
+ private entries = new SvelteMap();
+ private recordedAt = new Map();
+
+ /** Drops a clip removed from the form before it was sent. */
+ forget(name: string): void {
+ this.entries.delete(name);
+ this.recordedAt.delete(name);
+ }
+
+ /** The state of a clip's transcript, undefined when none was asked for in this session. */
+ get(name: string): VoiceTranscriptState | undefined {
+ return this.entries.get(name);
+ }
+
+ /** Follows a clip's transcript, and writes it into the message it went out with. */
+ track(name: string, transcript: Promise): void {
+ this.entries.set(name, { status: 'transcribing' });
+ this.recordedAt.set(name, Date.now());
+
+ transcript.then(
+ (text) => {
+ if (!this.entries.has(name)) return;
+
+ this.entries.set(name, { status: 'done', transcript: text });
+ void this.applyToMessage(name, text);
+ },
+ (error: unknown) => {
+ if (!this.entries.has(name)) return;
+
+ console.warn('Voice message transcription failed:', error);
+ this.entries.set(name, {
+ message: error instanceof Error ? error.message : String(error),
+ status: 'error'
+ });
+ }
+ );
+ }
+
+ /** The transcript once it is there. */
+ transcriptOf(name: string): string | undefined {
+ const entry = this.entries.get(name);
+
+ return entry?.status === 'done' ? entry.transcript : undefined;
+ }
+
+ private async applyToMessage(name: string, transcript: string): Promise {
+ for (const delay of [0, ...APPLY_RETRY_DELAYS_MS]) {
+ if (delay) await new Promise((resolve) => setTimeout(resolve, delay));
+
+ if (await this.writeTranscript(name, transcript)) return;
+ }
+ // not sent (still in the form, or dropped): a send reads it from here
+ }
+
+ private async writeTranscript(name: string, transcript: string): Promise {
+ const active = (conversationsStore.activeMessages as DatabaseMessage[]).find((message) =>
+ message.extra?.some((extra) => isPendingClip(extra, name))
+ );
+
+ if (active) {
+ // plain objects: IndexedDB cannot clone the state proxies
+ const extra = withTranscript($state.snapshot(active.extra), name, transcript)!;
+
+ await DatabaseService.updateMessage(active.id, { extra });
+ conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(active.id), {
+ extra
+ });
+
+ return true;
+ }
+
+ // the chat was left before the transcript came back: the message is newer than the clip
+ const since = this.recordedAt.get(name) ?? Date.now();
+ const stored = (await DatabaseService.getMessagesSince(since)).find(
+ (message: DatabaseMessage) => message.extra?.some((extra) => isPendingClip(extra, name))
+ );
+
+ if (!stored) return false;
+
+ await DatabaseService.updateMessage(stored.id, {
+ extra: withTranscript(stored.extra, name, transcript)!
+ });
+
+ return true;
+ }
+}
+
+export const voiceTranscriptsStore = new VoiceTranscriptsStore();
diff --git a/ui/apps/chat/src/lib/types/chat.d.ts b/ui/apps/chat/src/lib/types/chat.d.ts
index eca0dbec87f8..a72a8dfff984 100644
--- a/ui/apps/chat/src/lib/types/chat.d.ts
+++ b/ui/apps/chat/src/lib/types/chat.d.ts
@@ -48,6 +48,14 @@ export interface ChatUploadedFile {
};
isLoading?: boolean;
loadError?: string;
+ /** A voice message recorded in the chat form. */
+ voice?: ChatUploadedVoice;
+}
+
+/** What a voice message recorded in the chat form carries besides its WAV file. */
+export interface ChatUploadedVoice {
+ durationMs: number;
+ peaks: number[];
}
export interface ChatAttachmentDisplayItem {
diff --git a/ui/apps/chat/src/lib/types/database.d.ts b/ui/apps/chat/src/lib/types/database.d.ts
index 50f54dc0b556..e47d60fffb7c 100644
--- a/ui/apps/chat/src/lib/types/database.d.ts
+++ b/ui/apps/chat/src/lib/types/database.d.ts
@@ -38,6 +38,13 @@ export interface DatabaseMessageExtraAudioFile {
size?: number;
base64Data: string;
mimeType: string;
+ /** Recorded in the app as a voice message, as opposed to an uploaded audio file. */
+ voice?: true;
+ /** What was said, for reading; a model without audio input gets it in place of the audio. */
+ transcript?: string;
+ /** Normalised 0..1 waveform of the clip; computed from the audio when left out. */
+ peaks?: number[];
+ durationMs?: number;
}
export interface DatabaseMessageExtraVideoFile {
diff --git a/ui/apps/chat/src/lib/types/index.ts b/ui/apps/chat/src/lib/types/index.ts
index 68cd6e92f1e7..bfed7569c17f 100644
--- a/ui/apps/chat/src/lib/types/index.ts
+++ b/ui/apps/chat/src/lib/types/index.ts
@@ -57,6 +57,7 @@ export type {
export type {
AttachmentMenuItem,
ChatUploadedFile,
+ ChatUploadedVoice,
ChatAttachmentDisplayItem,
ChatMessageSiblingInfo,
ChatMessageActions,
diff --git a/ui/apps/chat/src/lib/utils/clipboard.ts b/ui/apps/chat/src/lib/utils/clipboard.ts
index a5c75e873317..c2b05985e0e0 100644
--- a/ui/apps/chat/src/lib/utils/clipboard.ts
+++ b/ui/apps/chat/src/lib/utils/clipboard.ts
@@ -110,6 +110,16 @@ export function formatMessageForClipboard(
extras?: DatabaseMessageExtra[],
asPlainText: boolean = false
): string {
+ // what was said in a voice message reads as the message's text
+ const transcripts =
+ extras?.flatMap((extra) =>
+ extra.type === AttachmentType.AUDIO && extra.transcript ? [extra.transcript] : []
+ ) ?? [];
+
+ if (transcripts.length > 0) {
+ content = [...transcripts, content].filter((part) => part.trim()).join('\n\n');
+ }
+
// Filter text-like attachments (TEXT, LEGACY_CONTEXT, MCP_PROMPT, and MCP_RESOURCE types)
const textAttachments =
extras?.filter(
diff --git a/ui/apps/chat/src/lib/utils/convert-files-to-extra.ts b/ui/apps/chat/src/lib/utils/convert-files-to-extra.ts
index 43d29f43c119..4e50b089c5bc 100644
--- a/ui/apps/chat/src/lib/utils/convert-files-to-extra.ts
+++ b/ui/apps/chat/src/lib/utils/convert-files-to-extra.ts
@@ -6,6 +6,7 @@ import { SETTINGS_KEYS } from '#lib/constants/index.ts';
import { AttachmentType, FileTypeCategory, SpecialFileType } from '#lib/enums/index.ts';
import { modelsStore } from '#lib/stores/models/index.svelte.ts';
import { settingsStore } from '#lib/stores/settings/index.svelte.ts';
+import { voiceTranscriptsStore } from '#lib/stores/voice-transcripts.svelte.ts';
import type {
ChatUploadedFile,
DatabaseMessageExtra,
@@ -83,13 +84,21 @@ export async function parseFilesToMessageExtras(
// Process audio files (MP3 and WAV)
try {
const base64Data = await readFileAsBase64(file.file);
+ // a voice message keeps its waveform, and its transcript once it is there
+ const transcript = file.voice ? voiceTranscriptsStore.transcriptOf(file.name) : undefined;
extras.push({
base64Data: base64Data,
mimeType: file.type,
name: file.name,
size: file.size,
- type: AttachmentType.AUDIO
+ type: AttachmentType.AUDIO,
+ ...(file.voice && {
+ durationMs: file.voice.durationMs,
+ peaks: file.voice.peaks,
+ voice: true as const,
+ ...(transcript !== undefined && { transcript })
+ })
});
} catch (error) {
console.error(`Failed to process audio file ${file.name}:`, error);
diff --git a/ui/apps/chat/tests/unit/voice-message-send.test.ts b/ui/apps/chat/tests/unit/voice-message-send.test.ts
new file mode 100644
index 000000000000..36858e504202
--- /dev/null
+++ b/ui/apps/chat/tests/unit/voice-message-send.test.ts
@@ -0,0 +1,119 @@
+// Guards how a voice message goes to a model: the audio alone to a model with audio input,
+// its transcript as text to one without, and the copied message carrying the transcript.
+
+import { AttachmentType, ContentPartType, MessageRole, MessageType } from '#lib/enums/index.ts';
+import { ChatService } from '#lib/services/chat.service.ts';
+import { modelsStore } from '#lib/stores/models/index.svelte.ts';
+import { withTranscript } from '#lib/stores/voice-transcripts.svelte.ts';
+import type { DatabaseMessage, DatabaseMessageExtra } from '#lib/types/index.ts';
+import { formatMessageForClipboard } from '#lib/utils/clipboard.ts';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+const voiceClip: DatabaseMessageExtra = {
+ base64Data: 'UklGRg==',
+ durationMs: 2400,
+ mimeType: 'audio/wav',
+ name: 'voice-1.wav',
+ peaks: [0.2, 1, 0.4],
+ transcript: 'What is the weather like on Mars?',
+ type: AttachmentType.AUDIO,
+ voice: true
+};
+
+function userMessage(extra: DatabaseMessageExtra[], content = ''): DatabaseMessage {
+ return {
+ children: [],
+ content,
+ convId: 'c1',
+ extra,
+ id: 'm1',
+ parent: null,
+ role: MessageRole.USER,
+ timestamp: 0,
+ type: MessageType.TEXT
+ } as DatabaseMessage;
+}
+
+afterEach(() => {
+ vi.restoreAllMocks();
+});
+
+describe('sending a voice message', () => {
+ it('sends the audio alone to a model with audio input', async () => {
+ const result = await ChatService.convertDbMessageToApiChatMessageData(
+ userMessage([voiceClip]),
+ { audioInput: true }
+ );
+ const parts = result.content as { type: string; text?: string }[];
+
+ expect(parts).toHaveLength(1);
+ expect(parts[0].type).toBe(ContentPartType.INPUT_AUDIO);
+ expect(JSON.stringify(parts)).not.toContain('Mars');
+ });
+
+ it('sends the transcript as text to a model without audio input', async () => {
+ const result = await ChatService.convertDbMessageToApiChatMessageData(
+ userMessage([voiceClip], 'Answer briefly.'),
+ { audioInput: false }
+ );
+ const parts = result.content as { type: string; text?: string }[];
+
+ expect(parts.map((part) => part.type)).toEqual([ContentPartType.TEXT, ContentPartType.TEXT]);
+ expect(parts[0].text).toContain('Voice message transcript');
+ expect(parts[0].text).toContain('What is the weather like on Mars?');
+ expect(parts[1].text).toBe('Answer briefly.');
+ });
+
+ it('keeps the audio when there is no transcript to send instead', async () => {
+ const { transcript: _transcript, ...withoutTranscript } = voiceClip as {
+ transcript?: string;
+ } & DatabaseMessageExtra;
+ const result = await ChatService.convertDbMessageToApiChatMessageData(
+ userMessage([withoutTranscript as DatabaseMessageExtra]),
+ { audioInput: false }
+ );
+
+ expect((result.content as { type: string }[])[0].type).toBe(ContentPartType.INPUT_AUDIO);
+ });
+
+ it('keeps the audio while the model modalities are unknown', () => {
+ vi.spyOn(modelsStore.props, 'getModelModalities').mockReturnValue(null);
+
+ expect(ChatService.conversionOptionsFor('some-model')).toEqual({});
+ });
+
+ it('reads the audio input from the model modalities', () => {
+ vi.spyOn(modelsStore.props, 'getModelModalities').mockReturnValue({
+ audio: false,
+ video: false,
+ vision: true
+ });
+
+ expect(ChatService.conversionOptionsFor('text-model')).toEqual({ audioInput: false });
+ });
+});
+
+describe('a transcript that arrives after the send', () => {
+ it('fills in the clip it belongs to only', () => {
+ const pending = { ...voiceClip, transcript: undefined } as DatabaseMessageExtra;
+ const other = { ...pending, name: 'voice-2.wav' } as DatabaseMessageExtra;
+ const next = withTranscript([pending, other], 'voice-1.wav', 'Hello');
+
+ expect(next?.[0]).toMatchObject({ name: 'voice-1.wav', transcript: 'Hello' });
+ expect(next?.[1]).toBe(other);
+ });
+
+ it('leaves a message without that clip alone', () => {
+ expect(withTranscript([voiceClip], 'voice-1.wav', 'Hello')).toBeNull();
+ expect(withTranscript(undefined, 'voice-1.wav', 'Hello')).toBeNull();
+ });
+});
+
+describe('copying a voice message', () => {
+ it('copies the transcript as the text', () => {
+ expect(formatMessageForClipboard('', [voiceClip])).toBe('What is the weather like on Mars?');
+ expect(formatMessageForClipboard('Briefly.', [voiceClip], true)).toBe(
+ 'What is the weather like on Mars?\n\nBriefly.'
+ );
+ });
+});
From a93442fc990e0d5d813dded6368dcdf0fe713eb7 Mon Sep 17 00:00:00 2001
From: Aleksander Grygier
Date: Sun, 15 Nov 2026 21:27:50 +0100
Subject: [PATCH 05/19] ui: add Voice settings and resolve what the microphone
does per model
The user should not have to think about which kind of voice input is in
use, so the app decides it from the conversation's model and two
settings. resolveVoiceMode is a small pure function: with Voice input on
Automatic and a chat model that has audio input, a recording is a voice
message, transcribed by that same model so a router swaps nothing.
Otherwise it is dictation, with the chat model when it can hear, else,
in router mode, with the chosen transcription model. When no model can
transcribe, the microphone is unavailable and says why.
Settings get a Voice section of their own rather than more rows in
General, which is already long: these four belong together and one of
them lists the server's models.
- Voice input: Automatic (default) or Always dictate.
- Transcription model: Same as chat model (default) or, in router mode,
any model with audio input; on a single-model server it only says
whether that model can transcribe. It notes that dictating with
another model loads it beside the chat model.
- Transcribe voice messages: on by default.
- Transcription language: empty for automatic.
The keys live in the settings registry like the others, so defaults,
storage, reset and export need nothing new. The model picker is drawn by
SettingsChatVoiceTab, the way Extensions and Tools draw their sections.
Assisted-by: Claude (Opus 5.5 Medium)
---
.../settings/SettingsChat/SettingsChat.svelte | 7 +
.../SettingsChat/SettingsChatVoiceTab.svelte | 109 +++++++++++++++
.../src/lib/components/app/settings/index.ts | 9 ++
ui/apps/chat/src/lib/constants/index.ts | 1 +
.../lib/constants/settings-keys.constants.ts | 5 +
.../src/lib/constants/settings.constants.ts | 58 +++++++-
.../chat/src/lib/constants/voice.constants.ts | 17 +++
ui/apps/chat/src/lib/enums/index.ts | 2 +
ui/apps/chat/src/lib/enums/voice.enums.ts | 17 +++
ui/apps/chat/src/lib/utils/index.ts | 8 ++
ui/apps/chat/src/lib/utils/voice-mode.ts | 71 ++++++++++
.../settings/SettingsSections.stories.svelte | 2 +
ui/apps/chat/tests/unit/voice-mode.test.ts | 124 ++++++++++++++++++
13 files changed, 425 insertions(+), 5 deletions(-)
create mode 100644 ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChatVoiceTab.svelte
create mode 100644 ui/apps/chat/src/lib/constants/voice.constants.ts
create mode 100644 ui/apps/chat/src/lib/enums/voice.enums.ts
create mode 100644 ui/apps/chat/src/lib/utils/voice-mode.ts
create mode 100644 ui/apps/chat/tests/unit/voice-mode.test.ts
diff --git a/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte b/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte
index 1ced25e50f2f..4c1e3281ef9b 100644
--- a/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte
+++ b/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte
@@ -7,6 +7,7 @@
SettingsChatImportExportTab,
SettingsChatMobileHeader,
SettingsChatToolsTab,
+ SettingsChatVoiceTab,
SettingsFooter,
SettingsResetButton
} from '#lib/components/app/settings/index.ts';
@@ -181,6 +182,12 @@
{:else if currentSection.slug === SETTINGS_SECTION_SLUGS.IMPORT_EXPORT}
+ {:else if currentSection.slug === SETTINGS_SECTION_SLUGS.VOICE && currentSection.fields}
+
{:else if currentSection.fields}
+ {SETTING_CONFIG_INFO[KEY]} Only models with audio input are listed.
+
+
+
+ A transcription model other than the chat model is loaded on the server when you dictate,
+ alongside the chat model. A server that keeps one model at a time (--models-max 1) unloads the chat model to do it.
+
+ {:else}
+
+ {#if singleModel}
+ This server runs one model, {singleModel}.
+ {singleModelHasAudio
+ ? 'It has audio input, so it transcribes your voice.'
+ : 'It has no audio input, so voice input is not available on this server.'}
+ {:else}
+ This server runs one model, which transcribes your voice when it has audio input.
+ {/if}
+
+ {/if}
+
+
+
+
diff --git a/ui/apps/chat/src/lib/components/app/settings/index.ts b/ui/apps/chat/src/lib/components/app/settings/index.ts
index 2d2aca5af17a..2d44c88921ba 100644
--- a/ui/apps/chat/src/lib/components/app/settings/index.ts
+++ b/ui/apps/chat/src/lib/components/app/settings/index.ts
@@ -52,6 +52,15 @@ export { default as SettingsResetButton } from './SettingsResetButton.svelte';
*/
export { default as SettingsChatImportExportTab } from './SettingsChat/SettingsChatImportExportTab.svelte';
+/**
+ * **SettingsChatVoiceTab** - Voice section of the settings
+ *
+ * The registry's voice fields, with the transcription model picker between them:
+ * in router mode it lists the server's models with audio input, on a single-model
+ * server it says whether that model can transcribe.
+ */
+export { default as SettingsChatVoiceTab } from './SettingsChat/SettingsChatVoiceTab.svelte';
+
/**
* Section wrapper for import/export sections. Displays a title, description,
* icon button, and optional summary of recent actions.
diff --git a/ui/apps/chat/src/lib/constants/index.ts b/ui/apps/chat/src/lib/constants/index.ts
index 46feb4a6dd4a..d7a8c982f403 100644
--- a/ui/apps/chat/src/lib/constants/index.ts
+++ b/ui/apps/chat/src/lib/constants/index.ts
@@ -74,3 +74,4 @@ export * from './read-media';
export * from './get-datetime';
export * from './browser-info';
export * from './search-chats';
+export * from './voice.constants';
diff --git a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts
index c94f97ff7d72..12c941e6faec 100644
--- a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts
+++ b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts
@@ -85,6 +85,11 @@ export const SETTINGS_KEYS = {
TYP_P: 'typ_p',
// General
USE_HUGGING_FACE_HUB: 'useHuggingFaceHub',
+ // Voice
+ VOICE_INPUT_MODE: 'voiceInputMode',
+ VOICE_TRANSCRIBE_MESSAGES: 'voiceTranscribeMessages',
+ VOICE_TRANSCRIPTION_LANGUAGE: 'voiceTranscriptionLanguage',
+ VOICE_TRANSCRIPTION_MODEL: 'voiceTranscriptionModel',
XTC_PROBABILITY: 'xtc_probability',
XTC_THRESHOLD: 'xtc_threshold'
} as const;
diff --git a/ui/apps/chat/src/lib/constants/settings.constants.ts b/ui/apps/chat/src/lib/constants/settings.constants.ts
index 669ccd6ecbb4..053311375b7c 100644
--- a/ui/apps/chat/src/lib/constants/settings.constants.ts
+++ b/ui/apps/chat/src/lib/constants/settings.constants.ts
@@ -6,19 +6,22 @@ import { SETTINGS_KEYS } from './settings-keys.constants';
import { TITLE_GENERATION } from './title-generation.constants';
import { FILE_GLOB_SEARCH_PICKERS } from './working-directory.constants';
import {
+ AudioLines,
Code,
Database,
File,
Funnel,
GlassWater,
ListRestart,
+ Mic,
Monitor,
Moon,
PencilRuler,
SlidersVertical,
- Sun
+ Sun,
+ TextCursorInput
} from '@lucide/svelte';
-import { SurfaceTheme, SyncableParameterType } from '#lib/enums/index.ts';
+import { SurfaceTheme, SyncableParameterType, VoiceInputMode } from '#lib/enums/index.ts';
import { SettingsFieldType } from '#lib/enums/settings.enums.ts';
import { ColorMode } from '#lib/enums/ui.enums.ts';
import type {
@@ -37,7 +40,8 @@ export const SETTINGS_SECTIONS = {
GENERAL: { slug: 'general', title: 'General' },
IMPORT_EXPORT: { slug: 'import-export', title: 'Import/Export' },
SAMPLING_PENALTIES: { slug: 'sampling-penalties', title: 'Sampling & Penalties' },
- TOOLS: { slug: 'tools', title: 'Tools' }
+ TOOLS: { slug: 'tools', title: 'Tools' },
+ VOICE: { slug: 'voice', title: 'Voice' }
} as const;
export const SETTINGS_SECTION_SLUGS = {
@@ -47,7 +51,8 @@ export const SETTINGS_SECTION_SLUGS = {
GENERAL: SETTINGS_SECTIONS.GENERAL.slug,
IMPORT_EXPORT: SETTINGS_SECTIONS.IMPORT_EXPORT.slug,
SAMPLING_PENALTIES: SETTINGS_SECTIONS.SAMPLING_PENALTIES.slug,
- TOOLS: SETTINGS_SECTIONS.TOOLS.slug
+ TOOLS: SETTINGS_SECTIONS.TOOLS.slug,
+ VOICE: SETTINGS_SECTIONS.VOICE.slug
} as const;
export const SETTINGS_SECTION_TITLES = {
@@ -57,7 +62,8 @@ export const SETTINGS_SECTION_TITLES = {
GENERAL: SETTINGS_SECTIONS.GENERAL.title,
IMPORT_EXPORT: SETTINGS_SECTIONS.IMPORT_EXPORT.title,
SAMPLING_PENALTIES: SETTINGS_SECTIONS.SAMPLING_PENALTIES.title,
- TOOLS: SETTINGS_SECTIONS.TOOLS.title
+ TOOLS: SETTINGS_SECTIONS.TOOLS.title,
+ VOICE: SETTINGS_SECTIONS.VOICE.title
} as const;
export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [
@@ -393,6 +399,48 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [
slug: SETTINGS_SECTION_SLUGS.DISPLAY,
title: SETTINGS_SECTION_TITLES.DISPLAY
},
+ // Voice: the transcription model picker lists the server's models (SettingsChatVoiceTab)
+ {
+ icon: Mic,
+ settings: [
+ {
+ defaultValue: VoiceInputMode.AUTO,
+ help: 'Automatic sends your voice to a model that understands audio, as a voice message with its transcript, and turns it into text in the message input for any other model. Always dictate turns it into text for every model, so only the text reaches the chat.',
+ key: SETTINGS_KEYS.VOICE_INPUT_MODE,
+ label: 'Voice input',
+ options: [
+ { icon: AudioLines, label: 'Automatic', value: VoiceInputMode.AUTO },
+ { icon: TextCursorInput, label: 'Always dictate', value: VoiceInputMode.DICTATION }
+ ],
+ type: SettingsFieldType.SELECT
+ },
+ {
+ defaultValue: '',
+ help: 'The model that turns speech into text when the chat model has no audio input. Empty uses the chat model.',
+ key: SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL,
+ label: 'Transcription model',
+ standaloneField: false,
+ type: SettingsFieldType.SELECT
+ },
+ {
+ defaultValue: true,
+ help: 'Write down what was said in each voice message, to read under its waveform. The chat model writes it, and a model without audio input gets it in place of the audio.',
+ key: SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES,
+ label: 'Transcribe voice messages',
+ type: SettingsFieldType.CHECKBOX
+ },
+ {
+ defaultValue: '',
+ help: 'The language you speak, as a two-letter code such as en or de, passed to the transcription. Leave it empty to let the model tell.',
+ key: SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE,
+ label: 'Transcription language',
+ placeholder: 'Automatic',
+ type: SettingsFieldType.INPUT
+ }
+ ],
+ slug: SETTINGS_SECTION_SLUGS.VOICE,
+ title: SETTINGS_SECTION_TITLES.VOICE
+ },
// MCP Servers (non-UI config object)
{
icon: PencilRuler,
diff --git a/ui/apps/chat/src/lib/constants/voice.constants.ts b/ui/apps/chat/src/lib/constants/voice.constants.ts
new file mode 100644
index 000000000000..9f8d9d69eeab
--- /dev/null
+++ b/ui/apps/chat/src/lib/constants/voice.constants.ts
@@ -0,0 +1,17 @@
+import { STORAGE_APP_NAME } from './storage.constants';
+
+/** What the chat form's microphone says it will do, or why it can't, on its button. */
+export const VOICE_MODE_MESSAGE = {
+ dictate: 'Dictate',
+ separateModelHint:
+ 'It may be loaded on the server alongside the chat model, which a server that keeps one model at a time unloads.',
+ unavailable:
+ 'No model with audio input is available. In router mode, pick a transcription model in Settings > Voice.',
+ voiceMessage: 'Send a voice message'
+} as const;
+
+/** Where the chat form remembers that it showed the hint about a separate transcription model. */
+export const VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.voiceSeparateModelHintSeen`;
+
+/** Name prefix of a voice message's WAV file; the rest makes it unique per recording. */
+export const VOICE_CLIP_NAME_PREFIX = 'voice-';
diff --git a/ui/apps/chat/src/lib/enums/index.ts b/ui/apps/chat/src/lib/enums/index.ts
index 29fd94373ee0..a4eed39d8190 100644
--- a/ui/apps/chat/src/lib/enums/index.ts
+++ b/ui/apps/chat/src/lib/enums/index.ts
@@ -121,3 +121,5 @@ export {
} from './tools.enums';
export { SplashOrientation } from './splash.enums';
+
+export { VoiceInputMode, VoiceMode } from './voice.enums';
diff --git a/ui/apps/chat/src/lib/enums/voice.enums.ts b/ui/apps/chat/src/lib/enums/voice.enums.ts
new file mode 100644
index 000000000000..4fb2c92bd7b0
--- /dev/null
+++ b/ui/apps/chat/src/lib/enums/voice.enums.ts
@@ -0,0 +1,17 @@
+/** The Voice input setting: what the microphone does with a model that understands audio. */
+export enum VoiceInputMode {
+ /** Send the voice to a model with audio input, dictate otherwise. */
+ AUTO = 'auto',
+ /** Always turn speech into text; only the text reaches the chat. */
+ DICTATION = 'dictation'
+}
+
+/** What a recording becomes for the conversation's model, as the chat form resolves it. */
+export enum VoiceMode {
+ /** Speech becomes text in the input; no audio is attached. */
+ DICTATION = 'dictation',
+ /** No model can take audio, so the microphone has nothing to do. */
+ UNAVAILABLE = 'unavailable',
+ /** The clip is attached and sent as audio, with a transcript to read. */
+ VOICE_MESSAGE = 'voice-message'
+}
diff --git a/ui/apps/chat/src/lib/utils/index.ts b/ui/apps/chat/src/lib/utils/index.ts
index a0544551393e..0737b08f58d4 100644
--- a/ui/apps/chat/src/lib/utils/index.ts
+++ b/ui/apps/chat/src/lib/utils/index.ts
@@ -384,3 +384,11 @@ export { groupModelFamilies, modelFamilyKey, type ModelFamilyGroup } from './mod
// Svelte actions
export { nearViewport } from './near-viewport';
+
+// Voice: what a recording becomes for the conversation's model
+export {
+ resolveVoiceMode,
+ voiceModeLabel,
+ type VoiceModeInput,
+ type VoiceModeResolution
+} from './voice-mode';
diff --git a/ui/apps/chat/src/lib/utils/voice-mode.ts b/ui/apps/chat/src/lib/utils/voice-mode.ts
new file mode 100644
index 000000000000..953615a48442
--- /dev/null
+++ b/ui/apps/chat/src/lib/utils/voice-mode.ts
@@ -0,0 +1,71 @@
+import { VOICE_MODE_MESSAGE } from '#lib/constants/voice.constants.ts';
+import { VoiceInputMode, VoiceMode } from '#lib/enums/index.ts';
+
+export interface VoiceModeInput {
+ /** The Voice input setting. */
+ setting: VoiceInputMode | string;
+ /** The conversation's effective model, null when none is picked. */
+ chatModel: string | null;
+ chatModelHasAudio: boolean;
+ isRouter: boolean;
+ /** The Transcription model setting; empty means the chat model. */
+ transcriptionModel: string;
+ transcriptionModelHasAudio: boolean;
+}
+
+export type VoiceModeResolution =
+ | {
+ mode: VoiceMode.VOICE_MESSAGE;
+ /** The chat model: it writes the transcript too, so a router swaps no model. */
+ transcriptionModel: string | null;
+ separateModel: false;
+ }
+ | {
+ mode: VoiceMode.DICTATION;
+ transcriptionModel: string | null;
+ /** A router model other than the chat model, which transcribing may load. */
+ separateModel: boolean;
+ }
+ | { mode: VoiceMode.UNAVAILABLE; reason: string };
+
+/**
+ * What a recording becomes for the conversation's model. Automatic sends the voice to a
+ * model with audio input and dictates otherwise; dictation transcribes with the chat model
+ * when it can hear, else (router mode) with the chosen transcription model.
+ */
+export function resolveVoiceMode(input: VoiceModeInput): VoiceModeResolution {
+ const { chatModel, chatModelHasAudio, isRouter, setting } = input;
+
+ if (setting !== VoiceInputMode.DICTATION && chatModelHasAudio) {
+ return { mode: VoiceMode.VOICE_MESSAGE, separateModel: false, transcriptionModel: chatModel };
+ }
+
+ if (chatModelHasAudio) {
+ return { mode: VoiceMode.DICTATION, separateModel: false, transcriptionModel: chatModel };
+ }
+
+ // a single-model server has no other model to transcribe with
+ const chosen = isRouter ? input.transcriptionModel.trim() : '';
+
+ if (chosen && input.transcriptionModelHasAudio) {
+ return {
+ mode: VoiceMode.DICTATION,
+ separateModel: chosen !== chatModel,
+ transcriptionModel: chosen
+ };
+ }
+
+ return { mode: VoiceMode.UNAVAILABLE, reason: VOICE_MODE_MESSAGE.unavailable };
+}
+
+/** What the microphone button says it will do, or why it can't. */
+export function voiceModeLabel(resolution: VoiceModeResolution): string {
+ switch (resolution.mode) {
+ case VoiceMode.VOICE_MESSAGE:
+ return VOICE_MODE_MESSAGE.voiceMessage;
+ case VoiceMode.DICTATION:
+ return VOICE_MODE_MESSAGE.dictate;
+ default:
+ return resolution.reason;
+ }
+}
diff --git a/ui/apps/chat/tests/stories/settings/SettingsSections.stories.svelte b/ui/apps/chat/tests/stories/settings/SettingsSections.stories.svelte
index 24b2c5c13b9b..de2d6b8f9063 100644
--- a/ui/apps/chat/tests/stories/settings/SettingsSections.stories.svelte
+++ b/ui/apps/chat/tests/stories/settings/SettingsSections.stories.svelte
@@ -17,6 +17,8 @@
+
+
diff --git a/ui/apps/chat/tests/unit/voice-mode.test.ts b/ui/apps/chat/tests/unit/voice-mode.test.ts
new file mode 100644
index 000000000000..316b7179b4e5
--- /dev/null
+++ b/ui/apps/chat/tests/unit/voice-mode.test.ts
@@ -0,0 +1,124 @@
+// Guards what the microphone does for a conversation's model: a voice message for a model
+// that hears, dictation otherwise, and why it can't when no model can transcribe.
+
+import {
+ SETTING_CONFIG_DEFAULT,
+ SETTINGS_CHAT_SECTIONS,
+ SETTINGS_KEYS,
+ SETTINGS_SECTION_SLUGS,
+ VOICE_MODE_MESSAGE
+} from '#lib/constants/index.ts';
+import { SettingsFieldType, VoiceInputMode, VoiceMode } from '#lib/enums/index.ts';
+import { resolveVoiceMode, type VoiceModeInput, voiceModeLabel } from '#lib/utils/voice-mode.ts';
+import { describe, expect, it } from 'vitest';
+
+const base: VoiceModeInput = {
+ chatModel: 'gemma-3n',
+ chatModelHasAudio: true,
+ isRouter: true,
+ setting: VoiceInputMode.AUTO,
+ transcriptionModel: '',
+ transcriptionModelHasAudio: false
+};
+
+describe('resolveVoiceMode', () => {
+ it('sends a voice message to a chat model with audio, transcribed by that model', () => {
+ expect(resolveVoiceMode(base)).toEqual({
+ mode: VoiceMode.VOICE_MESSAGE,
+ separateModel: false,
+ transcriptionModel: 'gemma-3n'
+ });
+ });
+
+ it('dictates with the chat model when the user always dictates', () => {
+ expect(resolveVoiceMode({ ...base, setting: VoiceInputMode.DICTATION })).toEqual({
+ mode: VoiceMode.DICTATION,
+ separateModel: false,
+ transcriptionModel: 'gemma-3n'
+ });
+ });
+
+ it('prefers the chat model over a chosen transcription model when it can hear', () => {
+ const result = resolveVoiceMode({
+ ...base,
+ setting: VoiceInputMode.DICTATION,
+ transcriptionModel: 'voxtral',
+ transcriptionModelHasAudio: true
+ });
+
+ expect(result).toMatchObject({ separateModel: false, transcriptionModel: 'gemma-3n' });
+ });
+
+ it('dictates with the chosen model for a chat model without audio, in router mode', () => {
+ expect(
+ resolveVoiceMode({
+ ...base,
+ chatModel: 'qwen3',
+ chatModelHasAudio: false,
+ transcriptionModel: 'voxtral',
+ transcriptionModelHasAudio: true
+ })
+ ).toEqual({ mode: VoiceMode.DICTATION, separateModel: true, transcriptionModel: 'voxtral' });
+ });
+
+ it('is unavailable when no model can transcribe', () => {
+ const noAudio = { ...base, chatModel: 'qwen3', chatModelHasAudio: false };
+
+ expect(resolveVoiceMode(noAudio)).toEqual({
+ mode: VoiceMode.UNAVAILABLE,
+ reason: VOICE_MODE_MESSAGE.unavailable
+ });
+ // a chosen model without audio input does not count
+ expect(
+ resolveVoiceMode({
+ ...noAudio,
+ transcriptionModel: 'llama',
+ transcriptionModelHasAudio: false
+ }).mode
+ ).toBe(VoiceMode.UNAVAILABLE);
+ });
+
+ it('ignores the transcription model on a single-model server', () => {
+ expect(
+ resolveVoiceMode({
+ ...base,
+ chatModelHasAudio: false,
+ isRouter: false,
+ transcriptionModel: 'voxtral',
+ transcriptionModelHasAudio: true
+ }).mode
+ ).toBe(VoiceMode.UNAVAILABLE);
+ });
+
+ it('labels the microphone with what it will do', () => {
+ expect(voiceModeLabel(resolveVoiceMode(base))).toBe('Send a voice message');
+ expect(voiceModeLabel(resolveVoiceMode({ ...base, setting: VoiceInputMode.DICTATION }))).toBe(
+ 'Dictate'
+ );
+ expect(voiceModeLabel(resolveVoiceMode({ ...base, chatModelHasAudio: false }))).toBe(
+ VOICE_MODE_MESSAGE.unavailable
+ );
+ });
+});
+
+describe('voice settings', () => {
+ const section = SETTINGS_CHAT_SECTIONS.find((s) => s.slug === SETTINGS_SECTION_SLUGS.VOICE);
+
+ it('default to automatic voice input, transcribed by the chat model', () => {
+ expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_INPUT_MODE]).toBe(VoiceInputMode.AUTO);
+ expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES]).toBe(true);
+ expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL]).toBe('');
+ expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE]).toBe('');
+ });
+
+ it('have a section of their own, the model picker drawn by it', () => {
+ const keys = section?.fields?.map((field) => field.key);
+
+ expect(keys).toEqual([
+ SETTINGS_KEYS.VOICE_INPUT_MODE,
+ SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES,
+ SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE
+ ]);
+ expect(section?.fields?.[0].type).toBe(SettingsFieldType.SELECT);
+ });
+});
From 8bb7b9dcfee687733ef2b951d5f3165c72d504ab Mon Sep 17 00:00:00 2001
From: Aleksander Grygier
Date: Sun, 15 Nov 2026 21:41:41 +0100
Subject: [PATCH 06/19] ui: record voice messages and dictation from the chat
form
The microphone now does what the conversation's model allows, without
the user choosing a mode: its label says "Send a voice message" or
"Dictate", and when no model can take audio it is disabled and says why
(shown in router mode only when some model there has audio input, so the
reason can be acted on; a single-model server without audio keeps the
send button as before). The microphone errors keep their warning state
and toast, now also for the voice engine's own capture errors.
Recording goes through the voice engine's VoiceRecorder instead of the
MediaRecorder flow. While it records, the actions row becomes a
recording row: cancel, the live waveform, the elapsed time and stop.
Escape or cancel drops the recording. The form keeps its two rows while
recording, so the message stays visible on desktop and the row fits the
phone layouts and the expanded sheet.
- Dictation writes the live transcript into the message at the caret
as it streams. DictationSpan replaces only the dictated stretch, moves
it when the user types before it, keeps what is typed after it, and
hands text the user edits over to them. When nothing streams, the row
shows Transcribing... after stop and the text goes in once it is back.
Cancel takes the dictated text out again. No audio is attached.
- A voice message streams its live text, if any, under the waveform,
not into the message. On stop the clip is attached as a voice file
with its peaks and length, and its transcript is followed by the voice
transcripts store (unless Transcribe voice messages is off).
Realtime transcription is skipped for servers reached through a tunnel
or with an API key, which a browser websocket cannot carry. The first
dictation with a separate router model shows a one-time note that it
loads beside the chat model.
The form takes an optional voiceRecorder, so tests and stories can run
it without a microphone.
Assisted-by: Claude (Opus 5.5 Medium)
---
.../app/chat/ChatForm/ChatForm.svelte | 160 ++++---
.../ChatFormActionRecord.svelte | 32 +-
.../ChatFormActions/ChatFormActions.svelte | 17 +-
.../ChatFormActionsRecording.svelte | 92 +++++
.../chat/src/lib/components/app/chat/index.ts | 14 +-
.../src/lib/constants/settings.constants.ts | 2 +-
.../chat/src/lib/constants/voice.constants.ts | 5 +-
.../lib/hooks/use-chat-form-voice.svelte.ts | 391 ++++++++++++++++++
ui/apps/chat/src/lib/utils/dictation-span.ts | 90 ++++
.../ChatFormActionRecordWrapper.svelte | 12 +-
.../microphone-unavailable.svelte.test.ts | 12 +-
.../form/ChatFormActionRecord.stories.svelte | 50 ++-
.../tests/stories/voice/VoiceLiveDemo.svelte | 2 +-
.../chat/tests/unit/dictation-span.test.ts | 90 ++++
14 files changed, 860 insertions(+), 109 deletions(-)
create mode 100644 ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionsRecording.svelte
create mode 100644 ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts
create mode 100644 ui/apps/chat/src/lib/utils/dictation-span.ts
create mode 100644 ui/apps/chat/tests/unit/dictation-span.test.ts
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte
index 0c44d125fb9c..d37dfd002b36 100644
--- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte
+++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte
@@ -5,6 +5,7 @@
ActionIcon,
ChatAttachmentsList,
ChatFormActions,
+ ChatFormActionsRecording,
ChatFormCurrentWorkingDirectory,
ChatFormInput,
ChatFormInputFileInputInvisible,
@@ -30,6 +31,10 @@
} from '#lib/enums/index.ts';
import { useChatFormExpansion } from '#lib/hooks/use-chat-form-expansion.svelte.ts';
import { useChatFormPickers } from '#lib/hooks/use-chat-form-pickers.svelte.ts';
+ import {
+ useChatFormVoice,
+ type VoiceRecorderLike
+ } from '#lib/hooks/use-chat-form-voice.svelte.ts';
import {
chatStore,
conversationsStore,
@@ -39,7 +44,8 @@
serverStore,
settingsStore,
toolsStore,
- uiStore
+ uiStore,
+ voiceTranscriptsStore
} from '#lib/stores/index.ts';
import type {
FileMentionEntry,
@@ -48,14 +54,7 @@
MCPResourceInfo,
PromptMessage
} from '#lib/types/index.ts';
- import {
- AudioRecorder,
- convertToWav,
- createAudioFile,
- describeMicrophoneError,
- isAudioRecordingSupported,
- microphoneUnavailableReason
- } from '#lib/utils/browser-only.ts';
+ import { microphoneUnavailableReason } from '#lib/utils/browser-only.ts';
import {
buildMentionInsertion,
findCommandToken,
@@ -65,6 +64,7 @@
parseClipboardContent,
uuid
} from '#lib/utils/index.ts';
+ import { VoiceRecorder } from '#lib/voice/index.ts';
import { onMount, untrack } from 'svelte';
import type { ClassValue } from 'svelte/elements';
import { toast } from 'svelte-sonner';
@@ -96,6 +96,9 @@
onUploadedFileRemove?: (fileId: string) => void;
onUploadedFilesChange?: (files: ChatUploadedFile[]) => void;
onValueChange?: (value: string) => void;
+
+ /** Records voice input; tests and stories pass a fake, the app a VoiceRecorder. */
+ voiceRecorder?: VoiceRecorderLike;
}
let {
@@ -117,21 +120,20 @@
showModelSelector = true,
singleRow = false,
uploadedFiles = $bindable([]),
- value = $bindable('')
+ value = $bindable(''),
+ voiceRecorder
}: Props = $props();
// Component References
- let audioRecorder: AudioRecorder | undefined;
let chatFormActionsRef: ChatFormActions | undefined = $state(undefined);
let fileInputRef: ChatFormInputFileInputInvisible | undefined = $state(undefined);
let pickersRef: { handleKeydown: (event: KeyboardEvent) => boolean } | undefined =
$state(undefined);
let inputRef: ChatFormInput | undefined = $state(undefined);
- // Audio Recording State
- let isRecording = $state(false);
- let recordingSupported = $state(false);
+ // Voice input
+ const recorder: VoiceRecorderLike = untrack(() => voiceRecorder) ?? new VoiceRecorder();
/** Why this page can't record at all (no HTTPS, no recorder), which no tap fixes. */
let micUnavailableReason = $state(null);
/** Why the last start failed (denied, no microphone, busy); the next tap tries again. */
@@ -189,6 +191,25 @@
}
});
+ const voice = useChatFormVoice({
+ addVoiceFile: (file) => {
+ uploadedFiles = [...uploadedFiles, file];
+ onUploadedFilesChange?.(uploadedFiles);
+ },
+ getCaretOffset: () => inputRef?.getCaretOffset() ?? value.length,
+ getRecorder: () => recorder,
+ getValue: () => value,
+ onMicError: (message) => (micError = message),
+ // on a phone placing the caret would open the keyboard
+ setCaretOffset: (offset) => {
+ if (!deviceStore.isMobile) inputRef?.setCaretOffset(offset);
+ },
+ setValue: (next) => {
+ value = next;
+ onValueChange?.(next);
+ }
+ });
+
async function handleWorkingDirectoryChange(newDir: string | null) {
// Committing a directory consumes the `/cwd` token; the chip's
// clear-X path has no token to consume.
@@ -263,6 +284,7 @@
singleRow &&
!deviceStore.isMobile &&
!isExpanded &&
+ !voice.isActive &&
!hasAttachments &&
!mcpStore.resources.hasAttachments &&
!value.includes('\n')
@@ -327,9 +349,7 @@
);
onMount(() => {
- recordingSupported = isAudioRecordingSupported();
micUnavailableReason = microphoneUnavailableReason();
- audioRecorder = new AudioRecorder();
});
/** Expands for the message as it stands, e.g. one put back after a send failed. */
@@ -375,6 +395,10 @@
onAttachmentRemove?.(index);
}
} else {
+ const removed = uploadedFiles.find((file) => file.id === fileId);
+
+ if (removed?.voice) voiceTranscriptsStore.forget(removed.name);
+
onUploadedFileRemove?.(fileId);
}
}
@@ -586,7 +610,7 @@
inputRef?.replaceText(built.newValue, built.caretOffset);
}
- async function handleMicClick() {
+ function handleMicClick() {
// a phone shows no tooltip on a tap, so the reason comes as a toast
if (micUnavailableReason) {
toast.error(micUnavailableReason);
@@ -594,36 +618,20 @@
return;
}
- if (!audioRecorder || !recordingSupported) {
- console.warn('Audio recording not supported');
-
- return;
- }
+ void voice.start();
+ }
- if (isRecording) {
- isRecording = false;
- try {
- const audioBlob = await audioRecorder.stopRecording();
- const wavBlob = await convertToWav(audioBlob);
- const audioFile = createAudioFile(wavBlob);
+ // Escape drops a recording, wherever the focus is
+ function handleWindowKeydown(event: KeyboardEvent) {
+ if (event.key !== KeyboardKey.ESCAPE || !voice.isActive || event.defaultPrevented) return;
- onFilesAdd?.([audioFile]);
- } catch (error) {
- console.error('Failed to stop recording:', error);
- }
- } else {
- try {
- await audioRecorder.startRecording();
- isRecording = true;
- micError = null;
- } catch (error) {
- micError = describeMicrophoneError(error);
- toast.error(micError);
- }
- }
+ event.preventDefault();
+ voice.cancel();
}
+
+
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte
index 8664af6b2ea7..2abe0f5b56ec 100644
--- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte
+++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte
@@ -2,34 +2,44 @@
import { Button } from '@ggml-org/llama-ui/ui/button';
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
import { Mic, MicOff, Square } from '@lucide/svelte';
- import { ICON_CLASS_DEFAULT } from '#lib/constants/index.ts';
+ import { ICON_CLASS_DEFAULT, VOICE_MODE_MESSAGE } from '#lib/constants/index.ts';
interface Props {
+ /** What a press does: send a voice message or dictate. */
+ action?: string;
class?: string;
disabled?: boolean;
- hasAudioModality?: boolean;
+ /** Said after the action in the tooltip, e.g. that another model may be loaded. */
+ hint?: string | null;
isLoading?: boolean;
isRecording?: boolean;
/** Why the microphone can't record (no HTTPS, access denied...), shown on the button. */
micWarning?: string | null;
+ /** Why no model can take the recording; disables the button. */
+ unavailableReason?: string | null;
onMicClick?: () => void;
}
let {
+ action = VOICE_MODE_MESSAGE.voiceMessage,
class: className = '',
disabled = false,
- hasAudioModality = false,
+ hint = null,
isLoading = false,
isRecording = false,
micWarning = null,
- onMicClick
+ onMicClick,
+ unavailableReason = null
}: Props = $props();
- // a model without audio input disables the button; a microphone that can't record keeps it
+ // no model to take the recording disables the button; a microphone that can't record keeps it
// tappable, so a tap can say why
- const showWarning = $derived(hasAudioModality && !isRecording && !!micWarning);
+ const showWarning = $derived(!unavailableReason && !isRecording && !!micWarning);
const tooltip = $derived(
- !hasAudioModality ? 'Current model does not support audio' : showWarning ? micWarning : null
+ isRecording
+ ? null
+ : (unavailableReason ??
+ (showWarning ? micWarning : [action, hint].filter(Boolean).join('. ')))
);
@@ -38,11 +48,11 @@
{:else if shouldShowRecordButton}
{:else}
+
+
+
diff --git a/ui/apps/chat/src/lib/components/app/chat/index.ts b/ui/apps/chat/src/lib/components/app/chat/index.ts
index fce3962c4fe4..32d26aca5a60 100644
--- a/ui/apps/chat/src/lib/components/app/chat/index.ts
+++ b/ui/apps/chat/src/lib/components/app/chat/index.ts
@@ -170,12 +170,20 @@ export { default as ChatForm } from './ChatForm/ChatForm.svelte';
export { default as ChatFormActionsAdd } from './ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte';
/**
- * Audio recording button with real-time recording indicator. Records audio
- * and converts to WAV format for upload. Only visible when the active model
- * supports audio modality and setting for automatic audio input is enabled. Shows recording duration while active.
+ * Microphone button: starts a voice message or a dictation (its label says which),
+ * or, disabled, says why no model can take a recording; while recording it is the
+ * stop button. Shown on an empty message when voice input is available and the
+ * setting for the microphone on empty input is on.
*/
export { default as ChatFormActionRecord } from './ChatForm/ChatFormActions/ChatFormActionRecord.svelte';
+/**
+ * The actions row while the chat form records: cancel, the live waveform and the
+ * elapsed time, and stop, with a voice message's streaming text under the waveform
+ * and a waiting state while dictated text is transcribed after recording.
+ */
+export { default as ChatFormActionsRecording } from './ChatForm/ChatFormActions/ChatFormActionsRecording.svelte';
+
/**
* Container for chat form action buttons. Arranges file attachment, audio record,
* and submit/stop buttons in a horizontal layout. Handles conditional visibility
diff --git a/ui/apps/chat/src/lib/constants/settings.constants.ts b/ui/apps/chat/src/lib/constants/settings.constants.ts
index 053311375b7c..cdd1689e4520 100644
--- a/ui/apps/chat/src/lib/constants/settings.constants.ts
+++ b/ui/apps/chat/src/lib/constants/settings.constants.ts
@@ -140,7 +140,7 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [
},
{
defaultValue: true,
- help: 'Automatically show microphone button instead of send button when textarea is empty for models with audio modality support.',
+ help: 'Show the microphone in place of the send button while the message is empty, whenever voice input is available (see Voice).',
key: SETTINGS_KEYS.AUTO_MIC_ON_EMPTY,
label: 'Show microphone on empty input',
type: SettingsFieldType.CHECKBOX
diff --git a/ui/apps/chat/src/lib/constants/voice.constants.ts b/ui/apps/chat/src/lib/constants/voice.constants.ts
index 9f8d9d69eeab..a74d3e3d02d0 100644
--- a/ui/apps/chat/src/lib/constants/voice.constants.ts
+++ b/ui/apps/chat/src/lib/constants/voice.constants.ts
@@ -3,8 +3,11 @@ import { STORAGE_APP_NAME } from './storage.constants';
/** What the chat form's microphone says it will do, or why it can't, on its button. */
export const VOICE_MODE_MESSAGE = {
dictate: 'Dictate',
+ dictating: 'Dictating',
+ recording: 'Recording a voice message',
separateModelHint:
- 'It may be loaded on the server alongside the chat model, which a server that keeps one model at a time unloads.',
+ 'The transcription model is loaded on the server alongside the chat model, which a server that keeps one model at a time unloads.',
+ transcribing: 'Transcribing...',
unavailable:
'No model with audio input is available. In router mode, pick a transcription model in Settings > Voice.',
voiceMessage: 'Send a voice message'
diff --git a/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts b/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts
new file mode 100644
index 000000000000..7a4bbc0b2080
--- /dev/null
+++ b/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts
@@ -0,0 +1,391 @@
+/**
+ * Voice input for the chat form.
+ *
+ * Resolves what the microphone does for the conversation's model (a voice message,
+ * dictation, or nothing, see resolveVoiceMode) and runs a recording: live levels and
+ * elapsed time, dictated text written into the message at the caret as it streams, and
+ * on stop either a voice message attachment, whose transcript follows, or the text.
+ */
+
+import {
+ SETTINGS_KEYS,
+ VOICE_CLIP_NAME_PREFIX,
+ VOICE_MODE_MESSAGE,
+ VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY
+} from '#lib/constants/index.ts';
+import { MimeTypeAudio, VoiceInputMode, VoiceMode } from '#lib/enums/index.ts';
+import {
+ modelsStore,
+ serverStore,
+ settingsStore,
+ voiceTranscriptsStore
+} from '#lib/stores/index.ts';
+import type { ChatUploadedFile } from '#lib/types/index.ts';
+import { getAuthHeaders } from '#lib/utils/api-headers.ts';
+import { describeMicrophoneError } from '#lib/utils/audio-recording.ts';
+import { withBase } from '#lib/utils/base-path.ts';
+import { DictationSpan } from '#lib/utils/dictation-span.ts';
+import { uuid } from '#lib/utils/index.ts';
+import { resolveVoiceMode, voiceModeLabel } from '#lib/utils/voice-mode.ts';
+import {
+ VoiceCaptureError,
+ type VoiceRecorderStatus,
+ type VoiceServerContext,
+ type VoiceSessionOptions,
+ type VoiceSessionResult
+} from '#lib/voice/index.ts';
+import { onDestroy } from 'svelte';
+import { toast } from 'svelte-sonner';
+
+/** What the form needs of a VoiceRecorder; tests and stories pass a fake. */
+export interface VoiceRecorderLike {
+ readonly levels: number[];
+ readonly live: boolean;
+ readonly liveText: string;
+ readonly status: VoiceRecorderStatus;
+ start(options?: VoiceSessionOptions): Promise;
+ stop(): Promise;
+ cancel(): void;
+}
+
+/** Where a recording is: off, waiting for the microphone, recording, or waiting for dictated text. */
+export type ChatFormVoiceState = 'idle' | 'starting' | 'recording' | 'transcribing';
+
+interface UseChatFormVoiceOptions {
+ getRecorder: () => VoiceRecorderLike | undefined;
+ getValue: () => string;
+ /** Writes the message as dictation changes it; not a user edit. */
+ setValue: (value: string) => void;
+ getCaretOffset: () => number;
+ /** Puts the caret after the dictated text once it is final. */
+ setCaretOffset: (offset: number) => void;
+ addVoiceFile: (file: ChatUploadedFile) => void;
+ /** Why the last start failed, for the microphone button; null once one worked. */
+ onMicError: (message: string | null) => void;
+}
+
+const ELAPSED_TICK_MS = 200;
+
+function readHintSeen(): boolean {
+ try {
+ return localStorage.getItem(VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY) === 'true';
+ } catch {
+ return false;
+ }
+}
+
+function writeHintSeen(): void {
+ try {
+ localStorage.setItem(VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY, 'true');
+ } catch {
+ // private mode: the hint shows again next time
+ }
+}
+
+function readAsDataUrl(blob: Blob): Promise {
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader();
+
+ reader.onload = () => resolve(reader.result as string);
+ reader.onerror = () => reject(reader.error);
+ reader.readAsDataURL(blob);
+ });
+}
+
+/** The microphone's server: its routes, its auth, and whether a websocket reaches it. */
+function serverContext(): { context: VoiceServerContext; realtime: boolean } {
+ const headers = getAuthHeaders();
+ // a browser websocket can not send the API key
+ const hasAuth = Object.keys(headers).some((key) => key.toLowerCase() === 'authorization');
+
+ return {
+ context: {
+ getHeaders: () => getAuthHeaders(),
+ resolveUrl: (path) => withBase(path.replace(/^\./, ''))
+ },
+ realtime: !hasAuth
+ };
+}
+
+export function useChatFormVoice(options: UseChatFormVoiceOptions) {
+ const config = $derived(settingsStore.config);
+ const isRouter = $derived(serverStore.isRouterMode);
+ const chatModel = $derived(modelsStore.activeModelId);
+ const transcriptionSetting = $derived(
+ String(config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL] ?? '')
+ );
+ const hasAudio = (model: string | null) => {
+ void modelsStore.props.cacheVersion;
+
+ return model ? modelsStore.props.modelSupportsAudio(model) : false;
+ };
+ const resolution = $derived(
+ resolveVoiceMode({
+ chatModel,
+ chatModelHasAudio: hasAudio(chatModel),
+ isRouter,
+ setting: String(config[SETTINGS_KEYS.VOICE_INPUT_MODE] ?? VoiceInputMode.AUTO),
+ transcriptionModel: transcriptionSetting,
+ transcriptionModelHasAudio: hasAudio(transcriptionSetting || null)
+ })
+ );
+ // a router with a model that hears keeps the button, so it can say where to pick it
+ const anyAudioModel = $derived.by(() => {
+ void modelsStore.props.cacheVersion;
+
+ return isRouter && modelsStore.models.some((option) => hasAudio(option.model));
+ });
+
+ let hintSeen = $state(readHintSeen());
+ let state = $state('idle');
+ let elapsedMs = $state(0);
+ // the mode of the recording under way, fixed when it starts
+ let recordingMode = $state(VoiceMode.UNAVAILABLE);
+ let ticker: ReturnType | undefined;
+ let span: DictationSpan | null = null;
+ let lastValue = '';
+ // bumped by cancel, so a transcript still on its way is dropped
+ let generation = 0;
+
+ const recorder = $derived(options.getRecorder());
+ const unavailableReason = $derived(
+ resolution.mode === VoiceMode.UNAVAILABLE ? resolution.reason : null
+ );
+ const hint = $derived(
+ resolution.mode === VoiceMode.DICTATION && resolution.separateModel && !hintSeen
+ ? VOICE_MODE_MESSAGE.separateModelHint
+ : null
+ );
+
+ function startTicker() {
+ const startedAt = Date.now();
+
+ elapsedMs = 0;
+ ticker = setInterval(() => (elapsedMs = Date.now() - startedAt), ELAPSED_TICK_MS);
+ }
+
+ function stopTicker() {
+ clearInterval(ticker);
+ ticker = undefined;
+ }
+
+ function writeDictation(text: string) {
+ if (!span) return;
+
+ const next = span.apply(options.getValue(), text);
+
+ lastValue = next;
+ options.setValue(next);
+ }
+
+ async function start() {
+ if (!recorder || state !== 'idle' || resolution.mode === VoiceMode.UNAVAILABLE) return;
+
+ const mode = resolution.mode;
+ const model = resolution.transcriptionModel;
+ const { context, realtime } = serverContext();
+ const language = String(config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE] ?? '').trim();
+ const transcribe =
+ mode === VoiceMode.DICTATION || config[SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES] !== false;
+
+ if (resolution.mode === VoiceMode.DICTATION && resolution.separateModel && !hintSeen) {
+ hintSeen = true;
+ writeHintSeen();
+ }
+
+ recordingMode = mode;
+ state = 'starting';
+ span = null;
+
+ if (mode === VoiceMode.DICTATION) {
+ lastValue = options.getValue();
+ span = new DictationSpan(lastValue, options.getCaretOffset());
+ }
+
+ const run = ++generation;
+
+ try {
+ await recorder.start({
+ onText: mode === VoiceMode.DICTATION ? (text) => writeDictation(text) : undefined,
+ realtime,
+ transcribe: transcribe
+ ? {
+ context,
+ language: language || undefined,
+ model: isRouter && model ? model : undefined
+ }
+ : undefined
+ });
+ } catch (error) {
+ if (run !== generation) return;
+
+ state = 'idle';
+ span = null;
+
+ const message =
+ error instanceof VoiceCaptureError ? error.message : describeMicrophoneError(error);
+
+ options.onMicError(message);
+ toast.error(message);
+
+ return;
+ }
+
+ if (run !== generation) return;
+
+ options.onMicError(null);
+ state = 'recording';
+ startTicker();
+ }
+
+ async function stop() {
+ if (!recorder || state !== 'recording') return;
+
+ const run = generation;
+ const mode = recordingMode;
+
+ stopTicker();
+
+ let result: VoiceSessionResult;
+
+ try {
+ result = await recorder.stop();
+ } catch (error) {
+ console.error('Failed to stop the recording:', error);
+ state = 'idle';
+
+ return;
+ }
+
+ if (run !== generation) return;
+
+ if (mode === VoiceMode.VOICE_MESSAGE) {
+ state = 'idle';
+ await attachVoiceMessage(result);
+
+ return;
+ }
+
+ await finishDictation(result, run);
+ }
+
+ async function attachVoiceMessage({ clip, transcript }: VoiceSessionResult) {
+ const name = `${VOICE_CLIP_NAME_PREFIX}${Date.now()}-${uuid().slice(0, 8)}.wav`;
+ const file = new File([clip.blob], name, { type: MimeTypeAudio.WAV });
+
+ if (transcript) voiceTranscriptsStore.track(name, transcript);
+
+ options.addVoiceFile({
+ file,
+ id: uuid(),
+ name,
+ preview: await readAsDataUrl(clip.blob),
+ size: file.size,
+ type: MimeTypeAudio.WAV,
+ voice: { durationMs: clip.durationMs, peaks: clip.peaks }
+ });
+ }
+
+ async function finishDictation({ liveText, transcript }: VoiceSessionResult, run: number) {
+ const dictated = span;
+
+ if (!transcript || !dictated) {
+ state = 'idle';
+
+ return;
+ }
+
+ // with live text the message already shows it; otherwise wait for the text
+ if (!liveText) state = 'transcribing';
+ else state = 'idle';
+
+ try {
+ const text = await transcript;
+
+ if (run !== generation) return;
+
+ if (text.trim() || !liveText) writeDictation(text);
+ } catch (error) {
+ if (run !== generation) return;
+
+ toast.error(error instanceof Error ? error.message : 'Transcription failed');
+ }
+
+ state = 'idle';
+ span = null;
+
+ if (!dictated.isEmpty) options.setCaretOffset(dictated.end);
+ }
+
+ /** Drops the recording and any text it dictated. */
+ function cancel() {
+ generation++;
+ stopTicker();
+ recorder?.cancel();
+
+ if (span) {
+ const next = span.remove(options.getValue());
+
+ options.setValue(next);
+ span = null;
+ }
+
+ state = 'idle';
+ }
+
+ /** Follows an edit the user typed while dictation writes into the message. */
+ function handleUserInput() {
+ const next = options.getValue();
+
+ span?.edited(lastValue, next);
+ lastValue = next;
+ }
+
+ onDestroy(() => {
+ if (state !== 'idle') cancel();
+ });
+
+ return {
+ cancel,
+ get elapsedMs() {
+ return elapsedMs;
+ },
+ handleUserInput,
+ get hint() {
+ return hint;
+ },
+ /** A recording (or its dictated text) is under way: the form shows the recording bar. */
+ get isActive() {
+ return state !== 'idle';
+ },
+ /** What the microphone button says it will do. */
+ get label() {
+ return voiceModeLabel(resolution);
+ },
+ get levels() {
+ return recorder?.levels ?? [];
+ },
+ /** Text streaming while a voice message is recorded, shown under the waveform. */
+ get liveCaption() {
+ return recordingMode === VoiceMode.VOICE_MESSAGE ? (recorder?.liveText ?? '') : '';
+ },
+ get mode() {
+ return state === 'idle' ? resolution.mode : recordingMode;
+ },
+ get resolution() {
+ return resolution;
+ },
+ /** Whether the form shows a microphone button at all. */
+ get showButton() {
+ return resolution.mode !== VoiceMode.UNAVAILABLE || anyAudioModel;
+ },
+ start,
+ get state() {
+ return state;
+ },
+ stop,
+ get unavailableReason() {
+ return unavailableReason;
+ }
+ };
+}
diff --git a/ui/apps/chat/src/lib/utils/dictation-span.ts b/ui/apps/chat/src/lib/utils/dictation-span.ts
new file mode 100644
index 000000000000..b8cb088d7dee
--- /dev/null
+++ b/ui/apps/chat/src/lib/utils/dictation-span.ts
@@ -0,0 +1,90 @@
+/**
+ * The stretch of the message that dictation writes: it starts at the caret, and each
+ * new version of the dictated text replaces only that stretch, so what the user typed
+ * before and after it stays. An edit before the stretch moves it; an edit inside it
+ * hands that text over to the user, and dictation goes on after it.
+ */
+export class DictationSpan {
+ // characters of the dictated text already handed over to the user
+ private handedOver = 0;
+ private length = 0;
+ // characters of the dictated text the stretch shows
+ private shown = 0;
+ private start: number;
+
+ /** Where the dictated text ends, e.g. to put the caret after it. */
+ get end(): number {
+ return this.start + this.length;
+ }
+
+ /** Whether the stretch shows any text. */
+ get isEmpty(): boolean {
+ return this.length === 0;
+ }
+
+ /**
+ * Writes the dictated text so far into `value` in place of the stretch, with a space
+ * on either side where it would otherwise run into the neighbouring text.
+ */
+ apply(value: string, text: string): string {
+ const body = text.slice(this.handedOver).trim();
+ const before = value.slice(0, this.start);
+ const after = value.slice(this.start + this.length);
+ const lead = body && before && !/\s$/.test(before) ? ' ' : '';
+ const trail = body && after && !/^\s/.test(after) ? ' ' : '';
+ const inserted = lead + body + trail;
+
+ this.length = inserted.length;
+ this.shown = text.length;
+
+ return before + inserted + after;
+ }
+
+ constructor(value: string, caret: number) {
+ this.start = Math.max(0, Math.min(caret, value.length));
+ }
+
+ /** Follows an edit the user made, `previous` to `next`. */
+ edited(previous: string, next: string): void {
+ const { from, insert, to } = changeBetween(previous, next);
+ const end = this.end;
+ const delta = insert.length - (to - from);
+
+ if (to <= this.start) {
+ this.start += delta;
+ } else if (from >= end) {
+ return;
+ } else {
+ // typed into the dictated text: it is theirs now, what comes next goes after it
+ this.handedOver = this.shown;
+ this.start = to <= end ? end + delta : from + insert.length;
+ this.length = 0;
+ }
+ }
+
+ /** Takes the stretch out of `value`, as a cancelled dictation leaves the message. */
+ remove(value: string): string {
+ const next = value.slice(0, this.start) + value.slice(this.start + this.length);
+
+ this.length = 0;
+
+ return next;
+ }
+}
+
+/** The single change that turns `current` into `next`: their differing middle. */
+function changeBetween(current: string, next: string) {
+ let from = 0;
+
+ while (from < current.length && from < next.length && current[from] === next[from]) from++;
+
+ let currentTo = current.length;
+ let nextTo = next.length;
+
+ while (currentTo > from && nextTo > from && current[currentTo - 1] === next[nextTo - 1]) {
+ currentTo--;
+ nextTo--;
+ }
+
+ return { from, insert: next.slice(from, nextTo), to: currentTo };
+}
diff --git a/ui/apps/chat/tests/client/components/ChatFormActionRecordWrapper.svelte b/ui/apps/chat/tests/client/components/ChatFormActionRecordWrapper.svelte
index cdc4a46ed3f6..d836cd6dd057 100644
--- a/ui/apps/chat/tests/client/components/ChatFormActionRecordWrapper.svelte
+++ b/ui/apps/chat/tests/client/components/ChatFormActionRecordWrapper.svelte
@@ -3,12 +3,16 @@
import ChatFormActionRecord from '#lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte';
let {
- hasAudioModality = true,
micWarning = null,
- onMicClick
- }: { hasAudioModality?: boolean; micWarning?: string | null; onMicClick?: () => void } = $props();
+ onMicClick,
+ unavailableReason = null
+ }: {
+ micWarning?: string | null;
+ onMicClick?: () => void;
+ unavailableReason?: string | null;
+ } = $props();
-
+
diff --git a/ui/apps/chat/tests/client/microphone-unavailable.svelte.test.ts b/ui/apps/chat/tests/client/microphone-unavailable.svelte.test.ts
index 361506341c61..47ea061c4279 100644
--- a/ui/apps/chat/tests/client/microphone-unavailable.svelte.test.ts
+++ b/ui/apps/chat/tests/client/microphone-unavailable.svelte.test.ts
@@ -4,7 +4,7 @@
// while staying tappable, so a tap on a phone, which shows no tooltip, can explain it.
import ChatFormActionRecordWrapper from './components/ChatFormActionRecordWrapper.svelte';
-import { MICROPHONE_UNAVAILABLE_MESSAGE } from '#lib/constants/index.ts';
+import { MICROPHONE_UNAVAILABLE_MESSAGE, VOICE_MODE_MESSAGE } from '#lib/constants/index.ts';
import { describeMicrophoneError, microphoneUnavailableReason } from '#lib/utils/browser-only.ts';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { render } from 'vitest-browser-svelte';
@@ -71,18 +71,18 @@ describe('the record button', () => {
const screen = render(ChatFormActionRecordWrapper, {});
await expect
- .element(screen.getByRole('button', { name: 'Start recording' }).last())
+ .element(screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last())
.toBeEnabled();
});
- it('stays disabled for a model without audio, whatever the microphone', async () => {
+ it('stays disabled when no model can take audio, whatever the microphone', async () => {
const screen = render(ChatFormActionRecordWrapper, {
- hasAudioModality: false,
- micWarning: MICROPHONE_UNAVAILABLE_MESSAGE.insecure
+ micWarning: MICROPHONE_UNAVAILABLE_MESSAGE.insecure,
+ unavailableReason: VOICE_MODE_MESSAGE.unavailable
});
await expect
- .element(screen.getByRole('button', { name: 'Start recording' }).last())
+ .element(screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last())
.toBeDisabled();
});
});
diff --git a/ui/apps/chat/tests/stories/chat/form/ChatFormActionRecord.stories.svelte b/ui/apps/chat/tests/stories/chat/form/ChatFormActionRecord.stories.svelte
index 476e9397c469..66143acfa1f9 100644
--- a/ui/apps/chat/tests/stories/chat/form/ChatFormActionRecord.stories.svelte
+++ b/ui/apps/chat/tests/stories/chat/form/ChatFormActionRecord.stories.svelte
@@ -1,8 +1,10 @@
-
-
-
+
+
+
-
-
+
+
+
+
+
+
-
+
-
+
-
-
+
+ {
+ const button = canvas.getAllByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage });
+
+ await expect(button.at(-1)).toBeDisabled();
// the tooltip trigger wraps the disabled button, hover the trigger
- await userEvent.hover(canvas.getAllByRole('button', { name: 'Start recording' })[0]);
+ await userEvent.hover(button[0]);
+ await expect(
+ (await screen.findAllByText(VOICE_MODE_MESSAGE.unavailable)).length
+ ).toBeGreaterThan(0);
+ }}
+/>
+
+ {
+ await userEvent.hover(canvas.getAllByRole('button', { name: VOICE_MODE_MESSAGE.dictate })[0]);
+ await expect((await screen.findAllByText(/alongside the chat model/)).length).toBeGreaterThan(
+ 0
+ );
}}
/>
diff --git a/ui/apps/chat/tests/stories/voice/VoiceLiveDemo.svelte b/ui/apps/chat/tests/stories/voice/VoiceLiveDemo.svelte
index c370f377496a..d85786b6dc58 100644
--- a/ui/apps/chat/tests/stories/voice/VoiceLiveDemo.svelte
+++ b/ui/apps/chat/tests/stories/voice/VoiceLiveDemo.svelte
@@ -50,5 +50,5 @@
{formatClipTime(elapsedMs)}
-
+
diff --git a/ui/apps/chat/tests/unit/dictation-span.test.ts b/ui/apps/chat/tests/unit/dictation-span.test.ts
new file mode 100644
index 000000000000..13f081be479d
--- /dev/null
+++ b/ui/apps/chat/tests/unit/dictation-span.test.ts
@@ -0,0 +1,90 @@
+// Guards how dictation writes into the message: at the caret, replacing only its own text
+// as the transcript grows, and keeping what the user typed around it.
+
+import { DictationSpan } from '#lib/utils/dictation-span.ts';
+import { describe, expect, it } from 'vitest';
+
+describe('DictationSpan', () => {
+ it('writes into an empty message and replaces its own text as it grows', () => {
+ const span = new DictationSpan('', 0);
+
+ let value = span.apply('', 'Hello');
+
+ expect(value).toBe('Hello');
+
+ value = span.apply(value, 'Hello world');
+ expect(value).toBe('Hello world');
+ expect(span.end).toBe(value.length);
+ });
+
+ it('inserts at the caret with spaces, keeping the text before and after', () => {
+ const start = 'Translate into German';
+ const span = new DictationSpan(start, 'Translate'.length);
+
+ let value = span.apply(start, 'good');
+
+ expect(value).toBe('Translate good into German');
+
+ value = span.apply(value, 'good morning');
+ expect(value).toBe('Translate good morning into German');
+ });
+
+ it('adds a space where the dictated text would run into a word', () => {
+ const span = new DictationSpan('Say', 3);
+
+ expect(span.apply('Say', 'hi')).toBe('Say hi');
+
+ const middle = new DictationSpan('ab', 1);
+
+ expect(middle.apply('ab', 'x')).toBe('a x b');
+ });
+
+ it('follows text typed before it while it streams', () => {
+ const span = new DictationSpan('', 0);
+
+ let value = span.apply('', 'one');
+
+ const typed = `Note: ${value}`;
+
+ span.edited(value, typed);
+ value = span.apply(typed, 'one two');
+
+ expect(value).toBe('Note: one two');
+ });
+
+ it('keeps text typed after it', () => {
+ const span = new DictationSpan('', 0);
+
+ let value = span.apply('', 'one');
+
+ const typed = `${value}!`;
+
+ span.edited(value, typed);
+ value = span.apply(typed, 'one two');
+
+ expect(value).toBe('one two !');
+ });
+
+ it('hands over text the user edits and dictates on after it', () => {
+ const span = new DictationSpan('', 0);
+
+ let value = span.apply('', 'helo');
+
+ const fixed = 'hello';
+
+ span.edited(value, fixed);
+ value = span.apply(fixed, 'helo there');
+
+ expect(value).toBe('hello there');
+ });
+
+ it('takes its text back out when the dictation is cancelled', () => {
+ const start = 'Before after';
+ const span = new DictationSpan(start, 6);
+ const value = span.apply(start, 'middle');
+
+ expect(value).toBe('Before middle after');
+ expect(span.remove(value)).toBe(start);
+ expect(span.isEmpty).toBe(true);
+ });
+});
From e0412fe79155f29a6dc2f23f49926d7a8237fccd Mon Sep 17 00:00:00 2001
From: Aleksander Grygier
Date: Sun, 15 Nov 2026 21:41:53 +0100
Subject: [PATCH 07/19] ui: draw audio attachments as playable waveforms with
their transcript
A voice message should read as a clip, not as a file chip. Audio
attachments now render with Llama UI's VoiceWaveform inside
VoiceTranscriptReveal, through a direct branch in the core attachment
item (voice is core, so it does not go through the extension renderers):
the removable sm chip in the chat form, and the md clip in a sent
message, on the user bubble's surface.
- The audio plays from an object URL made from the stored base64 (or the
uploaded file's data URL), revoked when the clip leaves the screen.
- A clip stored without peaks, an older voice message or an uploaded
audio file, gets them computed from its audio once per session, in
memory only; uploaded files keep their name under the waveform.
- The transcript opens under the clip on hover, keyboard focus or a tap,
and shows Transcribing... while the voice transcripts store waits for
it, Transcription unavailable when it failed, and nothing when none
was asked for.
- The full-screen preview keeps its player and shows the transcript.
Tests run the chat form with a fake recorder: the microphone per mode,
the recording row, dictation around typed text, the Transcribing...
state, cancel, and a voice message chip whose transcript arrives later
and goes with the send. Stories show the form recording each kind, a
recorded chip, and sent voice messages with the transcript, pending,
and computed peaks.
Assisted-by: Claude (Opus 5.5 Medium)
---
.../ChatAttachmentsListItem.svelte | 14 +-
.../ChatAttachmentsListItemVoice.svelte | 116 +++++++++
...tAttachmentsPreviewCurrentItemAudio.svelte | 29 ++-
.../chat/src/lib/components/app/chat/index.ts | 8 +
.../chat/src/lib/utils/voice-attachment.ts | 108 +++++++++
.../client/chat-form-voice.svelte.test.ts | 225 ++++++++++++++++++
.../ChatFormVoiceTestWrapper.svelte | 17 ++
.../chat/form/ChatFormVoice.stories.svelte | 117 +++++++++
.../chat/form/ChatFormVoiceStoryHost.svelte | 33 +++
.../messages/ChatMessageVoice.stories.svelte | 131 ++++++++++
.../voice/fake-voice-recorder.svelte.ts | 150 ++++++++++++
ui/apps/chat/tests/stories/voice/fixtures.ts | 46 +++-
12 files changed, 981 insertions(+), 13 deletions(-)
create mode 100644 ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemVoice.svelte
create mode 100644 ui/apps/chat/src/lib/utils/voice-attachment.ts
create mode 100644 ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts
create mode 100644 ui/apps/chat/tests/client/components/ChatFormVoiceTestWrapper.svelte
create mode 100644 ui/apps/chat/tests/stories/chat/form/ChatFormVoice.stories.svelte
create mode 100644 ui/apps/chat/tests/stories/chat/form/ChatFormVoiceStoryHost.svelte
create mode 100644 ui/apps/chat/tests/stories/chat/messages/ChatMessageVoice.stories.svelte
create mode 100644 ui/apps/chat/tests/stories/voice/fake-voice-recorder.svelte.ts
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte
index 5bb7a47d71ea..eb1a8163c43a 100644
--- a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte
+++ b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte
@@ -3,7 +3,8 @@
ChatAttachmentsListItemMcpPrompt,
ChatAttachmentsListItemMcpResource,
ChatAttachmentsListItemThumbnailFile,
- ChatAttachmentsListItemThumbnailImage
+ ChatAttachmentsListItemThumbnailImage,
+ ChatAttachmentsListItemVoice
} from '#lib/components/app/index.ts';
import { AttachmentType } from '#lib/enums/index.ts';
import type {
@@ -12,7 +13,7 @@
DatabaseMessageExtraMcpResource,
MCPResourceAttachment
} from '#lib/types/index.ts';
- import { isMcpPrompt, isMcpResource, isPdfFile } from '#lib/utils/index.ts';
+ import { isAudioFile, isMcpPrompt, isMcpResource, isPdfFile } from '#lib/utils/index.ts';
interface Props {
class?: string;
@@ -93,6 +94,15 @@
class="flex-shrink-0 {className} {scrollClasses}"
onclick={() => onMcpResourcePreview?.(mcpResource)}
/>
+{:else if isAudioFile(item.attachment, item.uploadedFile)}
+
+
{:else if item.isImage && item.preview}
+
+
+
diff --git a/ui/apps/chat/src/lib/components/app/chat/index.ts b/ui/apps/chat/src/lib/components/app/chat/index.ts
index 32d26aca5a60..f7626c73a4f5 100644
--- a/ui/apps/chat/src/lib/components/app/chat/index.ts
+++ b/ui/apps/chat/src/lib/components/app/chat/index.ts
@@ -79,6 +79,14 @@ export { default as ChatAttachmentsListItemMcpResource } from './ChatAttachments
*/
export { default as ChatAttachmentsListItemThumbnailFile } from './ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemThumbnailFile.svelte';
+/**
+ * An audio attachment drawn as a clip: a playable waveform (peaks computed from
+ * the audio when it was stored without them) and, for a voice message, its
+ * transcript under it on hover, focus or a tap, with the transcribing and failed
+ * states. `sm` in the chat form, removable; `md` in a sent message.
+ */
+export { default as ChatAttachmentsListItemVoice } from './ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemVoice.svelte';
+
/**
* Thumbnail for image attachments with lazy loading and error fallback.
* Displays image preview with configurable dimensions. Falls back to placeholder
diff --git a/ui/apps/chat/src/lib/utils/voice-attachment.ts b/ui/apps/chat/src/lib/utils/voice-attachment.ts
new file mode 100644
index 000000000000..79380ee7f7fb
--- /dev/null
+++ b/ui/apps/chat/src/lib/utils/voice-attachment.ts
@@ -0,0 +1,108 @@
+import { AttachmentType } from '#lib/enums/index.ts';
+import type { ChatAttachmentDisplayItem, DatabaseMessageExtraAudioFile } from '#lib/types/index.ts';
+import { computePeaksFromAudio } from '#lib/voice/index.ts';
+
+// peaks of clips stored without them, kept for the session only
+const peaksCache = new Map>();
+
+/** What a list item needs to draw an audio attachment as a clip. */
+export interface VoiceAttachmentSource {
+ /** Base64 or a URL of the audio, or the file itself. */
+ audio: string | Blob | null;
+ durationMs?: number;
+ mimeType: string;
+ name: string;
+ peaks?: number[];
+ /** Recorded in the app, as opposed to an uploaded file. */
+ voice: boolean;
+ transcript?: string;
+}
+
+/** The clip behind an audio item of the form or of a message. */
+export function voiceAttachmentSource(item: ChatAttachmentDisplayItem): VoiceAttachmentSource {
+ const uploaded = item.uploadedFile;
+
+ if (uploaded) {
+ return {
+ audio: uploaded.preview ?? (uploaded.file.size > 0 ? uploaded.file : null),
+ durationMs: uploaded.voice?.durationMs,
+ mimeType: uploaded.type,
+ name: uploaded.name,
+ peaks: uploaded.voice?.peaks,
+ voice: !!uploaded.voice
+ };
+ }
+
+ const attachment =
+ item.attachment?.type === AttachmentType.AUDIO
+ ? (item.attachment as DatabaseMessageExtraAudioFile)
+ : null;
+
+ return {
+ audio: attachment?.base64Data || null,
+ durationMs: attachment?.durationMs,
+ mimeType: attachment?.mimeType ?? '',
+ name: item.name,
+ peaks: attachment?.peaks,
+ transcript: attachment?.transcript,
+ voice: attachment?.voice === true
+ };
+}
+
+/** A URL an
- A transcription model other than the chat model is loaded on the server when you dictate,
- alongside the chat model. A server that keeps one model at a time (--models-max 1--models-max 1) unloads the chat model to do it.
{:else}
diff --git a/ui/apps/chat/src/lib/constants/settings.constants.ts b/ui/apps/chat/src/lib/constants/settings.constants.ts
index cdd1689e4520..9b5c29fde221 100644
--- a/ui/apps/chat/src/lib/constants/settings.constants.ts
+++ b/ui/apps/chat/src/lib/constants/settings.constants.ts
@@ -4,6 +4,7 @@ import { COMPACTION } from './compaction.constants';
import { DEFAULT_MCP_CONFIG } from './mcp.constants';
import { SETTINGS_KEYS } from './settings-keys.constants';
import { TITLE_GENERATION } from './title-generation.constants';
+import { VOICE_TRANSCRIPTION_MODEL_AUTO } from './voice.constants';
import { FILE_GLOB_SEARCH_PICKERS } from './working-directory.constants';
import {
AudioLines,
@@ -415,8 +416,8 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [
type: SettingsFieldType.SELECT
},
{
- defaultValue: '',
- help: 'The model that turns speech into text when the chat model has no audio input. Empty uses the chat model.',
+ defaultValue: VOICE_TRANSCRIPTION_MODEL_AUTO,
+ help: 'The model that turns speech into text when the chat model has no audio input. Automatic uses a model with audio input that is already loaded, so dictating loads nothing.',
key: SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL,
label: 'Transcription model',
standaloneField: false,
diff --git a/ui/apps/chat/src/lib/constants/voice.constants.ts b/ui/apps/chat/src/lib/constants/voice.constants.ts
index a74d3e3d02d0..3761d50f1cf3 100644
--- a/ui/apps/chat/src/lib/constants/voice.constants.ts
+++ b/ui/apps/chat/src/lib/constants/voice.constants.ts
@@ -9,7 +9,7 @@ export const VOICE_MODE_MESSAGE = {
'The transcription model is loaded on the server alongside the chat model, which a server that keeps one model at a time unloads.',
transcribing: 'Transcribing...',
unavailable:
- 'No model with audio input is available. In router mode, pick a transcription model in Settings > Voice.',
+ 'No model with audio input is available. In router mode, load one or pick a transcription model in Settings > Voice.',
voiceMessage: 'Send a voice message'
} as const;
@@ -18,3 +18,9 @@ export const VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.v
/** Name prefix of a voice message's WAV file; the rest makes it unique per recording. */
export const VOICE_CLIP_NAME_PREFIX = 'voice-';
+
+/**
+ * The Transcription model setting's Automatic choice: the chat model when it hears, else a
+ * model with audio input that is already loaded, so dictating loads nothing.
+ */
+export const VOICE_TRANSCRIPTION_MODEL_AUTO = '';
diff --git a/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts b/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts
index 7a4bbc0b2080..2c4bd45ae6ee 100644
--- a/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts
+++ b/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts
@@ -119,11 +119,24 @@ export function useChatFormVoice(options: UseChatFormVoiceOptions) {
return model ? modelsStore.props.modelSupportsAudio(model) : false;
};
+ // what Automatic transcribes with when the chat model can't hear: one already loaded
+ const loadedAudioModel = $derived.by(() => {
+ void modelsStore.props.cacheVersion;
+
+ if (!isRouter) return null;
+
+ return (
+ modelsStore.models.find(
+ (option) => hasAudio(option.model) && modelsStore.isModelLoaded(option.model)
+ )?.model ?? null
+ );
+ });
const resolution = $derived(
resolveVoiceMode({
chatModel,
chatModelHasAudio: hasAudio(chatModel),
isRouter,
+ loadedAudioModel: loadedAudioModel,
setting: String(config[SETTINGS_KEYS.VOICE_INPUT_MODE] ?? VoiceInputMode.AUTO),
transcriptionModel: transcriptionSetting,
transcriptionModelHasAudio: hasAudio(transcriptionSetting || null)
diff --git a/ui/apps/chat/src/lib/utils/voice-mode.ts b/ui/apps/chat/src/lib/utils/voice-mode.ts
index 953615a48442..77f831652315 100644
--- a/ui/apps/chat/src/lib/utils/voice-mode.ts
+++ b/ui/apps/chat/src/lib/utils/voice-mode.ts
@@ -1,4 +1,7 @@
-import { VOICE_MODE_MESSAGE } from '#lib/constants/voice.constants.ts';
+import {
+ VOICE_MODE_MESSAGE,
+ VOICE_TRANSCRIPTION_MODEL_AUTO
+} from '#lib/constants/voice.constants.ts';
import { VoiceInputMode, VoiceMode } from '#lib/enums/index.ts';
export interface VoiceModeInput {
@@ -8,9 +11,11 @@ export interface VoiceModeInput {
chatModel: string | null;
chatModelHasAudio: boolean;
isRouter: boolean;
- /** The Transcription model setting; empty means the chat model. */
+ /** The Transcription model setting; VOICE_TRANSCRIPTION_MODEL_AUTO picks one. */
transcriptionModel: string;
transcriptionModelHasAudio: boolean;
+ /** Router mode: a loaded model with audio input, which Automatic falls back to. */
+ loadedAudioModel?: string | null;
}
export type VoiceModeResolution =
@@ -31,7 +36,8 @@ export type VoiceModeResolution =
/**
* What a recording becomes for the conversation's model. Automatic sends the voice to a
* model with audio input and dictates otherwise; dictation transcribes with the chat model
- * when it can hear, else (router mode) with the chosen transcription model.
+ * when it can hear, else (router mode) with the chosen transcription model or, on
+ * Automatic, a model with audio input that is already loaded.
*/
export function resolveVoiceMode(input: VoiceModeInput): VoiceModeResolution {
const { chatModel, chatModelHasAudio, isRouter, setting } = input;
@@ -45,13 +51,24 @@ export function resolveVoiceMode(input: VoiceModeInput): VoiceModeResolution {
}
// a single-model server has no other model to transcribe with
- const chosen = isRouter ? input.transcriptionModel.trim() : '';
+ if (!isRouter) return { mode: VoiceMode.UNAVAILABLE, reason: VOICE_MODE_MESSAGE.unavailable };
- if (chosen && input.transcriptionModelHasAudio) {
+ const chosen = input.transcriptionModel.trim();
+
+ if (chosen !== VOICE_TRANSCRIPTION_MODEL_AUTO) {
+ if (input.transcriptionModelHasAudio) {
+ return {
+ mode: VoiceMode.DICTATION,
+ separateModel: chosen !== chatModel,
+ transcriptionModel: chosen
+ };
+ }
+ } else if (input.loadedAudioModel) {
+ // already loaded, so transcribing swaps no model in or out
return {
mode: VoiceMode.DICTATION,
- separateModel: chosen !== chatModel,
- transcriptionModel: chosen
+ separateModel: false,
+ transcriptionModel: input.loadedAudioModel
};
}
diff --git a/ui/apps/chat/tests/unit/voice-mode.test.ts b/ui/apps/chat/tests/unit/voice-mode.test.ts
index 316b7179b4e5..91935e0bf6cd 100644
--- a/ui/apps/chat/tests/unit/voice-mode.test.ts
+++ b/ui/apps/chat/tests/unit/voice-mode.test.ts
@@ -6,7 +6,8 @@ import {
SETTINGS_CHAT_SECTIONS,
SETTINGS_KEYS,
SETTINGS_SECTION_SLUGS,
- VOICE_MODE_MESSAGE
+ VOICE_MODE_MESSAGE,
+ VOICE_TRANSCRIPTION_MODEL_AUTO
} from '#lib/constants/index.ts';
import { SettingsFieldType, VoiceInputMode, VoiceMode } from '#lib/enums/index.ts';
import { resolveVoiceMode, type VoiceModeInput, voiceModeLabel } from '#lib/utils/voice-mode.ts';
@@ -78,6 +79,36 @@ describe('resolveVoiceMode', () => {
).toBe(VoiceMode.UNAVAILABLE);
});
+ it('falls back to a loaded model with audio input on Automatic, loading nothing', () => {
+ const noAudio = { ...base, chatModel: 'qwen3', chatModelHasAudio: false };
+
+ expect(resolveVoiceMode({ ...noAudio, loadedAudioModel: 'qwen3-asr' })).toEqual({
+ mode: VoiceMode.DICTATION,
+ separateModel: false,
+ transcriptionModel: 'qwen3-asr'
+ });
+ // a chosen model wins over the loaded one
+ expect(
+ resolveVoiceMode({
+ ...noAudio,
+ loadedAudioModel: 'qwen3-asr',
+ transcriptionModel: 'voxtral',
+ transcriptionModelHasAudio: true
+ })
+ ).toEqual({ mode: VoiceMode.DICTATION, separateModel: true, transcriptionModel: 'voxtral' });
+ // so does the chat model, when it hears
+ expect(resolveVoiceMode({ ...base, loadedAudioModel: 'qwen3-asr' }).mode).toBe(
+ VoiceMode.VOICE_MESSAGE
+ );
+ // and nothing is loaded behind the user's back
+ expect(resolveVoiceMode({ ...noAudio, loadedAudioModel: null }).mode).toBe(
+ VoiceMode.UNAVAILABLE
+ );
+ expect(
+ resolveVoiceMode({ ...noAudio, isRouter: false, loadedAudioModel: 'qwen3-asr' }).mode
+ ).toBe(VoiceMode.UNAVAILABLE);
+ });
+
it('ignores the transcription model on a single-model server', () => {
expect(
resolveVoiceMode({
@@ -104,10 +135,12 @@ describe('resolveVoiceMode', () => {
describe('voice settings', () => {
const section = SETTINGS_CHAT_SECTIONS.find((s) => s.slug === SETTINGS_SECTION_SLUGS.VOICE);
- it('default to automatic voice input, transcribed by the chat model', () => {
+ it('default to automatic voice input and transcription model', () => {
expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_INPUT_MODE]).toBe(VoiceInputMode.AUTO);
expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES]).toBe(true);
- expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL]).toBe('');
+ expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL]).toBe(
+ VOICE_TRANSCRIPTION_MODEL_AUTO
+ );
expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE]).toBe('');
});
From b9e11203410342ba23560958a2cd327da9b082f4 Mon Sep 17 00:00:00 2001
From: Aleksander Grygier
Date: Mon, 16 Nov 2026 08:58:51 +0100
Subject: [PATCH 10/19] ui: fill the recording waveform, and open voice message
transcripts
- The live waveform fills the row from the start: a recording shorter
than the row is stretched over all its bars, a longer one scrolls with
the newest level on the right. No dotted placeholder.
- A voice message's transcript is open under its clip, in the chat form
while it comes in and in the sent message, with the toggle beside it
to fold it.
- Settings > Voice > Collapse transcripts (off by default) keeps them
folded instead, opening on hover, keyboard focus or a tap.
Assisted-by: Claude (Opus 5.5 Medium)
(cherry picked from commit 2a93f2f948ed976e40f693ce5d3acf1168284b05)
---
.../ChatAttachmentsListItemVoice.svelte | 17 +++++----
.../lib/constants/settings-keys.constants.ts | 1 +
.../src/lib/constants/settings.constants.ts | 7 ++++
.../messages/ChatMessageVoice.stories.svelte | 36 ++++++++++++++++---
ui/apps/chat/tests/unit/voice-mode.test.ts | 2 ++
.../voice/VoiceTranscriptReveal.svelte | 20 ++++++++---
.../components/voice/VoiceWaveformLive.svelte | 5 +--
ui/lib/ui/src/lib/voice/waveform.spec.ts | 7 +++-
ui/lib/ui/src/lib/voice/waveform.ts | 11 +++---
9 files changed, 83 insertions(+), 23 deletions(-)
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemVoice.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemVoice.svelte
index ee50e1db4f66..07868fb66596 100644
--- a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemVoice.svelte
+++ b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemVoice.svelte
@@ -1,6 +1,7 @@
-
+
seedVoice({ transcript: TRANSCRIPT })}
name="VoiceMessage"
+ play={async ({ canvas, canvasElement }) => {
+ const reveal = await waitFor(() => {
+ const element = canvasElement.querySelector('[data-slot="voice-transcript-reveal"]');
+
+ if (!element) throw new Error('no voice message');
+
+ return element as HTMLElement;
+ });
+
+ await expect(reveal).toHaveAttribute('data-revealed');
+ await expect(canvas.getByText(TRANSCRIPT)).toBeInTheDocument();
+ await expect(canvas.getByRole('button', { name: 'Hide transcript' })).toBeInTheDocument();
+ }}
+>
+
+
+
+
+
+ seedVoice({ transcript: TRANSCRIPT }, '', { [SETTINGS_KEYS.VOICE_COLLAPSE_TRANSCRIPTS]: true })}
+ name="VoiceMessageCollapsed"
play={async ({ canvas, canvasElement, userEvent }) => {
const reveal = await waitFor(() => {
const element = canvasElement.querySelector('[data-slot="voice-transcript-reveal"]');
diff --git a/ui/apps/chat/tests/unit/voice-mode.test.ts b/ui/apps/chat/tests/unit/voice-mode.test.ts
index 91935e0bf6cd..9e0e68753e38 100644
--- a/ui/apps/chat/tests/unit/voice-mode.test.ts
+++ b/ui/apps/chat/tests/unit/voice-mode.test.ts
@@ -138,6 +138,7 @@ describe('voice settings', () => {
it('default to automatic voice input and transcription model', () => {
expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_INPUT_MODE]).toBe(VoiceInputMode.AUTO);
expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES]).toBe(true);
+ expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_COLLAPSE_TRANSCRIPTS]).toBe(false);
expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL]).toBe(
VOICE_TRANSCRIPTION_MODEL_AUTO
);
@@ -150,6 +151,7 @@ describe('voice settings', () => {
expect(keys).toEqual([
SETTINGS_KEYS.VOICE_INPUT_MODE,
SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES,
+ SETTINGS_KEYS.VOICE_COLLAPSE_TRANSCRIPTS,
SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE
]);
expect(section?.fields?.[0].type).toBe(SettingsFieldType.SELECT);
diff --git a/ui/lib/ui/src/lib/components/voice/VoiceTranscriptReveal.svelte b/ui/lib/ui/src/lib/components/voice/VoiceTranscriptReveal.svelte
index a5ed97addd42..a144fe3d70a8 100644
--- a/ui/lib/ui/src/lib/components/voice/VoiceTranscriptReveal.svelte
+++ b/ui/lib/ui/src/lib/components/voice/VoiceTranscriptReveal.svelte
@@ -1,7 +1,8 @@
-
+ (state.sent = (state.sent ?? 0) + 1)}
+ {voiceRecorder}
+ />
diff --git a/ui/apps/chat/tests/stories/chat/form/ChatFormVoice.stories.svelte b/ui/apps/chat/tests/stories/chat/form/ChatFormVoice.stories.svelte
index 189e7a9325bc..adf4ce014db1 100644
--- a/ui/apps/chat/tests/stories/chat/form/ChatFormVoice.stories.svelte
+++ b/ui/apps/chat/tests/stories/chat/form/ChatFormVoice.stories.svelte
@@ -56,18 +56,19 @@
args={{ class: FORM_CLASS, script: SCRIPT }}
beforeEach={() => audioServer(VoiceInputMode.AUTO)}
name="RecordingVoiceMessage"
- play={async ({ canvas, userEvent }) => {
+ play={async ({ canvas, canvasElement, userEvent }) => {
const mic = canvas.getAllByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage });
await userEvent.click(mic[mic.length - 1]);
await expect(await canvas.findByText(VOICE_MODE_MESSAGE.recording)).toBeInTheDocument();
await expect(canvas.getByRole('button', { name: 'Cancel recording' })).toBeInTheDocument();
- // the live text runs under the waveform, the message stays empty
- await waitFor(() => expect(canvas.getByText(/What is the/)).toBeInTheDocument(), {
- timeout: 3000
- });
- await expect(canvas.getByRole('textbox')).not.toHaveTextContent('What is the');
+ // the live text shows as the empty message box's placeholder, the message stays empty
+ await waitFor(
+ () => expect(canvasElement.querySelector('.cm-placeholder')).toHaveTextContent(/What is the/),
+ { timeout: 3000 }
+ );
+ // a placeholder shows only while the message itself is empty
}}
/>
diff --git a/ui/apps/chat/tests/stories/chat/messages/ChatMessageVoice.stories.svelte b/ui/apps/chat/tests/stories/chat/messages/ChatMessageVoice.stories.svelte
index 5caccdaf3338..bbb61b84c566 100644
--- a/ui/apps/chat/tests/stories/chat/messages/ChatMessageVoice.stories.svelte
+++ b/ui/apps/chat/tests/stories/chat/messages/ChatMessageVoice.stories.svelte
@@ -80,7 +80,8 @@
});
await expect(reveal).toHaveAttribute('data-revealed');
- await expect(canvas.getByText(TRANSCRIPT)).toBeInTheDocument();
+ // drawn word by word, see VoiceSpokenText
+ await expect(reveal).toHaveTextContent(TRANSCRIPT);
await expect(canvas.getByRole('button', { name: 'Hide transcript' })).toBeInTheDocument();
}}
>
@@ -93,7 +94,7 @@
beforeEach={() =>
seedVoice({ transcript: TRANSCRIPT }, '', { [SETTINGS_KEYS.VOICE_COLLAPSE_TRANSCRIPTS]: true })}
name="VoiceMessageCollapsed"
- play={async ({ canvas, canvasElement, userEvent }) => {
+ play={async ({ canvasElement, userEvent }) => {
const reveal = await waitFor(() => {
const element = canvasElement.querySelector('[data-slot="voice-transcript-reveal"]');
@@ -105,7 +106,8 @@
await expect(reveal).not.toHaveAttribute('data-revealed');
await userEvent.hover(reveal);
await waitFor(() => expect(reveal).toHaveAttribute('data-revealed'));
- await expect(canvas.getByText(TRANSCRIPT)).toBeInTheDocument();
+ // drawn word by word, see VoiceSpokenText
+ await expect(reveal).toHaveTextContent(TRANSCRIPT);
}}
>
diff --git a/ui/apps/chat/tests/stories/voice/VoiceTranscriptReveal.stories.svelte b/ui/apps/chat/tests/stories/voice/VoiceTranscriptReveal.stories.svelte
index 9ec5d412ced7..a57caaa69ac9 100644
--- a/ui/apps/chat/tests/stories/voice/VoiceTranscriptReveal.stories.svelte
+++ b/ui/apps/chat/tests/stories/voice/VoiceTranscriptReveal.stories.svelte
@@ -78,7 +78,8 @@
const root = reveal(canvasElement);
// closed, but in the DOM for screen readers
- await expect(canvas.getByText(TRANSCRIPT)).toBeInTheDocument();
+ // drawn word by word, see VoiceSpokenText
+ await expect(region(canvasElement)).toHaveTextContent(TRANSCRIPT);
await expect(root).not.toHaveAttribute('data-revealed');
await expect(getComputedStyle(region(canvasElement)).opacity).toBe('0');
diff --git a/ui/apps/chat/tests/unit/voice-pauses.test.ts b/ui/apps/chat/tests/unit/voice-pauses.test.ts
new file mode 100644
index 000000000000..69f075e25141
--- /dev/null
+++ b/ui/apps/chat/tests/unit/voice-pauses.test.ts
@@ -0,0 +1,52 @@
+// Guards where a recording is cut into pieces to transcribe while it goes on: at pauses
+// after speech, judged against the room's noise, and at the latest after a long stretch.
+
+import { PauseDetector } from '#lib/voice/pauses.ts';
+import { describe, expect, it } from 'vitest';
+
+const INTERVAL = 50;
+
+/** Pushes `ms` worth of one level; true if any of them cut. */
+function hold(detector: PauseDetector, level: number, ms: number): boolean {
+ let cut = false;
+
+ for (let t = 0; t < ms; t += INTERVAL) cut = detector.push(level) || cut;
+
+ return cut;
+}
+
+describe('PauseDetector', () => {
+ it('cuts at a pause after speech', () => {
+ const detector = new PauseDetector({ intervalMs: INTERVAL });
+
+ expect(hold(detector, 0.1, 300)).toBe(false);
+ expect(hold(detector, 0.6, 600)).toBe(false);
+ expect(hold(detector, 0.1, 300)).toBe(false);
+ expect(hold(detector, 0.1, 300)).toBe(true);
+ });
+
+ it('does not cut silence alone, nor a short sound', () => {
+ const detector = new PauseDetector({ intervalMs: INTERVAL });
+
+ expect(hold(detector, 0.1, 2000)).toBe(false);
+ expect(hold(detector, 0.6, 100)).toBe(false);
+ expect(hold(detector, 0.1, 1000)).toBe(false);
+ });
+
+ it('follows the noise floor of the room', () => {
+ const detector = new PauseDetector({ intervalMs: INTERVAL });
+
+ // a loud room: its noise is not speech, and speech above it still counts
+ expect(hold(detector, 0.45, 500)).toBe(false);
+ expect(hold(detector, 0.8, 600)).toBe(false);
+ expect(hold(detector, 0.45, 600)).toBe(true);
+ });
+
+ it('cuts a long stretch of speech at its next quiet moment', () => {
+ const detector = new PauseDetector({ intervalMs: INTERVAL, maxSegmentMs: 2000 });
+
+ expect(hold(detector, 0.1, 100)).toBe(false);
+ expect(hold(detector, 0.6, 2500)).toBe(false);
+ expect(detector.push(0.1)).toBe(true);
+ });
+});
diff --git a/ui/apps/chat/tests/unit/voice-realtime.test.ts b/ui/apps/chat/tests/unit/voice-realtime.test.ts
index be6bfc6623b1..176d4c2cca80 100644
--- a/ui/apps/chat/tests/unit/voice-realtime.test.ts
+++ b/ui/apps/chat/tests/unit/voice-realtime.test.ts
@@ -178,6 +178,65 @@ describe('RealtimeTranscriber', () => {
await expect(finished).resolves.toBe('First line.\nSecond line.');
});
+ it('transcribes a conversation model piece by piece, one at a time', async () => {
+ const transcriber = new RealtimeTranscriber(context, { sampleRate: 16_000 });
+ const connected = transcriber.connect();
+ const commits = () =>
+ socket().sent.filter((event) => event.type === 'input_audio_buffer.commit');
+ const completed = (transcript: string) =>
+ socket().emit({ transcript, type: 'conversation.item.input_audio_transcription.completed' });
+
+ socket().emit(sessionCreated('realtime'));
+ await connected;
+
+ // nothing to commit before any audio
+ expect(transcriber.canCommitSegment).toBe(false);
+ transcriber.append(Int16Array.from([1]));
+ transcriber.commitSegment();
+ expect(commits()).toEqual([{ type: 'input_audio_buffer.commit' }]);
+
+ // the next piece waits for the text of this one
+ transcriber.append(Int16Array.from([2]));
+ expect(transcriber.canCommitSegment).toBe(false);
+ completed('Hello there.');
+ expect(transcriber.text).toBe('Hello there.');
+ expect(transcriber.canCommitSegment).toBe(true);
+
+ transcriber.commitSegment();
+ transcriber.append(Int16Array.from([3]));
+
+ // the end waits for the piece in flight, then commits the rest
+ const finished = transcriber.finish();
+
+ await Promise.resolve();
+ expect(commits()).toHaveLength(2);
+ completed('How are you?');
+ await vi.waitFor(() => expect(commits()).toHaveLength(3));
+ expect(commits()[2]).toEqual({ final: true, type: 'input_audio_buffer.commit' });
+ completed('Fine.');
+
+ await expect(finished).resolves.toBe('Hello there. How are you? Fine.');
+ });
+
+ it('ends without a final commit when every piece was committed already', async () => {
+ const transcriber = new RealtimeTranscriber(context, { sampleRate: 16_000 });
+ const connected = transcriber.connect();
+
+ socket().emit(sessionCreated('realtime'));
+ await connected;
+ transcriber.append(Int16Array.from([1]));
+ transcriber.commitSegment();
+ socket().emit({
+ transcript: 'Only this.',
+ type: 'conversation.item.input_audio_transcription.completed'
+ });
+
+ await expect(transcriber.finish()).resolves.toBe('Only this.');
+ expect(
+ socket().sent.filter((event) => event.type === 'input_audio_buffer.commit')
+ ).toHaveLength(1);
+ });
+
it('marks a conversation session switched to transcription as not live', async () => {
const transcriber = new RealtimeTranscriber(context, { sampleRate: 16_000 });
const connected = transcriber.connect();
diff --git a/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte b/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte
new file mode 100644
index 000000000000..f9299cc1eac1
--- /dev/null
+++ b/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte
@@ -0,0 +1,53 @@
+
+
+
+{#each words as { start, word }, index (live ? index : start)}{word}{/each}
+
+
diff --git a/ui/lib/ui/src/lib/components/voice/VoiceTranscriptReveal.svelte b/ui/lib/ui/src/lib/components/voice/VoiceTranscriptReveal.svelte
index a144fe3d70a8..3c2d17aa98b5 100644
--- a/ui/lib/ui/src/lib/components/voice/VoiceTranscriptReveal.svelte
+++ b/ui/lib/ui/src/lib/components/voice/VoiceTranscriptReveal.svelte
@@ -10,6 +10,7 @@
import type { Snippet } from 'svelte';
import { Button } from '../ui/button';
import { Skeleton } from '../ui/skeleton';
+ import VoiceSpokenText from './VoiceSpokenText.svelte';
import { cn } from '../ui/utils';
import type { VoiceTranscriptStatus } from '../../voice/types';
@@ -21,6 +22,8 @@
status?: VoiceTranscriptStatus;
/** Start with the transcript open; read once, the toggle key changes it after that. */
defaultOpen?: boolean;
+ /** Characters of the transcript spoken so far, to follow playback; left out, all of it reads as spoken. */
+ spoken?: number | null;
class?: string;
}
@@ -28,6 +31,7 @@
children,
class: className,
defaultOpen = false,
+ spoken = null,
status,
transcript = null
}: Props = $props();
@@ -145,7 +149,7 @@
Transcription unavailable
{:else if transcript}
- Transcript: {transcript}
+ Transcript:
{:else}
No speech detected
diff --git a/ui/lib/ui/src/lib/components/voice/VoiceWaveform.svelte b/ui/lib/ui/src/lib/components/voice/VoiceWaveform.svelte
index 5cceabcb7099..a8efe8896d85 100644
--- a/ui/lib/ui/src/lib/components/voice/VoiceWaveform.svelte
+++ b/ui/lib/ui/src/lib/components/voice/VoiceWaveform.svelte
@@ -36,6 +36,11 @@
size?: VoiceWaveformSize;
/** What the clip is, for the play button and the slider. */
label?: string;
+ /**
+ * Bound out: where playback is, to follow a transcript with. Null until the clip first plays, the clip's end
+ * once it played through.
+ */
+ playbackMs?: number | null;
class?: string;
}
@@ -44,6 +49,7 @@
durationMs,
label = 'Voice message',
peaks,
+ playbackMs = $bindable(null),
size = 'md',
src
}: Props = $props();
@@ -71,6 +77,36 @@
// the length until playback starts, then where it is
let shownMs = $derived(!paused || elapsedMs > 0 ? elapsedMs : totalMs);
+ // playback as it goes, read every frame while playing: the media element's own updates come a few times a second
+ let started = $state(false);
+ let playedThrough = $state(false);
+
+ $effect(() => {
+ if (!started) {
+ playbackMs = null;
+
+ return;
+ }
+
+ if (playedThrough) {
+ playbackMs = totalMs;
+
+ return;
+ }
+
+ playbackMs = elapsedMs;
+
+ if (paused || !audio) return;
+
+ const media = audio;
+ let frame = requestAnimationFrame(function tick() {
+ playbackMs = Math.min(media.currentTime * 1000, totalMs || Infinity);
+ frame = requestAnimationFrame(tick);
+ });
+
+ return () => cancelAnimationFrame(frame);
+ });
+
$effect(() => {
if (!src) paused = true;
@@ -152,6 +188,9 @@
function handlePlay() {
if (playingAudio && playingAudio !== audio) playingAudio.pause();
+ started = true;
+ playedThrough = false;
+
playingAudio = audio ?? null;
}
@@ -160,6 +199,8 @@
}
function handleEnded() {
+ playedThrough = true;
+
if (audio) audio.currentTime = 0;
currentTime = 0;
diff --git a/ui/lib/ui/src/lib/components/voice/index.ts b/ui/lib/ui/src/lib/components/voice/index.ts
index a2ed351287a7..1274acd1605b 100644
--- a/ui/lib/ui/src/lib/components/voice/index.ts
+++ b/ui/lib/ui/src/lib/components/voice/index.ts
@@ -3,6 +3,7 @@
* component takes its data in (levels, peaks, an audio URL, a transcript); recording, analysing and transcribing stay
* in each app.
*/
+export { default as VoiceSpokenText } from "./VoiceSpokenText.svelte";
export { default as VoiceTranscriptReveal } from "./VoiceTranscriptReveal.svelte";
export { default as VoiceWaveform } from "./VoiceWaveform.svelte";
export { default as VoiceWaveformLive } from "./VoiceWaveformLive.svelte";
diff --git a/ui/lib/ui/src/lib/voice/index.ts b/ui/lib/ui/src/lib/voice/index.ts
index dd0b40018d30..5da0bb682e44 100644
--- a/ui/lib/ui/src/lib/voice/index.ts
+++ b/ui/lib/ui/src/lib/voice/index.ts
@@ -1,2 +1,3 @@
+export * from './spoken.js';
export * from './waveform.js';
export type * from './types.js';
diff --git a/ui/lib/ui/src/lib/voice/spoken.spec.ts b/ui/lib/ui/src/lib/voice/spoken.spec.ts
new file mode 100644
index 000000000000..22c33e0bf0aa
--- /dev/null
+++ b/ui/lib/ui/src/lib/voice/spoken.spec.ts
@@ -0,0 +1,37 @@
+import { speechFraction, spokenLength, spokenWords } from './spoken.js';
+import { describe, expect, it } from 'vite-plus/test';
+
+const TEXT = 'one two three four';
+
+describe('spoken text', () => {
+ it('lights nothing before playback and everything at its end', () => {
+ expect(spokenLength(TEXT, [1, 1, 1, 1], 4000, 0)).toBe(0);
+ expect(spokenLength(TEXT, [1, 1, 1, 1], 4000, 4000)).toBe(TEXT.length);
+ expect(spokenLength(TEXT, [1, 1, 1, 1], 0, 100)).toBe(TEXT.length);
+ });
+
+ it('lights a word from its start, as a whole', () => {
+ // a quarter of even speech reaches into "two"
+ const length = spokenLength(TEXT, [1, 1, 1, 1], 4000, 1000);
+
+ expect(TEXT.slice(0, length)).toBe('one two');
+ });
+
+ it('spreads the text over the speech, not over the pauses', () => {
+ // the first half is silence: nothing is spoken yet halfway through
+ expect(speechFraction([0, 0, 1, 1], 0.5)).toBe(0);
+ expect(speechFraction([0, 0, 1, 1], 0.75)).toBe(0.5);
+ expect(speechFraction([1, 0, 0, 1], 0.5)).toBe(0.5);
+ // no speech to go by: the clip's own progress
+ expect(speechFraction([], 0.3)).toBeCloseTo(0.3);
+ expect(speechFraction([0, 0], 0.3)).toBeCloseTo(0.3);
+ });
+
+ it('cuts a text into words with their spaces and starts', () => {
+ expect(spokenWords(' hi there ')).toEqual([
+ { start: 0, word: ' hi ' },
+ { start: 5, word: 'there ' }
+ ]);
+ expect(spokenWords('')).toEqual([]);
+ });
+});
diff --git a/ui/lib/ui/src/lib/voice/spoken.ts b/ui/lib/ui/src/lib/voice/spoken.ts
new file mode 100644
index 000000000000..2dd61949cd5c
--- /dev/null
+++ b/ui/lib/ui/src/lib/voice/spoken.ts
@@ -0,0 +1,70 @@
+/** A peak under this share of the clip's loudest one counts as a pause, not speech. */
+const SPEECH_FLOOR = 0.15;
+
+/**
+ * How many characters of `text` have been spoken `elapsedMs` into a clip, estimated
+ * from its waveform: the text is spread over the parts with speech, so pauses move
+ * nothing on, and a word counts as spoken from its start. A transcript without word
+ * timings can only be followed this way; it is close, not exact.
+ */
+export function spokenLength(
+ text: string,
+ peaks: readonly number[],
+ durationMs: number,
+ elapsedMs: number
+): number {
+ if (!(durationMs > 0) || elapsedMs >= durationMs) return text.length;
+ if (!(elapsedMs > 0)) return 0;
+
+ const fraction = speechFraction(peaks, elapsedMs / durationMs);
+
+ return wordEnd(text, Math.floor(fraction * text.length));
+}
+
+/** The share of the clip's speech that lies before `position` (0..1 of its length). */
+export function speechFraction(peaks: readonly number[], position: number): number {
+ const loudest = Math.max(0, ...peaks.filter(Number.isFinite));
+
+ if (peaks.length === 0 || loudest <= 0) return clamp(position);
+
+ const weights = peaks.map((peak) => (Number.isFinite(peak) && peak > loudest * SPEECH_FLOOR ? 1 : 0));
+ const total = weights.reduce((sum, weight) => sum + weight, 0);
+
+ if (total === 0) return clamp(position);
+
+ const at = clamp(position) * weights.length;
+ const whole = Math.floor(at);
+ let before = 0;
+
+ for (let i = 0; i < whole; i++) before += weights[i];
+
+ if (whole < weights.length) before += weights[whole] * (at - whole);
+
+ return before / total;
+}
+
+/** The end of the word `index` falls in, so a word is lit as a whole. */
+function wordEnd(text: string, index: number): number {
+ if (index <= 0) return 0;
+ if (index >= text.length) return text.length;
+
+ const next = text.slice(index).search(/\s/);
+
+ return next === -1 ? text.length : index + next;
+}
+
+function clamp(value: number): number {
+ return Math.min(Math.max(value, 0), 1);
+}
+
+/** A text cut into words, each with the whitespace after it and where it starts. */
+export function spokenWords(text: string): { start: number; word: string }[] {
+ const words: { start: number; word: string }[] = [];
+ const pattern = /\s*\S+\s*|\s+/g;
+
+ for (let match = pattern.exec(text); match; match = pattern.exec(text)) {
+ words.push({ start: match.index, word: match[0] });
+ }
+
+ return words;
+}
From bfc1722d42aca4d22edf3cc66ebaa72d66bc7c2a Mon Sep 17 00:00:00 2001
From: Aleksander Grygier
Date: Mon, 16 Nov 2026 10:36:41 +0100
Subject: [PATCH 13/19] ui: let the capture test start with silence
A capture starts with its first audio block, which can come before a
stream's first sound, so the steady tone is checked from where it begins.
Assisted-by: Claude (Opus 5.5 Medium)
(cherry picked from commit a41ee5337b83fef685a5cbaff176925f1647ce9d)
---
ui/apps/chat/tests/client/voice-capture.svelte.test.ts | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/ui/apps/chat/tests/client/voice-capture.svelte.test.ts b/ui/apps/chat/tests/client/voice-capture.svelte.test.ts
index 344ccd2778c3..00a99dba43fd 100644
--- a/ui/apps/chat/tests/client/voice-capture.svelte.test.ts
+++ b/ui/apps/chat/tests/client/voice-capture.svelte.test.ts
@@ -93,8 +93,13 @@ describe('VoiceCapture', () => {
expect(clip.durationMs).toBeLessThan(5000);
expect(clip.peaks).toHaveLength(64);
expect(Math.max(...clip.peaks)).toBe(1);
- // a steady tone: no span of the clip is silent
- expect(clip.peaks.filter((peak) => peak > 0.5).length).toBeGreaterThan(48);
+ // a steady tone: once it reaches the capture, no span of the clip is silent. The capture
+ // starts with its first audio block, which can come before the tone stream's first sound
+ const toneFrom = clip.peaks.findIndex((peak) => peak > 0.5);
+
+ expect(toneFrom).toBeGreaterThanOrEqual(0);
+ expect(clip.peaks.slice(toneFrom).every((peak) => peak > 0.5)).toBe(true);
+ expect(clip.peaks.length - toneFrom).toBeGreaterThan(16);
const header = new DataView(await clip.blob.slice(0, 44).arrayBuffer());
const pcmBytes = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
From a87c46671a298ac96bfecee62646f1732c41989e Mon Sep 17 00:00:00 2001
From: Aleksander Grygier
Date: Mon, 16 Nov 2026 11:10:06 +0100
Subject: [PATCH 14/19] ui: show live voice text sooner, and keep silence from
becoming "yeah"
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
- A model that transcribes per commit (LFM3-Omni) gets shorter pieces:
cut at a 0.35 s pause after speech, or at the next quiet moment once a
piece reaches 3 s, so the text follows the speaker within a few
seconds in dictation, the message box and voice mode.
- When the recording stops, the audio after the last piece is committed
only if speech was heard in it. LFM3-Omni transcribes a commit of pure
silence as "Yeah." (or "はい。"), which ended up at the end of the text.
A recording whose only piece is that rest still commits it.
Assisted-by: Claude (Opus 5.5 Medium)
(cherry picked from commit 43e52cbab0fba9d8e84b9491890a406e3c6c90f3)
---
ui/apps/chat/src/lib/voice/constants.ts | 11 ++++++
ui/apps/chat/src/lib/voice/pauses.ts | 5 +++
ui/apps/chat/src/lib/voice/realtime.ts | 12 ++++--
ui/apps/chat/src/lib/voice/session.ts | 24 +++++++++---
.../chat/tests/unit/voice-realtime.test.ts | 38 +++++++++++++++++++
5 files changed, 81 insertions(+), 9 deletions(-)
diff --git a/ui/apps/chat/src/lib/voice/constants.ts b/ui/apps/chat/src/lib/voice/constants.ts
index 499c39cad5f5..fd5e6031cdcf 100644
--- a/ui/apps/chat/src/lib/voice/constants.ts
+++ b/ui/apps/chat/src/lib/voice/constants.ts
@@ -37,6 +37,17 @@ export const VOICE_MICROPHONE_CONSTRAINTS: MediaTrackConstraints = {
noiseSuppression: false
};
+/**
+ * How live text is cut into pieces for a model that transcribes per commit: at a pause
+ * after speech, or at the next quiet moment once a piece is this long. Short pieces keep
+ * the text close behind the speaker; the cut always falls between words.
+ */
+export const VOICE_PIECE = {
+ MAX_MS: 3000,
+ MIN_SPEECH_MS: 250,
+ PAUSE_MS: 350
+} as const;
+
/** How long a started capture waits for its first audio before it counts as recording anyway. */
export const VOICE_FIRST_AUDIO_TIMEOUT_MS = 1500;
diff --git a/ui/apps/chat/src/lib/voice/pauses.ts b/ui/apps/chat/src/lib/voice/pauses.ts
index bf84ff769be9..d0aa9d693156 100644
--- a/ui/apps/chat/src/lib/voice/pauses.ts
+++ b/ui/apps/chat/src/lib/voice/pauses.ts
@@ -31,6 +31,11 @@ export class PauseDetector {
private segmentMs = 0;
private speechMs = 0;
+ /** Whether speech was heard since the last cut. */
+ get heardSpeech(): boolean {
+ return this.speechMs > 0;
+ }
+
constructor(options: PauseDetectorOptions) {
this.intervalMs = options.intervalMs;
this.pauseMs = options.pauseMs ?? 500;
diff --git a/ui/apps/chat/src/lib/voice/realtime.ts b/ui/apps/chat/src/lib/voice/realtime.ts
index 75bf62282e1e..521cc48344bb 100644
--- a/ui/apps/chat/src/lib/voice/realtime.ts
+++ b/ui/apps/chat/src/lib/voice/realtime.ts
@@ -187,10 +187,12 @@ export class RealtimeTranscriber {
/**
* Ends the input (`input_audio_buffer.commit` with `final: true`) and resolves with
- * the whole transcript once the last item completes.
+ * the whole transcript once the last item completes. With `commitRest: false`, a
+ * session that transcribes per commit leaves out the audio after the last piece,
+ * e.g. when it is silence, which such a model may hear a word in.
* @throws RealtimeError when the session fails, closes or times out first
*/
- async finish(): Promise {
+ async finish(options: { commitRest?: boolean } = {}): Promise {
if (this.state === 'connecting') await this.connecting;
if (this.state !== 'open') throw new RealtimeError('The realtime session is not open.');
@@ -219,13 +221,15 @@ export class RealtimeTranscriber {
if (this.state !== 'finishing') return done;
- if (this.audioSinceCommit) {
+ const commitRest = this.live || options.commitRest !== false || this.completedText === '';
+
+ if (this.audioSinceCommit && commitRest) {
this.finalSent = true;
this.inFlight++;
this.audioSinceCommit = false;
this.send({ final: true, type: VOICE_REALTIME_EVENTS.COMMIT });
} else {
- // everything was committed in pieces already
+ // everything was committed in pieces already, or what is left holds no speech
this.resolveFinish();
}
diff --git a/ui/apps/chat/src/lib/voice/session.ts b/ui/apps/chat/src/lib/voice/session.ts
index dbe0e5f8fbba..1a6a65a2f885 100644
--- a/ui/apps/chat/src/lib/voice/session.ts
+++ b/ui/apps/chat/src/lib/voice/session.ts
@@ -1,5 +1,5 @@
import { VoiceCapture } from './capture';
-import { VOICE_LEVELS_INTERVAL_MS } from './constants';
+import { VOICE_LEVELS_INTERVAL_MS, VOICE_PIECE } from './constants';
import { RealtimeUnsupportedError } from './errors';
import { PauseDetector } from './pauses';
import { RealtimeTranscriber } from './realtime';
@@ -31,6 +31,8 @@ export async function startVoiceSession(options: VoiceSessionOptions = {}): Prom
const transcribe = options.transcribe;
let transcriber: RealtimeTranscriber | null = null;
+ // speech heard since the last piece was committed; until pieces are cut, assume so
+ let speechSinceCommit = true;
let liveText: string | null = null;
let ended = false;
@@ -59,15 +61,26 @@ export async function startVoiceSession(options: VoiceSessionOptions = {}): Prom
options.onRealtime?.(realtime.live);
- // a model that transcribes per commit gets the speech piece by piece, at its pauses
+ // a model that transcribes per commit gets the speech piece by piece, at its pauses:
+ // short ones, so the text keeps up with the speaker
if (!realtime.live) {
const pauses = new PauseDetector({
- intervalMs: capture.levelsIntervalMs ?? VOICE_LEVELS_INTERVAL_MS
+ intervalMs: capture.levelsIntervalMs ?? VOICE_LEVELS_INTERVAL_MS,
+ maxSegmentMs: VOICE_PIECE.MAX_MS,
+ minSpeechMs: VOICE_PIECE.MIN_SPEECH_MS,
+ pauseMs: VOICE_PIECE.PAUSE_MS
});
unsubscribers.push(
capture.onLevels((levels) => {
- if (pauses.push(levels[levels.length - 1] ?? 0)) realtime.commitSegment();
+ const cut = pauses.push(levels[levels.length - 1] ?? 0);
+
+ if (pauses.heardSpeech) speechSinceCommit = true;
+
+ if (cut && realtime.canCommitSegment) {
+ realtime.commitSegment();
+ speechSinceCommit = false;
+ }
})
);
}
@@ -130,7 +143,8 @@ export async function startVoiceSession(options: VoiceSessionOptions = {}): Prom
let transcript: Promise;
if (realtime?.ready) {
- transcript = realtime.finish().catch((error) => {
+ // a silent rest is left out: a model may hear a word ("yeah") in silence
+ transcript = realtime.finish({ commitRest: speechSinceCommit }).catch((error) => {
console.warn('Realtime transcript failed, transcribing over HTTP:', error);
return overHttp();
diff --git a/ui/apps/chat/tests/unit/voice-realtime.test.ts b/ui/apps/chat/tests/unit/voice-realtime.test.ts
index 176d4c2cca80..86d495ffab59 100644
--- a/ui/apps/chat/tests/unit/voice-realtime.test.ts
+++ b/ui/apps/chat/tests/unit/voice-realtime.test.ts
@@ -237,6 +237,44 @@ describe('RealtimeTranscriber', () => {
).toHaveLength(1);
});
+ it('leaves out a silent rest after the last piece, but never the only one', async () => {
+ const commits = () =>
+ socket().sent.filter((event) => event.type === 'input_audio_buffer.commit');
+ const transcriber = new RealtimeTranscriber(context, { sampleRate: 16_000 });
+ const connected = transcriber.connect();
+
+ socket().emit(sessionCreated('realtime'));
+ await connected;
+ transcriber.append(Int16Array.from([1]));
+ transcriber.commitSegment();
+ socket().emit({
+ transcript: 'All said.',
+ type: 'conversation.item.input_audio_transcription.completed'
+ });
+ // silence after the last piece: a model may hear "yeah" in it
+ transcriber.append(Int16Array.from([0]));
+
+ await expect(transcriber.finish({ commitRest: false })).resolves.toBe('All said.');
+ expect(commits()).toHaveLength(1);
+
+ // with nothing transcribed yet, the rest is all there is: it is committed
+ const second = new RealtimeTranscriber(context, { sampleRate: 16_000 });
+ const secondConnected = second.connect();
+
+ socket().emit(sessionCreated('realtime'));
+ await secondConnected;
+ second.append(Int16Array.from([1]));
+
+ const finished = second.finish({ commitRest: false });
+
+ await vi.waitFor(() => expect(commits()).toHaveLength(1));
+ socket().emit({
+ transcript: 'Short.',
+ type: 'conversation.item.input_audio_transcription.completed'
+ });
+ await expect(finished).resolves.toBe('Short.');
+ });
+
it('marks a conversation session switched to transcription as not live', async () => {
const transcriber = new RealtimeTranscriber(context, { sampleRate: 16_000 });
const connected = transcriber.connect();
From ef4b89b34c544cb28aae9c337079159b3fe8b3e0 Mon Sep 17 00:00:00 2001
From: Aleksander Grygier
Date: Mon, 16 Nov 2026 12:21:34 +0100
Subject: [PATCH 15/19] ui: transcribe a recording cut at pauses once more as a
whole
Live text comes piece by piece, cut at the speaker's short pauses, and a
model reads each piece as a sentence of its own: "I went. To the store."
When the recording ends, one pass over the whole clip gives the final
text, punctuated by meaning, in dictation and in voice mode alike. The
pieces' text stays the live text until then, and is kept when that pass
fails or the recording is longer than two minutes.
Assisted-by: Claude (Opus 5.5 Medium)
(cherry picked from commit 359dd9632e6c88000a1d23c2575bd1e6fcdd9ec1)
---
ui/apps/chat/src/lib/voice/constants.ts | 7 ++
ui/apps/chat/src/lib/voice/realtime.ts | 7 ++
ui/apps/chat/src/lib/voice/session.ts | 22 +++++-
ui/apps/chat/src/lib/voice/types.ts | 5 +-
ui/apps/chat/tests/unit/voice-session.test.ts | 71 ++++++++++++++++++-
5 files changed, 108 insertions(+), 4 deletions(-)
diff --git a/ui/apps/chat/src/lib/voice/constants.ts b/ui/apps/chat/src/lib/voice/constants.ts
index fd5e6031cdcf..7004254429d8 100644
--- a/ui/apps/chat/src/lib/voice/constants.ts
+++ b/ui/apps/chat/src/lib/voice/constants.ts
@@ -48,6 +48,13 @@ export const VOICE_PIECE = {
PAUSE_MS: 350
} as const;
+/**
+ * Longest recording that is transcribed once more as a whole when it ends. Pieces cut at
+ * the speaker's pauses come back as sentences of their own ("I went. To the store."); one
+ * pass over the whole clip punctuates by meaning. Past this, the pieces' text is kept.
+ */
+export const VOICE_RETRANSCRIBE_MAX_MS = 120_000;
+
/** How long a started capture waits for its first audio before it counts as recording anyway. */
export const VOICE_FIRST_AUDIO_TIMEOUT_MS = 1500;
diff --git a/ui/apps/chat/src/lib/voice/realtime.ts b/ui/apps/chat/src/lib/voice/realtime.ts
index 521cc48344bb..50fa6c54cd97 100644
--- a/ui/apps/chat/src/lib/voice/realtime.ts
+++ b/ui/apps/chat/src/lib/voice/realtime.ts
@@ -83,6 +83,7 @@ export class RealtimeTranscriber {
private readonly options: RealtimeTranscriberOptions;
private pendingConnect: Pending | null = null;
private pendingFinish: Pending | null = null;
+ private pieces = 0;
private queue: string[] = [];
private sentAudio = false;
private socket: WebSocket | null = null;
@@ -97,6 +98,11 @@ export class RealtimeTranscriber {
return this.state === 'open' && !this.live && this.audioSinceCommit && this.inFlight === 0;
}
+ /** Pieces committed with commitSegment(), the final commit not counted. */
+ get piecesCommitted(): number {
+ return this.pieces;
+ }
+
/** True between session.created and finish() or close(). */
get ready(): boolean {
return this.state === 'open';
@@ -138,6 +144,7 @@ export class RealtimeTranscriber {
if (!this.canCommitSegment) return;
this.inFlight++;
+ this.pieces++;
this.audioSinceCommit = false;
this.send({ type: VOICE_REALTIME_EVENTS.COMMIT });
}
diff --git a/ui/apps/chat/src/lib/voice/session.ts b/ui/apps/chat/src/lib/voice/session.ts
index 1a6a65a2f885..c812504a10c7 100644
--- a/ui/apps/chat/src/lib/voice/session.ts
+++ b/ui/apps/chat/src/lib/voice/session.ts
@@ -1,5 +1,5 @@
import { VoiceCapture } from './capture';
-import { VOICE_LEVELS_INTERVAL_MS, VOICE_PIECE } from './constants';
+import { VOICE_LEVELS_INTERVAL_MS, VOICE_PIECE, VOICE_RETRANSCRIBE_MAX_MS } from './constants';
import { RealtimeUnsupportedError } from './errors';
import { PauseDetector } from './pauses';
import { RealtimeTranscriber } from './realtime';
@@ -143,12 +143,30 @@ export async function startVoiceSession(options: VoiceSessionOptions = {}): Prom
let transcript: Promise;
if (realtime?.ready) {
+ // more than one piece: the text was cut at pauses, maybe inside a sentence
+ const split = !realtime.live && realtime.piecesCommitted + (speechSinceCommit ? 1 : 0) > 1;
// a silent rest is left out: a model may hear a word ("yeah") in silence
- transcript = realtime.finish({ commitRest: speechSinceCommit }).catch((error) => {
+ const pieces = realtime.finish({ commitRest: speechSinceCommit }).catch((error) => {
console.warn('Realtime transcript failed, transcribing over HTTP:', error);
return overHttp();
});
+
+ transcript =
+ split && clip.durationMs <= VOICE_RETRANSCRIBE_MAX_MS
+ ? // the whole clip once more, punctuated by meaning, not by the pauses
+ overHttp().then(
+ (text) => text || pieces,
+ (error) => {
+ console.warn(
+ 'Transcribing the whole recording failed, keeping its pieces:',
+ error
+ );
+
+ return pieces;
+ }
+ )
+ : pieces;
} else {
// still connecting at the end: the HTTP request is the shorter wait
realtime?.close();
diff --git a/ui/apps/chat/src/lib/voice/types.ts b/ui/apps/chat/src/lib/voice/types.ts
index ec9bfdb6c6cd..a470adecaed9 100644
--- a/ui/apps/chat/src/lib/voice/types.ts
+++ b/ui/apps/chat/src/lib/voice/types.ts
@@ -111,7 +111,10 @@ export interface VoiceSessionResult {
clip: VoiceClip;
/** Text the realtime socket gave while speaking, null when it streamed none */
liveText: string | null;
- /** Resolves with the final transcript (realtime, else HTTP); null when not asked to transcribe */
+ /**
+ * Resolves with the final transcript (realtime, else HTTP; a recording transcribed in pieces
+ * once more as a whole); null when not asked to transcribe
+ */
transcript: Promise | null;
}
diff --git a/ui/apps/chat/tests/unit/voice-session.test.ts b/ui/apps/chat/tests/unit/voice-session.test.ts
index 68f564e9bd80..7053321fb210 100644
--- a/ui/apps/chat/tests/unit/voice-session.test.ts
+++ b/ui/apps/chat/tests/unit/voice-session.test.ts
@@ -45,10 +45,16 @@ const clip: VoiceClip = {
function fakeCapture() {
const pcm = new Set();
+ const levels = new Set<(levels: number[]) => void>();
const capture = {
cancel: vi.fn(),
levels: [0, 0],
- onLevels: vi.fn(() => () => {}),
+ levelsIntervalMs: 50,
+ onLevels: vi.fn((listener: (levels: number[]) => void) => {
+ levels.add(listener);
+
+ return () => levels.delete(listener);
+ }),
onPcm: (listener: VoicePcmListener) => {
pcm.add(listener);
@@ -67,6 +73,10 @@ function fakeCapture() {
return {
capture: capture as unknown as VoiceCapture,
emitPcm: (chunk: Int16Array) => pcm.forEach((listener) => listener(chunk)),
+ /** Holds one input level for `ms`, as the capture reports it every 50 ms. */
+ holdLevel: (level: number, ms: number) => {
+ for (let t = 0; t < ms; t += 50) levels.forEach((listener) => listener([level]));
+ },
raw: capture
};
}
@@ -141,6 +151,65 @@ describe('startVoiceSession', () => {
expect(fetchMock).not.toHaveBeenCalled();
});
+ describe('a recording transcribed in pieces', () => {
+ const COMPLETED = 'conversation.item.input_audio_transcription.completed';
+
+ /** Speaks two pieces with a pause between them and stops. */
+ async function speakTwoPieces(fetchMock: ReturnType) {
+ const onText = vi.fn();
+ const { capture, emitPcm, holdLevel } = fakeCapture();
+ const session = await startVoiceSession({
+ capture,
+ onText,
+ transcribe: { context: serverContext(fetchMock as unknown as typeof fetch) }
+ });
+ const ws = FakeWebSocket.instances[0];
+
+ // a model that holds conversations transcribes per commit
+ ws.emit({ session: { type: 'realtime' }, type: 'session.created' });
+ await flush();
+
+ emitPcm(Int16Array.from([1]));
+ holdLevel(0.05, 200);
+ holdLevel(0.6, 500);
+ holdLevel(0.05, 400);
+
+ expect(ws.sent.filter((event) => event.type === 'input_audio_buffer.commit')).toHaveLength(1);
+
+ ws.emit({ transcript: 'I went.', type: COMPLETED });
+ expect(onText).toHaveBeenLastCalledWith('I went.');
+
+ emitPcm(Int16Array.from([2]));
+ holdLevel(0.6, 500);
+
+ const result = await session.stop();
+
+ ws.emit({ transcript: 'To the store.', type: COMPLETED });
+
+ return result;
+ }
+
+ it('is transcribed once more as a whole, punctuated by meaning', async () => {
+ const fetchMock = vi.fn(
+ async () => new Response(JSON.stringify({ text: 'I went to the store.' }))
+ );
+ const result = await speakTwoPieces(fetchMock);
+
+ expect(result.liveText).toBe('I went.');
+ await expect(result.transcript).resolves.toBe('I went to the store.');
+ expect(fetchMock).toHaveBeenCalledOnce();
+ });
+
+ it('keeps its pieces when the whole pass fails', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ const fetchMock = vi.fn(async () => new Response('busy', { status: 503 }));
+ const result = await speakTwoPieces(fetchMock);
+
+ await expect(result.transcript).resolves.toBe('I went. To the store.');
+ warn.mockRestore();
+ });
+ });
+
it('falls back to HTTP when the model cannot stream', async () => {
const fetchMock = vi.fn(async () => new Response(JSON.stringify({ text: 'From HTTP.' })));
const onRealtime = vi.fn();
From f588748141df1d62147fcb95a114f051c3c7ecf7 Mon Sep 17 00:00:00 2001
From: Aleksander Grygier
Date: Tue, 17 Nov 2026 11:54:48 +0100
Subject: [PATCH 16/19] ui: give live text over HTTP for models without a
realtime session
A model that hears but cannot hold a realtime session, such as
Qwen3-ASR, was transcribed only after the recording, so dictation
showed no text while speaking. The recording is now cut at the
speaker's pauses as with a realtime session, and each piece sent to
/v1/audio/transcriptions on its own, one at a time, its text joined
live. The audio is kept from the start, so what is said while the
socket answers goes with the first piece. The whole clip is still
transcribed once more at the end, and the pieces' text is kept if
that fails. A server a websocket cannot reach (an API key, a remote
server) gets live text the same way.
Assisted-by: Claude (Opus 5.5 Medium)
---
ui/apps/chat/src/lib/voice/http-pieces.ts | 94 ++++++++++++++
ui/apps/chat/src/lib/voice/realtime.ts | 2 +-
ui/apps/chat/src/lib/voice/session.ts | 120 +++++++++++++-----
ui/apps/chat/tests/unit/voice-session.test.ts | 75 +++++++++++
4 files changed, 260 insertions(+), 31 deletions(-)
create mode 100644 ui/apps/chat/src/lib/voice/http-pieces.ts
diff --git a/ui/apps/chat/src/lib/voice/http-pieces.ts b/ui/apps/chat/src/lib/voice/http-pieces.ts
new file mode 100644
index 000000000000..37a98e0bbd2b
--- /dev/null
+++ b/ui/apps/chat/src/lib/voice/http-pieces.ts
@@ -0,0 +1,94 @@
+import { joinPieces } from './realtime';
+import { encodeWav } from './wav';
+
+export interface HttpPieceTranscriberOptions {
+ /** Transcribes one piece, e.g. with transcribeClip() */
+ transcribe: (wav: Blob, signal: AbortSignal) => Promise;
+ /** Rate of the PCM chunks given to append() */
+ sampleRate: number;
+ /** The whole text so far, each time a piece adds to it */
+ onText?: (text: string) => void;
+}
+
+/**
+ * Live text without a realtime session: for a model that hears but cannot hold one
+ * (Qwen3-ASR), or a server a browser websocket cannot reach. The recording is cut into
+ * pieces at the speaker's pauses (see cut()), each sent to `/v1/audio/transcriptions`
+ * as a WAV of its own, one at a time, and their text joined in order.
+ */
+export class HttpPieceTranscriber {
+ /** The pieces' text so far */
+ text = '';
+
+ private chunks: Int16Array[] = [];
+ private closed = false;
+ private inFlight: AbortController | null = null;
+ private length = 0;
+ private readonly options: HttpPieceTranscriberOptions;
+
+ /**
+ * Whether a piece can be cut now: audio since the last one, and none still being
+ * transcribed, so pieces come back in order; meanwhile the next one grows.
+ */
+ get canCut(): boolean {
+ return !this.closed && this.inFlight === null && this.length > 0;
+ }
+
+ /** Takes a PCM16 mono chunk at `options.sampleRate`. */
+ append(chunk: Int16Array): void {
+ if (this.closed || chunk.length === 0) return;
+
+ this.chunks.push(chunk);
+ this.length += chunk.length;
+ }
+
+ /** Drops what is pending and stops a piece being transcribed. */
+ close(): void {
+ this.closed = true;
+ this.inFlight?.abort();
+ this.inFlight = null;
+ this.chunks = [];
+ this.length = 0;
+ }
+
+ constructor(options: HttpPieceTranscriberOptions) {
+ this.options = options;
+ }
+
+ /** Ends the current piece and transcribes it; see canCut. */
+ cut(): void {
+ if (!this.canCut) return;
+
+ const samples = new Int16Array(this.length);
+
+ let offset = 0;
+
+ for (const chunk of this.chunks) {
+ samples.set(chunk, offset);
+ offset += chunk.length;
+ }
+
+ this.chunks = [];
+ this.length = 0;
+
+ const controller = new AbortController();
+
+ this.inFlight = controller;
+ this.options
+ .transcribe(encodeWav(samples, this.options.sampleRate), controller.signal)
+ .then(
+ (piece) => {
+ if (controller.signal.aborted || !piece.trim()) return;
+
+ this.text = joinPieces(this.text, piece.trim());
+ this.options.onText?.(this.text);
+ },
+ (error) => {
+ if (!controller.signal.aborted) console.warn('Transcribing a piece failed:', error);
+ }
+ )
+ .finally(() => {
+ if (this.inFlight === controller) this.inFlight = null;
+ });
+ }
+}
diff --git a/ui/apps/chat/src/lib/voice/realtime.ts b/ui/apps/chat/src/lib/voice/realtime.ts
index 50fa6c54cd97..5ccdc2ab8436 100644
--- a/ui/apps/chat/src/lib/voice/realtime.ts
+++ b/ui/apps/chat/src/lib/voice/realtime.ts
@@ -418,7 +418,7 @@ export class RealtimeTranscriber {
}
/** Two transcribed pieces as one text, a space between them where neither has one. */
-function joinPieces(before: string, piece: string): string {
+export function joinPieces(before: string, piece: string): string {
if (!before) return piece;
if (!piece) return before;
diff --git a/ui/apps/chat/src/lib/voice/session.ts b/ui/apps/chat/src/lib/voice/session.ts
index c812504a10c7..d3e81092a449 100644
--- a/ui/apps/chat/src/lib/voice/session.ts
+++ b/ui/apps/chat/src/lib/voice/session.ts
@@ -1,6 +1,7 @@
import { VoiceCapture } from './capture';
import { VOICE_LEVELS_INTERVAL_MS, VOICE_PIECE, VOICE_RETRANSCRIBE_MAX_MS } from './constants';
import { RealtimeUnsupportedError } from './errors';
+import { HttpPieceTranscriber } from './http-pieces';
import { PauseDetector } from './pauses';
import { RealtimeTranscriber } from './realtime';
import { transcribeClip } from './transcribe';
@@ -10,8 +11,10 @@ import type { VoiceSession, VoiceSessionOptions, VoiceSessionResult } from './ty
* Starts capturing and, when asked to transcribe, tries the realtime socket at the
* same time. Realtime gives live text: a live ASR model streams it, any other model
* that hears transcribes the recording piece by piece, cut at the speaker's pauses.
- * When the model cannot stream, the socket does not open, or the session fails, the
- * clip is transcribed over HTTP after recording.
+ * When the model cannot hold a realtime session (Qwen3-ASR) or the socket does not
+ * open, the pieces go over HTTP instead, one request each, so the text is live too.
+ * Either way, a recording cut into pieces gets one more pass over the whole clip
+ * at the end, punctuated by meaning rather than by the pauses.
* @throws VoiceCaptureError when the microphone does not start
*/
export async function startVoiceSession(options: VoiceSessionOptions = {}): Promise {
@@ -36,19 +39,69 @@ export async function startVoiceSession(options: VoiceSessionOptions = {}): Prom
let liveText: string | null = null;
let ended = false;
+ const showText = (text: string) => {
+ liveText = text;
+ options.onText?.(text);
+ };
+ // cuts the recording at the speaker's pauses: short ones, so the text keeps up with them
+ const cutAtPauses = (cut: () => boolean) => {
+ const pauses = new PauseDetector({
+ intervalMs: capture.levelsIntervalMs ?? VOICE_LEVELS_INTERVAL_MS,
+ maxSegmentMs: VOICE_PIECE.MAX_MS,
+ minSpeechMs: VOICE_PIECE.MIN_SPEECH_MS,
+ pauseMs: VOICE_PIECE.PAUSE_MS
+ });
+
+ unsubscribers.push(
+ capture.onLevels((levels) => {
+ const atPause = pauses.push(levels[levels.length - 1] ?? 0);
+
+ if (pauses.heardSpeech) speechSinceCommit = true;
+
+ if (atPause && cut()) speechSinceCommit = false;
+ })
+ );
+ };
+ // pieces over HTTP, for when realtime is not to be had; the audio is kept from the start,
+ // so nothing said while the socket answers is lost if the model turns out unable to stream
+ const httpPieces = transcribe
+ ? new HttpPieceTranscriber({
+ onText: showText,
+ sampleRate: capture.sampleRate,
+ transcribe: (wav, signal) =>
+ transcribeClip(transcribe.context, wav, {
+ filename: 'piece.wav',
+ language: transcribe.language,
+ model: transcribe.model,
+ signal
+ })
+ })
+ : null;
+
+ let piecesOverHttp = false;
+
+ if (httpPieces) unsubscribers.push(capture.onPcm((chunk) => httpPieces.append(chunk)));
+
+ const startHttpPieces = () => {
+ if (!httpPieces || ended) return;
+
+ piecesOverHttp = true;
+ cutAtPauses(() => {
+ if (!httpPieces.canCut) return false;
+
+ httpPieces.cut();
+
+ return true;
+ });
+ };
+
if (transcribe && options.realtime !== false) {
const realtime = new RealtimeTranscriber(transcribe.context, {
autoload: transcribe.autoload,
language: transcribe.language,
model: transcribe.model,
- onCompleted: (text) => {
- liveText = text;
- options.onText?.(text);
- },
- onDelta: (_delta, text) => {
- liveText = text;
- options.onText?.(text);
- },
+ onCompleted: showText,
+ onDelta: (_delta, text) => showText(text),
sampleRate: capture.sampleRate
});
const unsubscribePcm = capture.onPcm((chunk) => realtime.append(chunk));
@@ -57,32 +110,22 @@ export async function startVoiceSession(options: VoiceSessionOptions = {}): Prom
unsubscribers.push(unsubscribePcm);
realtime.connect().then(
() => {
+ // the socket has it all from here
+ httpPieces?.close();
+
if (ended) return;
options.onRealtime?.(realtime.live);
- // a model that transcribes per commit gets the speech piece by piece, at its pauses:
- // short ones, so the text keeps up with the speaker
+ // a model that transcribes per commit gets the speech piece by piece, at its pauses
if (!realtime.live) {
- const pauses = new PauseDetector({
- intervalMs: capture.levelsIntervalMs ?? VOICE_LEVELS_INTERVAL_MS,
- maxSegmentMs: VOICE_PIECE.MAX_MS,
- minSpeechMs: VOICE_PIECE.MIN_SPEECH_MS,
- pauseMs: VOICE_PIECE.PAUSE_MS
- });
-
- unsubscribers.push(
- capture.onLevels((levels) => {
- const cut = pauses.push(levels[levels.length - 1] ?? 0);
+ cutAtPauses(() => {
+ if (!realtime.canCommitSegment) return false;
- if (pauses.heardSpeech) speechSinceCommit = true;
+ realtime.commitSegment();
- if (cut && realtime.canCommitSegment) {
- realtime.commitSegment();
- speechSinceCommit = false;
- }
- })
- );
+ return true;
+ });
}
},
(error) => {
@@ -91,14 +134,16 @@ export async function startVoiceSession(options: VoiceSessionOptions = {}): Prom
if (ended) return;
if (!(error instanceof RealtimeUnsupportedError)) {
- console.warn('Realtime transcription unavailable, transcribing after recording:', error);
+ console.warn('Realtime transcription unavailable, transcribing over HTTP:', error);
}
options.onRealtime?.(false);
+ startHttpPieces();
}
);
} else if (transcribe) {
options.onRealtime?.(false);
+ startHttpPieces();
}
const release = () => {
@@ -114,6 +159,7 @@ export async function startVoiceSession(options: VoiceSessionOptions = {}): Prom
release();
capture.cancel();
transcriber?.close();
+ httpPieces?.close();
},
capture,
async stop(): Promise {
@@ -125,6 +171,7 @@ export async function startVoiceSession(options: VoiceSessionOptions = {}): Prom
} catch (error) {
release();
transcriber?.close();
+ httpPieces?.close();
throw error;
}
@@ -167,9 +214,22 @@ export async function startVoiceSession(options: VoiceSessionOptions = {}): Prom
}
)
: pieces;
+ } else if (piecesOverHttp && httpPieces) {
+ // the whole clip now, punctuated by meaning; a piece still on its way would hold the server up
+ const fromPieces = httpPieces.text;
+
+ httpPieces.close();
+ transcript = overHttp().catch((error) => {
+ if (!fromPieces) throw error;
+
+ console.warn('Transcribing the whole recording failed, keeping its pieces:', error);
+
+ return fromPieces;
+ });
} else {
// still connecting at the end: the HTTP request is the shorter wait
realtime?.close();
+ httpPieces?.close();
transcript = overHttp();
}
diff --git a/ui/apps/chat/tests/unit/voice-session.test.ts b/ui/apps/chat/tests/unit/voice-session.test.ts
index 7053321fb210..d6d67dfadef9 100644
--- a/ui/apps/chat/tests/unit/voice-session.test.ts
+++ b/ui/apps/chat/tests/unit/voice-session.test.ts
@@ -243,6 +243,81 @@ describe('startVoiceSession', () => {
warn.mockRestore();
});
+ describe('a model that cannot stream (Qwen3-ASR)', () => {
+ /** Answers a piece with "Piece N." and the whole clip with `whole`, or fails it. */
+ function server(whole: string | null) {
+ let pieces = 0;
+
+ return vi.fn(async (_url: string, init: RequestInit) => {
+ const file = (init.body as FormData).get('file') as File;
+
+ if (file.name === 'piece.wav') {
+ return new Response(JSON.stringify({ text: `Piece ${++pieces}.` }));
+ }
+
+ return whole === null
+ ? new Response('busy', { status: 503 })
+ : new Response(JSON.stringify({ text: whole }));
+ });
+ }
+
+ /** Speaks two pieces with a pause after each, over HTTP once the socket says no. */
+ async function speakOverHttp(fetchMock: ReturnType) {
+ const onText = vi.fn();
+ const { capture, emitPcm, holdLevel } = fakeCapture();
+ const session = await startVoiceSession({
+ capture,
+ onText,
+ transcribe: { context: serverContext(fetchMock as unknown as typeof fetch), model: 'asr' }
+ });
+
+ // said before the socket answered: kept for the first piece
+ emitPcm(Int16Array.from([1, 2]));
+ FakeWebSocket.instances[0].emit({
+ error: { message: 'The current model does not support realtime transcription.' },
+ type: 'error'
+ });
+ await flush();
+
+ holdLevel(0.05, 200);
+ holdLevel(0.6, 500);
+ holdLevel(0.05, 400);
+ await flush();
+ expect(onText).toHaveBeenLastCalledWith('Piece 1.');
+
+ emitPcm(Int16Array.from([3]));
+ holdLevel(0.6, 500);
+ holdLevel(0.05, 400);
+ await flush();
+ expect(onText).toHaveBeenLastCalledWith('Piece 1. Piece 2.');
+
+ return session.stop();
+ }
+
+ it('gives live text piece by piece over HTTP, then the whole clip', async () => {
+ const fetchMock = server('Whole.');
+ const result = await speakOverHttp(fetchMock);
+
+ expect(result.liveText).toBe('Piece 1. Piece 2.');
+ await expect(result.transcript).resolves.toBe('Whole.');
+ // two pieces and the whole clip
+ expect(fetchMock).toHaveBeenCalledTimes(3);
+
+ const first = (fetchMock.mock.calls[0][1].body as FormData).get('file') as File;
+
+ // the audio from before the socket answered went with the first piece
+ expect(first.size).toBeGreaterThan(44);
+ });
+
+ it('keeps the pieces when the whole clip fails', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ const result = await speakOverHttp(server(null));
+
+ await expect(result.transcript).resolves.toBe('Piece 1. Piece 2.');
+ warn.mockRestore();
+ });
+ });
+
it('uses HTTP when realtime is still connecting at the end', async () => {
const fetchMock = vi.fn(async () => new Response(JSON.stringify({ text: 'Late.' })));
const { capture } = fakeCapture();
From 7d1a07c0d1808f5d19dda012a034606a232f334d Mon Sep 17 00:00:00 2001
From: Aleksander Grygier
Date: Tue, 17 Nov 2026 12:02:33 +0100
Subject: [PATCH 17/19] ui: say when the browser processes the microphone
despite the raw request
The browser keeps one processing setting per microphone: while a call,
another tab or voice mode has it open with echo cancellation, a
dictation that asked for the raw signal is processed too, sounds
distant, and the system may lower other sound. The capture now warns
when it gets processing it did not ask for, and its development log
names the microphone, its rate and what processing is on.
Assisted-by: Claude (Opus 5.5 Medium)
---
ui/apps/chat/src/lib/voice/capture.ts | 40 ++++++++++++++++++++++++++-
1 file changed, 39 insertions(+), 1 deletion(-)
diff --git a/ui/apps/chat/src/lib/voice/capture.ts b/ui/apps/chat/src/lib/voice/capture.ts
index fc843c59c85c..e081a868f189 100644
--- a/ui/apps/chat/src/lib/voice/capture.ts
+++ b/ui/apps/chat/src/lib/voice/capture.ts
@@ -161,6 +161,9 @@ export class VoiceCapture {
}
});
timing.mark('microphone');
+ timing.note(
+ checkProcessing(stream, { ...VOICE_MICROPHONE_CONSTRAINTS, ...this.options.constraints })
+ );
if (dropped()) throw new DOMException('Voice capture cancelled', 'AbortError');
@@ -338,10 +341,42 @@ function releaseDevice(stream: MediaStream | null, context: AudioContext | null)
if (context) idleCaptureContext(context);
}
+const PROCESSING = ['echoCancellation', 'noiseSuppression', 'autoGainControl'] as const;
+
+/**
+ * What the browser gave for the microphone, e.g. "MacBook Pro Microphone, 48000 Hz, raw".
+ * Processing that was asked off but is on is warned about: the browser keeps one setting
+ * per device, so a page or app that has the same microphone open with echo cancellation
+ * (a call, voice mode) gets the recording processed too, and the system may lower other
+ * sound while it lasts.
+ */
+function checkProcessing(stream: MediaStream, asked: MediaTrackConstraints): string {
+ const track = stream.getAudioTracks()[0];
+
+ if (!track) return 'no audio track';
+
+ const settings = track.getSettings();
+ const on = PROCESSING.filter((key) => settings[key] === true);
+ const unwanted = on.filter((key) => asked[key] === false);
+
+ if (unwanted.length > 0) {
+ console.warn(
+ `[voice] ${track.label || 'The microphone'} is processed (${unwanted.join(', ')}) although the raw signal was asked for; another tab or app may have it open for a call.`
+ );
+ }
+
+ const rate = settings.sampleRate ? `${settings.sampleRate} Hz` : null;
+
+ return [track.label || 'microphone', rate, on.length > 0 ? on.join(' + ') : 'raw']
+ .filter(Boolean)
+ .join(', ');
+}
+
/** How long each step of a start took, logged in development to see where a slow start goes. */
function startTiming() {
const startedAt = performance.now();
const steps: string[] = [];
+ const notes: string[] = [];
let last = startedAt;
@@ -352,11 +387,14 @@ function startTiming() {
steps.push(`${step} ${Math.round(now - last)} ms`);
last = now;
},
+ note(text: string) {
+ notes.push(text);
+ },
report() {
if (!import.meta.env?.DEV) return;
console.debug(
- `[voice] recording after ${Math.round(performance.now() - startedAt)} ms: ${steps.join(', ')}`
+ `[voice] recording after ${Math.round(performance.now() - startedAt)} ms: ${steps.join(', ')}${notes.length > 0 ? ` (${notes.join('; ')})` : ''}`
);
}
};
From c3efab211f9217edef88a16c3baeafee05570c75 Mon Sep 17 00:00:00 2001
From: Aleksander Grygier
Date: Tue, 17 Nov 2026 16:25:48 +0100
Subject: [PATCH 18/19] ui: keep the microphone beside the stop and send
buttons
While a reply streams, or once the message has text, the microphone no longer
gives way to the main button: it stays beside it as a quieter, secondary
button, so more can be dictated without clearing the message or waiting for
the reply.
Assisted-by: Claude (Opus 5.5 Medium)
---
.../ChatFormActionRecord.svelte | 9 +++-
.../ChatFormActions/ChatFormActions.svelte | 19 +++++++-
.../client/chat-form-voice.svelte.test.ts | 46 ++++++++++++++++++-
.../ChatFormVoiceTestWrapper.svelte | 5 +-
4 files changed, 73 insertions(+), 6 deletions(-)
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte
index 0b90b0e48d2e..7c3dcfe49860 100644
--- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte
+++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte
@@ -17,6 +17,8 @@
micWarning?: string | null;
/** Why no model can take the recording; disables the button. */
unavailableReason?: string | null;
+ /** A quieter button, beside the main one (stop or send). */
+ secondary?: boolean;
onMicClick?: () => void;
}
@@ -29,6 +31,7 @@
isRecording = false,
micWarning = null,
onMicClick,
+ secondary = false,
unavailableReason = null
}: Props = $props();
@@ -73,7 +76,7 @@
{/if}
- {#if isLoading && !canSend}
+ {#if shouldShowRecordBeside}
+
+ {/if}
+
+ {#if showStop}