From 26fa41a7672fa122f9719ced749dd529f861b2cd Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 16 Nov 2026 10:37:11 +0100 Subject: [PATCH 01/10] ui: add voice mode, a spoken conversation with the chat model The chat form's main button on an empty message is now voice mode (a waveform); the microphone steps back beside it and records as before. Voice mode is a full-screen conversation: - it listens with the browser's echo cancellation on, shows what is said as it is transcribed, and sends it after a pause of about a second; - the reply streams on screen and is read aloud sentence by sentence through /v1/audio/speech, by the chat model when it speaks or else by another model that does (router mode), each word lit as it is spoken; - speaking over the reply, or tapping the microphone, stops it and listens; the microphone and the speaker can be muted, and with no model that speaks the replies are shown without being read. The voice engine gains synthesizeSpeech(), readAloudPieces() and ReplyReader; models report output_modalities, so the app knows which can speak. Assisted-by: Claude (Opus 5.5 Medium) (cherry picked from commit 8c580d021e27f80f487b4598bb0e0f4cf3b96cb3) --- .../app/chat/ChatForm/ChatForm.svelte | 16 +- .../ChatFormActionRecord.svelte | 2 +- .../ChatFormActionVoiceMode.svelte | 46 ++ .../ChatFormActions/ChatFormActions.svelte | 17 + .../app/chat/ChatScreen/ChatScreen.svelte | 7 +- .../app/chat/ChatScreen/ChatScreenForm.svelte | 1 + .../app/chat/VoiceMode/VoiceModeScreen.svelte | 220 ++++++++ .../chat/src/lib/components/app/chat/index.ts | 10 + .../chat/src/lib/constants/voice.constants.ts | 19 + .../hooks/use-voice-conversation.svelte.ts | 479 ++++++++++++++++++ .../src/lib/stores/models/index.svelte.ts | 1 + ui/apps/chat/src/lib/stores/ui.svelte.ts | 2 + ui/apps/chat/src/lib/types/api.d.ts | 2 + ui/apps/chat/src/lib/types/models.d.ts | 2 + ui/apps/chat/src/lib/voice/constants.ts | 1 + ui/apps/chat/src/lib/voice/index.ts | 4 + ui/apps/chat/src/lib/voice/read-aloud.ts | 61 +++ .../chat/src/lib/voice/reply-reader.svelte.ts | 240 +++++++++ ui/apps/chat/src/lib/voice/speech.ts | 49 ++ .../client/chat-form-voice.svelte.test.ts | 73 ++- .../ChatFormVoiceTestWrapper.svelte | 5 +- .../chat/tests/unit/voice-read-aloud.test.ts | 44 ++ 22 files changed, 1293 insertions(+), 8 deletions(-) create mode 100644 ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionVoiceMode.svelte create mode 100644 ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte create mode 100644 ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts create mode 100644 ui/apps/chat/src/lib/voice/read-aloud.ts create mode 100644 ui/apps/chat/src/lib/voice/reply-reader.svelte.ts create mode 100644 ui/apps/chat/src/lib/voice/speech.ts create mode 100644 ui/apps/chat/tests/unit/voice-read-aloud.test.ts diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte index 5b296c62dced..12b66d18e57c 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -64,7 +64,7 @@ parseClipboardContent, uuid } from '#lib/utils/index.ts'; - import { VoiceRecorder, warmUpVoiceCapture } from '#lib/voice/index.ts'; + import { unlockVoicePlayback, VoiceRecorder, warmUpVoiceCapture } from '#lib/voice/index.ts'; import { onMount, tick, untrack } from 'svelte'; import type { ClassValue } from 'svelte/elements'; import { toast } from 'svelte-sonner'; @@ -99,6 +99,8 @@ /** Records voice input; tests and stories pass a fake, the app a VoiceRecorder. */ voiceRecorder?: VoiceRecorderLike; + /** Offers voice mode, a spoken conversation, as the main button of an empty message. */ + voiceMode?: boolean; } let { @@ -121,6 +123,7 @@ singleRow = false, uploadedFiles = $bindable([]), value = $bindable(''), + voiceMode = false, voiceRecorder }: Props = $props(); @@ -668,6 +671,16 @@ onSubmit?.(); } + // voice mode leads only when what is said can be transcribed: the chat model hears, or a + // transcription model is set or loaded; else the microphone leads, saying why it cannot + let voiceModeAvailable = $derived(voiceMode && !voice.unavailableReason); + + // inside the click: a browser that plays sound only after a gesture lets the replies play later + function handleVoiceModeClick() { + unlockVoicePlayback(); + uiStore.voiceModeOpen = true; + } + // Escape drops a recording, wherever the focus is function handleWindowKeydown(event: KeyboardEvent) { if (event.key !== KeyboardKey.ESCAPE || !voice.isActive || event.defaultPrevented) return; @@ -862,6 +875,7 @@ {onStop} onSystemPromptClick={() => onSystemPromptClick?.({ files: uploadedFiles, message: value })} + onVoiceModeClick={voiceModeAvailable ? handleVoiceModeClick : undefined} onWorkingDirectoryClick={isWorkingDirectoryInAddMenu ? pickers.handleWorkingDirectoryOpen : undefined} diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte index 73069301474e..439e4f98cea3 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte @@ -17,7 +17,7 @@ micWarning?: string | null; /** Why no model can take the recording; disables the button. */ unavailableReason?: string | null; - /** A quieter button, beside the main one (stop or send). */ + /** A quieter button, beside the main one: stop, send or voice mode. */ secondary?: boolean; /** With `secondary`: filled like the stop button it stands beside, not bare. */ filled?: boolean; diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionVoiceMode.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionVoiceMode.svelte new file mode 100644 index 000000000000..67bba80b724c --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionVoiceMode.svelte @@ -0,0 +1,46 @@ + + + +
+ + + + + + +

{unavailableReason ?? VOICE_CONVERSATION_MESSAGE.open}

+
+
+
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte index a523300286a8..f66224f0b4ad 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte @@ -6,6 +6,7 @@ ChatFormActionRecord, ChatFormActionsAdd, ChatFormActionSubmit, + ChatFormActionVoiceMode, ChatFormContextGauge, ModelsSelector } from '#lib/components/app/index.ts'; @@ -50,6 +51,10 @@ workingDirectory?: string | null; onFileUpload?: () => void; onMicClick?: () => void; + /** Opens voice mode; left out, the microphone is the main button. */ + onVoiceModeClick?: () => void; + /** Why voice mode cannot listen. */ + voiceModeUnavailableReason?: string | null; onStop?: () => void; onSystemPromptClick?: () => void; onWorkingDirectoryClick?: () => void; @@ -69,11 +74,13 @@ onMicClick, onStop, onSystemPromptClick, + onVoiceModeClick, onWorkingDirectoryClick, showAddButton = true, showModelSelector = true, uploadedFiles = [], voice, + voiceModeUnavailableReason = null, workingDirectory = null }: Props = $props(); @@ -236,6 +243,7 @@ /> {:else if shouldShowRecordButton} + + + {#if onVoiceModeClick} + + {/if} {:else} {/if} +{#if uiStore.voiceModeOpen} + +{/if} + diff --git a/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte new file mode 100644 index 000000000000..b90db50b10de --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte @@ -0,0 +1,220 @@ + + + + + + diff --git a/ui/apps/chat/src/lib/components/app/chat/index.ts b/ui/apps/chat/src/lib/components/app/chat/index.ts index f7626c73a4f5..554d77333a0f 100644 --- a/ui/apps/chat/src/lib/components/app/chat/index.ts +++ b/ui/apps/chat/src/lib/components/app/chat/index.ts @@ -185,6 +185,16 @@ export { default as ChatFormActionsAdd } from './ChatForm/ChatFormActions/ChatFo */ export { default as ChatFormActionRecord } from './ChatForm/ChatFormActions/ChatFormActionRecord.svelte'; +/** + * **ChatFormActionVoiceMode** - Opens voice mode, the chat form's main button while the message is empty + */ +export { default as ChatFormActionVoiceMode } from './ChatForm/ChatFormActions/ChatFormActionVoiceMode.svelte'; + +/** + * **VoiceModeScreen** - Full-screen spoken conversation with the chat model + */ +export { default as VoiceModeScreen } from './VoiceMode/VoiceModeScreen.svelte'; + /** * The actions row while the chat form records: cancel, the live waveform and the * elapsed time, and stop, with a voice message's streaming text under the waveform diff --git a/ui/apps/chat/src/lib/constants/voice.constants.ts b/ui/apps/chat/src/lib/constants/voice.constants.ts index 7c49dcfff478..b152017e11e1 100644 --- a/ui/apps/chat/src/lib/constants/voice.constants.ts +++ b/ui/apps/chat/src/lib/constants/voice.constants.ts @@ -16,6 +16,25 @@ export const VOICE_MODE_MESSAGE = { voiceMessage: 'Send a voice message' } as const; +/** What voice mode says: its states, buttons and the reason it cannot read replies aloud. */ +export const VOICE_CONVERSATION_MESSAGE = { + close: 'Leave voice mode', + greeting: "Let's talk", + interrupt: 'Interrupt the reply', + listening: 'Listening...', + micOff: 'Microphone off', + mute: 'Turn the microphone off', + noSpeechModel: + 'No model that speaks is available, so replies are shown, not read aloud. Load a model that generates audio to hear them.', + open: 'Voice mode', + speakerOff: 'Stop reading replies aloud', + speakerOn: 'Read replies aloud', + starting: 'Starting...', + thinking: 'Thinking...', + transcribing: 'Transcribing...', + unmute: 'Turn the microphone on' +} as const; + /** Where the chat form remembers that it showed the hint about a separate transcription model. */ export const VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.voiceSeparateModelHintSeen`; diff --git a/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts new file mode 100644 index 000000000000..3ab477a20a34 --- /dev/null +++ b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts @@ -0,0 +1,479 @@ +/** + * Voice mode: a spoken conversation with the chat model, hands-free. + * + * Each turn listens (echo cancellation on, as the reply plays from the same device), + * shows what is said as it is transcribed, and ends after a pause in speech: the text + * goes to the chat as a message. The reply streams on screen and, when a model that + * speaks is available, is read aloud sentence by sentence; then the next turn listens. + * Speaking while the reply plays stops it and starts a new turn. + */ + +import { SETTINGS_KEYS } from '#lib/constants/index.ts'; +import { MessageRole, VoiceInputMode, VoiceMode } from '#lib/enums/index.ts'; +import { + chatStore, + conversationsStore, + modelsStore, + serverStore, + settingsStore +} from '#lib/stores/index.ts'; +import { getAuthHeaders } from '#lib/utils/api-headers.ts'; +import { describeMicrophoneError } from '#lib/utils/audio-recording.ts'; +import { withBase } from '#lib/utils/base-path.ts'; +import { resolveVoiceMode } from '#lib/utils/voice-mode.ts'; +import { + PauseDetector, + ReplyReader, + resolveMicrophone, + synthesizeSpeech, + VOICE_LEVELS_INTERVAL_MS, + VOICE_MICROPHONE_AUTO, + VoiceCapture, + VoiceCaptureError, + VoiceRecorder, + type VoiceServerContext +} from '#lib/voice/index.ts'; +import { onDestroy, untrack } from 'svelte'; + +/** Where a voice conversation is. */ +export type VoiceConversationState = + | 'idle' + | 'starting' + | 'listening' + | 'transcribing' + | 'thinking' + | 'speaking' + | 'paused'; + +/** One exchange: what was said, and the reply. */ +export interface VoiceTurn { + id: number; + user: string; + reply: string; + /** The message went to the chat, so its reply is followed */ + sent: boolean; + done: boolean; +} + +// voice mode talks to a device that also plays the reply: the browser's clean-up removes its echo +const CONVERSATION_CONSTRAINTS: MediaTrackConstraints = { + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true +}; +/** Silence after speech that ends a turn. */ +const TURN_PAUSE_MS = 900; +/** Speech over the reply that interrupts it. */ +const BARGE_IN_MS = 300; + +function serverContext(): { context: VoiceServerContext; realtime: boolean } { + const headers = getAuthHeaders(); + const hasAuth = Object.keys(headers).some((key) => key.toLowerCase() === 'authorization'); + + return { + context: { + getHeaders: () => getAuthHeaders(), + resolveUrl: (path) => withBase(path.replace(/^\./, '')) + }, + realtime: !hasAuth + }; +} + +export function useVoiceConversation() { + const config = $derived(settingsStore.config); + const isRouter = $derived(serverStore.isRouterMode); + const chatModel = $derived(modelsStore.activeModelId); + const hasAudio = (model: string | null) => { + void modelsStore.props.cacheVersion; + + return model ? modelsStore.props.modelSupportsAudio(model) : false; + }; + const transcriptionSetting = $derived( + String(config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL] ?? '') + ); + // what is said always becomes text here, so it is resolved as dictation + const resolution = $derived.by(() => { + void modelsStore.props.cacheVersion; + + const loadedAudioModel = isRouter + ? (modelsStore.models.find( + (option) => hasAudio(option.model) && modelsStore.isModelLoaded(option.model) + )?.model ?? null) + : null; + + return resolveVoiceMode({ + chatModel, + chatModelHasAudio: hasAudio(chatModel), + isRouter, + loadedAudioModel, + setting: VoiceInputMode.DICTATION, + transcriptionModel: transcriptionSetting, + transcriptionModelHasAudio: hasAudio(transcriptionSetting || null) + }); + }); + // the chat model reads its own replies when it can speak, else another model that can + const speechModel = $derived.by(() => { + const options = modelsStore.models; + const chat = options.find((option) => option.model === chatModel); + + if (chat?.speaks) return chat.model; + + return isRouter ? (options.find((option) => option.speaks)?.model ?? null) : null; + }); + + let state = $state('idle'); + let micMuted = $state(false); + let speakerMuted = $state(false); + let liveText = $state(''); + let levels = $state([]); + let error = $state(null); + let turns = $state([]); + let recorder: VoiceRecorder | null = null; + let monitor: VoiceCapture | null = null; + let generation = 0; + let turnId = 0; + // the current reply was seen loading: until then, not loading does not mean done + let replyStarted = false; + + const reader = new ReplyReader({ + onIdle: () => { + if (state === 'speaking' || state === 'thinking') void listen(); + }, + synthesize: (text, signal) => { + const { context } = serverContext(); + + return synthesizeSpeech(context, text, { + model: isRouter && speechModel ? speechModel : undefined, + signal + }); + } + }); + const currentTurn = $derived(turns[turns.length - 1]); + + // the reply as it streams, then as stored once done; the conversation is read here, as a + // first message creates it while being sent + $effect(() => { + const turn = currentTurn; + const convId = conversationsStore.activeConversation?.id; + + if (!turn || !turn.sent || turn.done || !convId) return; + + const streaming = chatStore.getChatStreaming(convId); + const loading = chatStore.isChatLoading(convId); + + untrack(() => { + if (loading) { + replyStarted = true; + + if (!streaming) return; + + turn.reply = streaming.response; + reader.update(streaming.response, false); + + return; + } + + if (replyStarted) finishReply(turn); + }); + }); + + /** The reply is complete: as stored, read to its end. */ + function finishReply(turn: VoiceTurn) { + if (turn.done) return; + + const last = conversationsStore.activeMessages.at(-1); + const text = last?.role === MessageRole.ASSISTANT ? last.content : turn.reply; + + turn.reply = text; + turn.done = true; + reader.update(text, true); + } + + // the reply plays: speaking over it may interrupt it + $effect(() => { + if (reader.speaking && state === 'thinking') { + untrack(() => { + state = 'speaking'; + void watchForBargeIn(generation); + }); + } + }); + + async function microphone(): Promise { + return resolveMicrophone( + String(config[SETTINGS_KEYS.VOICE_MICROPHONE] ?? VOICE_MICROPHONE_AUTO) + ); + } + + function stopRecorder() { + recorder?.cancel(); + recorder = null; + } + + function stopMonitor() { + monitor?.cancel(); + monitor = null; + } + + /** Starts a turn: listens until the speaker pauses, then sends what was said. */ + async function listen() { + stopMonitor(); + stopRecorder(); + liveText = ''; + levels = []; + + if (micMuted) { + state = 'paused'; + + return; + } + + if (resolution.mode === VoiceMode.UNAVAILABLE) { + error = resolution.reason; + state = 'paused'; + + return; + } + + const run = ++generation; + const model = resolution.transcriptionModel; + const { context, realtime } = serverContext(); + const language = String(config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE] ?? '').trim(); + const pauses = new PauseDetector({ + intervalMs: VOICE_LEVELS_INTERVAL_MS, + maxSegmentMs: Number.POSITIVE_INFINITY, + pauseMs: TURN_PAUSE_MS + }); + const active = new VoiceRecorder(); + + recorder = active; + state = 'starting'; + + try { + const deviceId = await microphone(); + + await active.start({ + capture: { constraints: CONVERSATION_CONSTRAINTS, ...(deviceId ? { deviceId } : {}) }, + onLevels: (next) => { + levels = next as number[]; + + if (run === generation && state === 'listening' && pauses.push(next.at(-1) ?? 0)) { + void endTurn(run); + } + }, + onText: (text) => { + if (run === generation) liveText = text; + }, + realtime, + transcribe: { + context, + language: language || undefined, + model: isRouter && model ? model : undefined + } + }); + } catch (cause) { + if (run !== generation) return; + + error = cause instanceof VoiceCaptureError ? cause.message : describeMicrophoneError(cause); + state = 'paused'; + + return; + } + + if (run !== generation) { + active.cancel(); + + return; + } + + error = null; + state = 'listening'; + } + + /** The speaker paused: what they said goes to the chat. */ + async function endTurn(run: number) { + const active = recorder; + + if (!active || run !== generation) return; + + state = 'transcribing'; + recorder = null; + + let text = liveText; + + try { + const result = await active.stop(); + + text = (await (result.transcript ?? Promise.resolve(text))) || result.liveText || text; + } catch (cause) { + console.warn('Voice mode: the turn could not be transcribed:', cause); + } + + if (run !== generation) return; + + text = text.trim(); + + // nothing understood: listen again + if (!text) { + void listen(); + + return; + } + + const turn: VoiceTurn = { done: false, id: ++turnId, reply: '', sent: false, user: text }; + + turns.push(turn); + liveText = ''; + reader.reset(); + reader.muted = speakerMuted || !speechModel; + replyStarted = false; + state = 'thinking'; + + const sending = chatStore.sendMessage(text); + + // followed from here: the message is on its way + turns[turns.length - 1].sent = true; + + try { + await sending; + + // a reply that was never seen streaming (an error, a very short one) ends here + if (run === generation) finishReply(turns[turns.length - 1]); + } catch (cause) { + if (run !== generation) return; + + error = cause instanceof Error ? cause.message : String(cause); + void listen(); + } + } + + /** While the reply plays, speech over it stops it and starts a new turn. */ + async function watchForBargeIn(run: number) { + if (micMuted || speakerMuted || !speechModel || monitor) return; + + const capture = new VoiceCapture({ constraints: CONVERSATION_CONSTRAINTS }); + + let speechMs = 0; + let floor = 1; + + monitor = capture; + capture.onLevels((next) => { + const level = next.at(-1) ?? 0; + + floor = Math.min(level, floor + 0.002); + speechMs = level > floor + 0.2 ? speechMs + VOICE_LEVELS_INTERVAL_MS : 0; + + if (speechMs >= BARGE_IN_MS && run === generation && reader.speaking) interrupt(); + }); + + try { + await capture.start(); + } catch { + // no barge-in without a microphone; the turn still follows the reply + } + + if (run !== generation || monitor !== capture || state !== 'speaking') { + capture.cancel(); + + if (monitor === capture) monitor = null; + } + } + + /** Stops the reply where it is, and listens. */ + function interrupt() { + reader.stop(); + + const convId = conversationsStore.activeConversation?.id; + + if (convId && chatStore.isChatLoading(convId)) void chatStore.stopGeneration(); + + const turn = currentTurn; + + if (turn) turn.done = true; + + void listen(); + } + + /** Starts the conversation; the click that opened voice mode unlocked playback (unlockVoicePlayback). */ + function open() { + error = null; + turns = []; + void listen(); + } + + function close() { + generation++; + stopRecorder(); + stopMonitor(); + reader.stop(); + state = 'idle'; + liveText = ''; + } + + function toggleMic() { + micMuted = !micMuted; + + if (micMuted) { + if (state === 'listening' || state === 'starting') { + generation++; + stopRecorder(); + state = 'paused'; + } + + stopMonitor(); + + return; + } + + if (state === 'paused') void listen(); + } + + function toggleSpeaker() { + speakerMuted = !speakerMuted; + reader.muted = speakerMuted || !speechModel; + + if (speakerMuted) stopMonitor(); + } + + onDestroy(close); + + return { + /** Whether replies can be read aloud: a model that speaks is available. */ + get canSpeak() { + return !!speechModel; + }, + close, + get error() { + return error; + }, + interrupt, + get levels() { + return levels; + }, + get liveText() { + return liveText; + }, + get micMuted() { + return micMuted; + }, + open, + get reader() { + return reader; + }, + get speakerMuted() { + return speakerMuted; + }, + get state() { + return state; + }, + toggleMic, + toggleSpeaker, + get turns() { + return turns; + }, + /** Why voice mode cannot listen, or null. */ + get unavailableReason() { + return resolution.mode === VoiceMode.UNAVAILABLE ? resolution.reason : null; + } + }; +} diff --git a/ui/apps/chat/src/lib/stores/models/index.svelte.ts b/ui/apps/chat/src/lib/stores/models/index.svelte.ts index 67518bf8bddf..7928defaad30 100644 --- a/ui/apps/chat/src/lib/stores/models/index.svelte.ts +++ b/ui/apps/chat/src/lib/stores/models/index.svelte.ts @@ -690,6 +690,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { model: modelId, name: this.toDisplayName(displayNameSource), parsedId: ModelsService.parseModelId(modelId), + speaks: item.architecture?.output_modalities?.includes('audio') ?? false, tags: item.tags ?? [] }; }) diff --git a/ui/apps/chat/src/lib/stores/ui.svelte.ts b/ui/apps/chat/src/lib/stores/ui.svelte.ts index 707648d06de7..1ba36014c014 100644 --- a/ui/apps/chat/src/lib/stores/ui.svelte.ts +++ b/ui/apps/chat/src/lib/stores/ui.svelte.ts @@ -107,6 +107,8 @@ class UiStore { sidebarSectionOrder = $state(loadSidebarSectionOrder()); /** Expanded desktop sidebar width in px; null keeps the stylesheet default. */ sidebarWidth = $state(this.loadSidebarWidth()); + /** Whether the full-screen voice mode is open: a spoken conversation with the chat model. */ + voiceModeOpen = $state(false); #isSidebarExpanded = $state(this.loadSidebarExpanded()); diff --git a/ui/apps/chat/src/lib/types/api.d.ts b/ui/apps/chat/src/lib/types/api.d.ts index abdd1b3de36e..371913fc4c26 100644 --- a/ui/apps/chat/src/lib/types/api.d.ts +++ b/ui/apps/chat/src/lib/types/api.d.ts @@ -121,6 +121,8 @@ export interface ApiModelDataEntry { export interface ApiModelArchitecture { /** Accepted input modalities, always contains "text" */ input_modalities: string[]; + /** Generated output modalities: "audio" when the model can speak */ + output_modalities?: string[]; } /** diff --git a/ui/apps/chat/src/lib/types/models.d.ts b/ui/apps/chat/src/lib/types/models.d.ts index 9d0594073406..9d4a7c88882b 100644 --- a/ui/apps/chat/src/lib/types/models.d.ts +++ b/ui/apps/chat/src/lib/types/models.d.ts @@ -23,6 +23,8 @@ export interface ModelOption { /** Draft sidecars downloaded alongside the model, e.g. `mtp` or `dflash`. */ draftSidecars?: ModelSidecarBadge[]; modalities?: ModelModalities; + /** The model can generate speech, e.g. to read replies aloud. */ + speaks?: boolean; details?: ApiModelDetails['details']; meta?: ApiModelDataEntry['meta']; parsedId?: ParsedModelId; diff --git a/ui/apps/chat/src/lib/voice/constants.ts b/ui/apps/chat/src/lib/voice/constants.ts index 7004254429d8..d38d050300f6 100644 --- a/ui/apps/chat/src/lib/voice/constants.ts +++ b/ui/apps/chat/src/lib/voice/constants.ts @@ -73,6 +73,7 @@ export const VOICE_BASE64_SLICE = 0x8000; /** Server routes, relative so a server under a subpath still works. */ export const VOICE_API_PATHS = { REALTIME: './v1/realtime', + SPEECH: './v1/audio/speech', TRANSCRIPTIONS: './v1/audio/transcriptions' } as const; diff --git a/ui/apps/chat/src/lib/voice/index.ts b/ui/apps/chat/src/lib/voice/index.ts index ed54f2db83b6..28f157753798 100644 --- a/ui/apps/chat/src/lib/voice/index.ts +++ b/ui/apps/chat/src/lib/voice/index.ts @@ -33,7 +33,11 @@ export { VOICE_MICROPHONE_DEFAULT } from './microphone'; export { computePeaks, computePeaksFromAudio, decodeAudio, mixToMono } from './peaks'; +export { PauseDetector, type PauseDetectorOptions } from './pauses'; +export { type ReadAloudPiece, readAloudPieces, toSpeech } from './read-aloud'; export { RealtimeTranscriber, realtimeUrl } from './realtime'; +export { ReplyReader, type ReplyReaderOptions, unlockVoicePlayback } from './reply-reader.svelte'; +export { synthesizeSpeech, type SynthesizeSpeechOptions } from './speech'; export { startVoiceSession } from './session'; export { transcribeClip } from './transcribe'; export type * from './types'; diff --git a/ui/apps/chat/src/lib/voice/read-aloud.ts b/ui/apps/chat/src/lib/voice/read-aloud.ts new file mode 100644 index 000000000000..d7ec29d2fd3b --- /dev/null +++ b/ui/apps/chat/src/lib/voice/read-aloud.ts @@ -0,0 +1,61 @@ +/** + * Cuts a reply into pieces to read aloud while it streams: whole sentences (or lines), + * each read as soon as it is complete, and the rest once the reply is done. Markdown + * marks are left out of what is spoken; the offsets point into the reply as written, + * so the text on screen can follow the voice. + */ + +export interface ReadAloudPiece { + /** Where the piece starts and ends in the reply as written */ + start: number; + end: number; + /** What is spoken: the piece without markdown marks */ + speech: string; +} + +// a sentence ends at . ! ? (with closing quotes or brackets) before whitespace, or a line ends; +// a list number such as "1." does not end one +const PIECE_END = /(?|[-*+]|\d+[.)])\s+/gm, '') + .replace(/[*_~]{1,3}([^*_~]+)[*_~]{1,3}/g, '$1') + .replace(/\s+/g, ' ') + .trim(); +} diff --git a/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts b/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts new file mode 100644 index 000000000000..3dcaa78f1f8d --- /dev/null +++ b/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts @@ -0,0 +1,240 @@ +import { type ReadAloudPiece, readAloudPieces } from './read-aloud'; + +// one element plays every reply: unlocked once inside a click, it may play later (iOS asks for that) +let sharedAudio: HTMLAudioElement | null = null; + +function playbackElement(): HTMLAudioElement { + sharedAudio ??= new Audio(); + + return sharedAudio; +} + +/** + * Plays nothing, inside a click: browsers that let a page play sound only after a gesture + * (iOS) then let replies play later. Call it in the handler of the click that opens voice mode. + */ +export function unlockVoicePlayback(): void { + if (typeof Audio === 'undefined') return; + + const audio = playbackElement(); + + audio.muted = true; + audio.play().catch(() => {}); + audio.pause(); + audio.muted = false; +} + +export interface ReplyReaderOptions { + /** Speaks a piece of text, e.g. with synthesizeSpeech(); the WAV or other audio to play */ + synthesize: (text: string, signal: AbortSignal) => Promise; + /** Called once the reply is done and all of it has been read (or skipped while muted) */ + onIdle?: () => void; +} + +/** + * Reads a reply aloud while it streams: give it the reply as it grows with update(), + * and it speaks each complete sentence in order, fetching the next one while the + * current one plays. `spoken` follows the voice through the reply as written, for + * VoiceSpokenText. Muted, it reads nothing and `spoken` keeps up with the text. + */ +export class ReplyReader { + /** Whether a piece is being fetched or played */ + speaking = $state(false); + /** Characters of the reply read so far */ + spoken = $state(0); + + private audio: HTMLAudioElement | null = null; + private controller = new AbortController(); + private done = false; + private frame: number | null = null; + private idle = true; + private isMuted = false; + private readonly options: ReplyReaderOptions; + private playing = false; + // bookkeeping the UI never reads + // eslint-disable-next-line svelte/prefer-svelte-reactivity + private prefetched = new Map>(); + private queue: ReadAloudPiece[] = []; + private queuedTo = 0; + private text = ''; + + get muted(): boolean { + return this.isMuted; + } + + set muted(value: boolean) { + this.isMuted = value; + + if (value) this.silence(); + } + + constructor(options: ReplyReaderOptions) { + this.options = options; + } + + /** Starts a new reply. */ + reset(): void { + this.stop(); + this.controller = new AbortController(); + this.text = ''; + this.done = false; + this.queuedTo = 0; + this.spoken = 0; + this.idle = false; + } + + /** Stops reading: the current piece, and what was still to come. */ + stop(): void { + this.controller.abort(); + this.queue = []; + this.prefetched.clear(); + this.stopAudio(); + this.playing = false; + this.speaking = false; + this.idle = true; + } + + /** The reply so far; `done` once it is complete. */ + update(text: string, done: boolean): void { + this.text = text; + this.done = done; + + for (const piece of readAloudPieces(text, this.queuedTo, done)) { + this.queue.push(piece); + this.queuedTo = piece.end; + } + + void this.pump(); + } + + private element(): HTMLAudioElement { + this.audio ??= playbackElement(); + + return this.audio; + } + + private fetchPiece(piece: ReadAloudPiece): Promise { + let pending = this.prefetched.get(piece); + + if (!pending) { + pending = this.options.synthesize(piece.speech, this.controller.signal); + // a failed fetch is handled where it is played + pending.catch(() => {}); + this.prefetched.set(piece, pending); + } + + return pending; + } + + private finishIfDone(): void { + if (this.playing || this.queue.length > 0 || !this.done || this.queuedTo < this.text.length) { + return; + } + + this.speaking = false; + this.spoken = this.text.length; + + if (!this.idle) { + this.idle = true; + this.options.onIdle?.(); + } + } + + private async play(piece: ReadAloudPiece, blob: Blob, signal: AbortSignal): Promise { + const audio = this.element(); + const url = URL.createObjectURL(blob); + + try { + await new Promise((resolve) => { + const follow = () => { + if (audio.duration > 0 && Number.isFinite(audio.duration)) { + const share = Math.min(audio.currentTime / audio.duration, 1); + + this.spoken = Math.max( + this.spoken, + piece.start + Math.round((piece.end - piece.start) * share) + ); + } + + this.frame = requestAnimationFrame(follow); + }; + const end = () => { + audio.onended = null; + audio.onerror = null; + signal.removeEventListener('abort', end); + resolve(); + }; + + audio.onended = end; + audio.onerror = end; + signal.addEventListener('abort', end); + audio.src = url; + audio.play().then( + () => (this.frame = requestAnimationFrame(follow)), + () => end() + ); + }); + } finally { + if (this.frame !== null) cancelAnimationFrame(this.frame); + + this.frame = null; + URL.revokeObjectURL(url); + } + } + + private async pump(): Promise { + if (this.playing) return; + + const piece = this.queue.shift(); + + if (!piece) { + this.finishIfDone(); + + return; + } + + const { signal } = this.controller; + + this.playing = true; + this.speaking = true; + + if (piece.speech && !this.isMuted) { + // the next piece is fetched while this one plays + if (this.queue[0]) this.fetchPiece(this.queue[0]); + + try { + const blob = await this.fetchPiece(piece); + + if (!signal.aborted && !this.isMuted) await this.play(piece, blob, signal); + } catch (error) { + if (!signal.aborted) console.warn('Reading the reply aloud failed:', error); + } + } + + this.prefetched.delete(piece); + + if (signal.aborted) return; + + this.spoken = Math.max(this.spoken, piece.end); + this.playing = false; + void this.pump(); + } + + /** Muting mid-piece: the voice stops, the text keeps up. */ + private silence(): void { + this.stopAudio(); + this.spoken = Math.max(this.spoken, this.queuedTo); + } + + private stopAudio(): void { + if (this.frame !== null) cancelAnimationFrame(this.frame); + + this.frame = null; + + const audio = this.audio; + + if (audio && !audio.paused) audio.pause(); + + audio?.dispatchEvent(new Event('ended')); + } +} diff --git a/ui/apps/chat/src/lib/voice/speech.ts b/ui/apps/chat/src/lib/voice/speech.ts new file mode 100644 index 000000000000..0cf584965550 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/speech.ts @@ -0,0 +1,49 @@ +import { VOICE_API_PATHS } from './constants'; +import { VoiceTranscriptionError } from './errors'; +import type { VoiceServerContext } from './types'; + +export interface SynthesizeSpeechOptions { + /** Model id, required by a router server */ + model?: string; + /** One of the model's voices (`voices` of GET /props); left out, its default */ + voice?: string; + signal?: AbortSignal; +} + +/** + * Speaks `text` with llama-server's POST /v1/audio/speech and returns the WAV. + * @throws VoiceTranscriptionError with the server's message when it refuses + */ +export async function synthesizeSpeech( + context: VoiceServerContext, + text: string, + options: SynthesizeSpeechOptions = {} +): Promise { + const response = await (context.fetch ?? fetch)(context.resolveUrl(VOICE_API_PATHS.SPEECH), { + body: JSON.stringify({ + input: text, + response_format: 'wav', + ...(options.model ? { model: options.model } : {}), + ...(options.voice ? { voice: options.voice } : {}) + }), + headers: { ...context.getHeaders?.(), 'Content-Type': 'application/json' }, + method: 'POST', + signal: options.signal + }); + + if (!response.ok) { + let message = `Speech failed (${response.status})`; + + try { + const data = await response.json(); + + if (typeof data?.error?.message === 'string') message = data.error.message; + } catch { + // not JSON: keep the status + } + + throw new VoiceTranscriptionError(message, response.status); + } + + return await response.blob(); +} diff --git a/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts b/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts index 395bde627cc5..058c7e02ed98 100644 --- a/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts +++ b/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts @@ -2,11 +2,15 @@ // what the microphone offers per model, the recording row, dictated text written into the // message around what the user types, and a voice message whose transcript comes later. -import { modelServerProps, seedStores } from '../stories/chat/form/fixtures'; +import { modelServerProps, routerModels, seedStores } from '../stories/chat/form/fixtures'; import { deferred, FakeVoiceRecorder } from '../stories/voice/fake-voice-recorder.svelte'; import ChatFormVoiceTestWrapper from './components/ChatFormVoiceTestWrapper.svelte'; import { EditorView } from '@codemirror/view'; -import { SETTINGS_KEYS, VOICE_MODE_MESSAGE } from '#lib/constants/index.ts'; +import { + SETTINGS_KEYS, + VOICE_CONVERSATION_MESSAGE, + VOICE_MODE_MESSAGE +} from '#lib/constants/index.ts'; import { ServerRole, VoiceInputMode } from '#lib/enums/index.ts'; import { voiceTranscriptsStore } from '#lib/stores/index.ts'; import { parseFilesToMessageExtras } from '#lib/utils/browser-only.ts'; @@ -39,10 +43,37 @@ function seed( }); } -async function renderForm({ isLoading = false } = {}) { +/** A router whose chat model (Qwen3) can't hear while another (Gemma) can, none of them loaded. */ +function seedRouter(transcriptionModel = '') { + const [chat, hearing] = routerModels; + + restore = seedStores({ + config: { + autoMicOnEmpty: true, + [SETTINGS_KEYS.VOICE_INPUT_MODE]: VoiceInputMode.AUTO, + [SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL]: transcriptionModel + }, + modelProps: { + [chat.model]: modelServerProps(), + [hearing.model]: modelServerProps({ + modalities: { audio: true, video: false, vision: false } + }) + }, + models: routerModels, + selectedModelId: chat.id, + serverRole: ServerRole.ROUTER + }); +} + +async function renderForm(options: { isLoading?: boolean; voiceMode?: boolean } = {}) { const recorder = new FakeVoiceRecorder(); const state = $state({ files: [] as ChatUploadedFile[], sent: 0, value: '' }); - const screen = render(ChatFormVoiceTestWrapper, { isLoading, state, voiceRecorder: recorder }); + const screen = render(ChatFormVoiceTestWrapper, { + isLoading: options.isLoading, + state, + voiceMode: options.voiceMode, + voiceRecorder: recorder + }); await tick(); @@ -141,6 +172,40 @@ describe('the microphone', () => { }); }); +describe('voice mode', () => { + it('is the main button when the model hears', async () => { + seed(true); + + const { screen } = await renderForm({ voiceMode: true }); + + await expect + .element(screen.getByRole('button', { name: VOICE_CONVERSATION_MESSAGE.open }).last()) + .toBeInTheDocument(); + }); + + it('leaves the lead to the microphone when nothing can transcribe what is said', async () => { + seedRouter(); + + const { screen } = await renderForm({ voiceMode: true }); + + // the microphone stays, saying why it can't record + await expect + .element(screen.getByRole('button', { name: VOICE_MODE_MESSAGE.unavailable }).last()) + .toBeInTheDocument(); + expect(screen.container.innerHTML).not.toContain(VOICE_CONVERSATION_MESSAGE.open); + }); + + it('is the main button again once a transcription model is chosen', async () => { + seedRouter(routerModels[1].model); + + const { screen } = await renderForm({ voiceMode: true }); + + await expect + .element(screen.getByRole('button', { name: VOICE_CONVERSATION_MESSAGE.open }).last()) + .toBeInTheDocument(); + }); +}); + describe('recording', () => { it('asks for the raw microphone, and for the processing that is switched on', async () => { seed(true, VoiceInputMode.DICTATION); diff --git a/ui/apps/chat/tests/client/components/ChatFormVoiceTestWrapper.svelte b/ui/apps/chat/tests/client/components/ChatFormVoiceTestWrapper.svelte index 34dfe4ce77ef..a286a23b6689 100644 --- a/ui/apps/chat/tests/client/components/ChatFormVoiceTestWrapper.svelte +++ b/ui/apps/chat/tests/client/components/ChatFormVoiceTestWrapper.svelte @@ -9,9 +9,11 @@ /** The form's message and files, kept in step with it for the test to read, and how often it sent. */ state: { files: ChatUploadedFile[]; sent?: number; value: string }; voiceRecorder: VoiceRecorderLike; + /** Offer voice mode, as the chat screen does. */ + voiceMode?: boolean; } - let { isLoading = false, state, voiceRecorder }: Props = $props(); + let { isLoading = false, state, voiceMode = false, voiceRecorder }: Props = $props(); @@ -20,6 +22,7 @@ bind:value={state.value} {isLoading} onSubmit={() => (state.sent = (state.sent ?? 0) + 1)} + {voiceMode} {voiceRecorder} /> diff --git a/ui/apps/chat/tests/unit/voice-read-aloud.test.ts b/ui/apps/chat/tests/unit/voice-read-aloud.test.ts new file mode 100644 index 000000000000..403182ab6efc --- /dev/null +++ b/ui/apps/chat/tests/unit/voice-read-aloud.test.ts @@ -0,0 +1,44 @@ +// Guards how voice mode reads a streaming reply aloud: whole sentences as soon as they are +// complete, the rest once the reply is done, without markdown marks, with offsets into the +// reply as written so the text on screen can follow the voice. + +import { readAloudPieces, toSpeech } from '#lib/voice/index.ts'; +import { describe, expect, it } from 'vitest'; + +describe('readAloudPieces', () => { + it('reads complete sentences while the reply streams', () => { + const text = 'Sure, here you go. The first one is'; + const pieces = readAloudPieces(text, 0, false); + + expect(pieces.map((piece) => piece.speech)).toEqual(['Sure, here you go.']); + expect(text.slice(pieces[0].start, pieces[0].end)).toBe('Sure, here you go.'); + }); + + it('reads the rest once the reply is done, from where it left off', () => { + const text = 'Sure, here you go. The first one is short'; + const [first] = readAloudPieces(text, 0, false); + const rest = readAloudPieces(text, first.end, true); + + expect(rest.map((piece) => piece.speech)).toEqual(['The first one is short']); + expect(rest[0].end).toBe(text.length); + }); + + it('does not end a sentence at a list number, and reads lines on their own', () => { + const text = 'Jokes:\n1. Why are fish clever? They live in schools.\n2. Next'; + // a bare line break is a piece with nothing to say: it only moves the reading on + const pieces = readAloudPieces(text, 0, true) + .map((piece) => piece.speech) + .filter(Boolean); + + expect(pieces).toEqual(['Jokes:', 'Why are fish clever?', 'They live in schools.', 'Next']); + }); +}); + +describe('toSpeech', () => { + it('leaves out markdown marks', () => { + expect(toSpeech('## **Quantization.** Some `values`, see [the docs](https://x.y).')).toBe( + 'Quantization. Some values, see the docs.' + ); + expect(toSpeech('```js\nconst a = 1;\n```')).toBe(''); + }); +}); From 15b6ea3af022ee3ee17b36a5fc24a244f0dd4b70 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 16 Nov 2026 10:37:11 +0100 Subject: [PATCH 02/10] ui: move the reasoning effort into the model picker The chat form's row no longer holds the reasoning button: the model picker ends with a Reasoning row, in place of Manage models, that shows the level in use and opens the levels under it. A phone's model drawer has the same row. A desktop server with a single model, whose selector opens the manager instead of a picker, keeps the button beside it. Assisted-by: Claude (Opus 5.5 Medium) (cherry picked from commit f491a9d0e9d999f4e0450a606f760963164a9172) --- .../ChatFormActions/ChatFormActions.svelte | 5 +- .../ModelsSelector/ModelsSelector.svelte | 17 ++++ .../ModelsSelectorDropdown.svelte | 29 ++++--- .../ModelsSelectorReasoning.svelte | 87 +++++++++++++++++++ 4 files changed, 125 insertions(+), 13 deletions(-) create mode 100644 ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorReasoning.svelte diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte index f66224f0b4ad..44368c396ca2 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte @@ -2,7 +2,6 @@ import { Button } from '@ggml-org/llama-ui/ui/button'; import { LoaderCircle, SkipForward, Square } from '@lucide/svelte'; import { - ChatFormActionReasoning, ChatFormActionRecord, ChatFormActionsAdd, ChatFormActionSubmit, @@ -178,12 +177,12 @@ {/if} {#if showModelSelector} - - + {/if} diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte index 3b24b83a0472..c64bc49c21f1 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte @@ -1,8 +1,10 @@ + +{#snippet header()} + {#if reasoning.isOff} + + {:else} + + {/if} + + Reasoning + + {current} +{/snippet} + +{#snippet levelLabel(level: ReasoningEffortLevel)} + {@const tokenLabel = reasoning.tokenLabel(level)} + + {#if reasoning.isSelected(level)} + + {:else} +
+ {/if} + + {level.label} + + {#if tokenLabel} + {tokenLabel} + {/if} +{/snippet} + +{#if variant === 'menu'} + + + +
+ {#each reasoning.levels as level (level.value)} + reasoning.select(level)}> + {@render levelLabel(level)} + + {/each} +
+
+
+{:else} +
+ +
+ {#each reasoning.levels as level (level.value)} + + {/each} +
+
+
+{/if} From 9e4052652b71367cf60e84012233ba4ed5a54c3f Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 16 Nov 2026 11:10:06 +0100 Subject: [PATCH 03/10] ui: read replies aloud faster in voice mode - Settings > Voice > Reading speed: 1x to 2x, 1.25x by default, with the voice's pitch kept. - The reply's first piece may end at a clause (a comma, colon or dash after 24 characters), so the voice starts before the first sentence is complete; later pieces stay whole sentences. - The next two pieces are fetched while one plays. Assisted-by: Claude (Opus 5.5 Medium) (cherry picked from commit e71055f9506ab44593e9868368f3329eaedc839a) --- .../lib/constants/settings-keys.constants.ts | 1 + .../src/lib/constants/settings.constants.ts | 15 +++++++++++++ .../hooks/use-voice-conversation.svelte.ts | 1 + ui/apps/chat/src/lib/voice/read-aloud.ts | 22 +++++++++++++++++-- .../chat/src/lib/voice/reply-reader.svelte.ts | 15 ++++++++++--- ui/apps/chat/tests/unit/voice-mode.test.ts | 1 + .../chat/tests/unit/voice-read-aloud.test.ts | 15 +++++++++++++ 7 files changed, 65 insertions(+), 5 deletions(-) diff --git a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts index 7b067c468095..dcaed3da880a 100644 --- a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts +++ b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts @@ -94,6 +94,7 @@ export const SETTINGS_KEYS = { VOICE_INPUT_MODE: 'voiceInputMode', VOICE_MICROPHONE: 'voiceMicrophone', VOICE_NOISE_SUPPRESSION: 'voiceNoiseSuppression', + VOICE_READING_SPEED: 'voiceReadingSpeed', VOICE_TRANSCRIBE_MESSAGES: 'voiceTranscribeMessages', VOICE_TRANSCRIPTION_LANGUAGE: 'voiceTranscriptionLanguage', VOICE_TRANSCRIPTION_MODEL: 'voiceTranscriptionModel', diff --git a/ui/apps/chat/src/lib/constants/settings.constants.ts b/ui/apps/chat/src/lib/constants/settings.constants.ts index 4544a88df01f..ebf19c82715f 100644 --- a/ui/apps/chat/src/lib/constants/settings.constants.ts +++ b/ui/apps/chat/src/lib/constants/settings.constants.ts @@ -12,6 +12,7 @@ import { Database, File, Funnel, + Gauge, GlassWater, ListRestart, Mic, @@ -481,6 +482,20 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [ label: 'Automatic gain', type: SettingsFieldType.CHECKBOX }, + { + defaultValue: '1.25', + help: 'How fast voice mode reads replies aloud. The voice keeps its pitch.', + key: SETTINGS_KEYS.VOICE_READING_SPEED, + label: 'Reading speed', + options: [ + { icon: Gauge, label: '1×', value: '1' }, + { icon: Gauge, label: '1.25×', value: '1.25' }, + { icon: Gauge, label: '1.5×', value: '1.5' }, + { icon: Gauge, label: '1.75×', value: '1.75' }, + { icon: Gauge, label: '2×', value: '2' } + ], + type: SettingsFieldType.SELECT + }, { defaultValue: '', help: 'The language you speak, as a two-letter code such as en or de, passed to the transcription. Leave it empty to let the model tell.', diff --git a/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts index 3ab477a20a34..bd8369c52f98 100644 --- a/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts +++ b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts @@ -139,6 +139,7 @@ export function useVoiceConversation() { onIdle: () => { if (state === 'speaking' || state === 'thinking') void listen(); }, + playbackRate: () => Number(config[SETTINGS_KEYS.VOICE_READING_SPEED]) || 1.25, synthesize: (text, signal) => { const { context } = serverContext(); diff --git a/ui/apps/chat/src/lib/voice/read-aloud.ts b/ui/apps/chat/src/lib/voice/read-aloud.ts index d7ec29d2fd3b..cf375edcfd65 100644 --- a/ui/apps/chat/src/lib/voice/read-aloud.ts +++ b/ui/apps/chat/src/lib/voice/read-aloud.ts @@ -16,12 +16,22 @@ export interface ReadAloudPiece { // a sentence ends at . ! ? (with closing quotes or brackets) before whitespace, or a line ends; // a list number such as "1." does not end one const PIECE_END = /(? Promise; /** Called once the reply is done and all of it has been read (or skipped while muted) */ onIdle?: () => void; + /** Speed of the voice, e.g. 1.25; the pitch is kept. Read before each piece. */ + playbackRate?: () => number; } /** @@ -99,7 +101,8 @@ export class ReplyReader { this.text = text; this.done = done; - for (const piece of readAloudPieces(text, this.queuedTo, done)) { + // the first piece may end at a clause, so the voice starts sooner + for (const piece of readAloudPieces(text, this.queuedTo, done, this.queuedTo === 0)) { this.queue.push(piece); this.queuedTo = piece.end; } @@ -169,6 +172,10 @@ export class ReplyReader { audio.onerror = end; signal.addEventListener('abort', end); audio.src = url; + // set after src: loading a source resets the rate + audio.preservesPitch = true; + audio.playbackRate = this.options.playbackRate?.() ?? 1; + audio.defaultPlaybackRate = audio.playbackRate; audio.play().then( () => (this.frame = requestAnimationFrame(follow)), () => end() @@ -199,8 +206,10 @@ export class ReplyReader { this.speaking = true; if (piece.speech && !this.isMuted) { - // the next piece is fetched while this one plays - if (this.queue[0]) this.fetchPiece(this.queue[0]); + // the next pieces are fetched while this one plays + for (const next of this.queue.slice(0, 2)) { + if (next.speech) this.fetchPiece(next); + } try { const blob = await this.fetchPiece(piece); diff --git a/ui/apps/chat/tests/unit/voice-mode.test.ts b/ui/apps/chat/tests/unit/voice-mode.test.ts index 7d2f09225f66..a5c39085f901 100644 --- a/ui/apps/chat/tests/unit/voice-mode.test.ts +++ b/ui/apps/chat/tests/unit/voice-mode.test.ts @@ -158,6 +158,7 @@ describe('voice settings', () => { SETTINGS_KEYS.VOICE_NOISE_SUPPRESSION, SETTINGS_KEYS.VOICE_ECHO_CANCELLATION, SETTINGS_KEYS.VOICE_AUTO_GAIN, + SETTINGS_KEYS.VOICE_READING_SPEED, SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE ]); expect(section?.fields?.[0].type).toBe(SettingsFieldType.SELECT); diff --git a/ui/apps/chat/tests/unit/voice-read-aloud.test.ts b/ui/apps/chat/tests/unit/voice-read-aloud.test.ts index 403182ab6efc..1df51a68a375 100644 --- a/ui/apps/chat/tests/unit/voice-read-aloud.test.ts +++ b/ui/apps/chat/tests/unit/voice-read-aloud.test.ts @@ -34,6 +34,21 @@ describe('readAloudPieces', () => { }); }); +describe('the first piece', () => { + it('may end at a clause, so the voice starts before the sentence is done', () => { + const text = 'Quantum physics describes how very small things, like electrons and photons'; + const [first] = readAloudPieces(text, 0, false, true); + + expect(first.speech).toBe('Quantum physics describes how very small things,'); + // later pieces wait for whole sentences + expect(readAloudPieces(text, first.end, false)).toEqual([]); + }); + + it('waits for the sentence when its first clause is short', () => { + expect(readAloudPieces('Sure, here are ten jokes about', 0, false, true)).toEqual([]); + }); +}); + describe('toSpeech', () => { it('leaves out markdown marks', () => { expect(toSpeech('## **Quantization.** Some `values`, see [the docs](https://x.y).')).toBe( From fbda2527c28c02d0f5b68faacae377d9ea3382db Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 16 Nov 2026 11:25:42 +0100 Subject: [PATCH 04/10] ui: go on reading after a false interruption, and close the gaps between pieces - A sound over a reply read aloud no longer stops it: the reading pauses and voice mode listens. Speech within a second, judged against the room's noise measured before, confirms the interruption; otherwise the reply goes on from where it paused. The reply's own echo or a noise used to stop it and leave the microphone listening. Tapping the microphone still interrupts outright. - Read-aloud clips lose the silence the speech model puts around them (LFM3-Omni: about 140 ms before, 240 ms after), down to 40 ms and 90 ms, so pieces, and paragraphs after an empty line, follow closely. Assisted-by: Claude (Opus 5.5 Medium) (cherry picked from commit 653f357c1ccb2e81ede9552a1a9576aa57acecfe) --- .../hooks/use-voice-conversation.svelte.ts | 77 +++++++++++++++-- ui/apps/chat/src/lib/voice/pauses.ts | 3 + .../chat/src/lib/voice/reply-reader.svelte.ts | 44 +++++++++- ui/apps/chat/src/lib/voice/trim-silence.ts | 83 +++++++++++++++++++ ui/apps/chat/tests/unit/voice-pauses.test.ts | 13 +++ .../tests/unit/voice-trim-silence.test.ts | 46 ++++++++++ 6 files changed, 258 insertions(+), 8 deletions(-) create mode 100644 ui/apps/chat/src/lib/voice/trim-silence.ts create mode 100644 ui/apps/chat/tests/unit/voice-trim-silence.test.ts diff --git a/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts index bd8369c52f98..19c144eb0e4b 100644 --- a/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts +++ b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts @@ -63,8 +63,10 @@ const CONVERSATION_CONSTRAINTS: MediaTrackConstraints = { }; /** Silence after speech that ends a turn. */ const TURN_PAUSE_MS = 900; -/** Speech over the reply that interrupts it. */ +/** Sound over the reply that pauses it, to hear whether it is speech. */ const BARGE_IN_MS = 300; +/** How long a paused reply waits for speech before it goes on: no speech, it was a noise or its own echo. */ +const BARGE_IN_CONFIRM_MS = 1000; function serverContext(): { context: VoiceServerContext; realtime: boolean } { const headers = getAuthHeaders(); @@ -134,6 +136,9 @@ export function useVoiceConversation() { let turnId = 0; // the current reply was seen loading: until then, not loading does not mean done let replyStarted = false; + // the reply is paused over a sound, until speech confirms the interruption or a second passes + let tentative = false; + let tentativeTimer: ReturnType | undefined; const reader = new ReplyReader({ onIdle: () => { @@ -216,8 +221,12 @@ export function useVoiceConversation() { monitor = null; } - /** Starts a turn: listens until the speaker pauses, then sends what was said. */ - async function listen() { + /** + * Starts a turn: listens until the speaker pauses, then sends what was said. `floor` is + * the room's noise level when known; `tentative` listens over a paused reply, which + * goes on unless speech is heard within a second. + */ + async function listen(options: { floor?: number; tentative?: boolean } = {}) { stopMonitor(); stopRecorder(); liveText = ''; @@ -241,6 +250,7 @@ export function useVoiceConversation() { const { context, realtime } = serverContext(); const language = String(config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE] ?? '').trim(); const pauses = new PauseDetector({ + floor: options.floor, intervalMs: VOICE_LEVELS_INTERVAL_MS, maxSegmentMs: Number.POSITIVE_INFINITY, pauseMs: TURN_PAUSE_MS @@ -258,9 +268,14 @@ export function useVoiceConversation() { onLevels: (next) => { levels = next as number[]; - if (run === generation && state === 'listening' && pauses.push(next.at(-1) ?? 0)) { - void endTurn(run); - } + if (run !== generation) return; + + const cut = pauses.push(next.at(-1) ?? 0); + + // speech over the paused reply: the interruption was meant + if (tentative && pauses.heardSpeech) confirmInterrupt(); + + if (state === 'listening' && cut) void endTurn(run); }, onText: (text) => { if (run === generation) liveText = text; @@ -289,6 +304,13 @@ export function useVoiceConversation() { error = null; state = 'listening'; + + if (options.tentative && tentative) { + clearTimeout(tentativeTimer); + tentativeTimer = setTimeout(() => { + if (run === generation && tentative && !pauses.heardSpeech) resumeReading(); + }, BARGE_IN_CONFIRM_MS); + } } /** The speaker paused: what they said goes to the chat. */ @@ -364,7 +386,9 @@ export function useVoiceConversation() { floor = Math.min(level, floor + 0.002); speechMs = level > floor + 0.2 ? speechMs + VOICE_LEVELS_INTERVAL_MS : 0; - if (speechMs >= BARGE_IN_MS && run === generation && reader.speaking) interrupt(); + if (speechMs >= BARGE_IN_MS && run === generation && reader.speaking && !reader.paused) { + bargeIn(floor); + } }); try { @@ -380,8 +404,45 @@ export function useVoiceConversation() { } } + /** A sound over the reply: it pauses, and the next second tells whether it was speech. */ + function bargeIn(floor: number) { + reader.pause(); + tentative = true; + void listen({ floor, tentative: true }); + } + + /** No speech came: the reply goes on from where it paused. */ + function resumeReading() { + tentative = false; + clearTimeout(tentativeTimer); + generation++; + stopRecorder(); + liveText = ''; + levels = []; + state = 'speaking'; + reader.resume(); + void watchForBargeIn(generation); + } + + /** Speech came: the reply stops for good, and the turn that started goes on. */ + function confirmInterrupt() { + tentative = false; + clearTimeout(tentativeTimer); + reader.stop(); + + const convId = conversationsStore.activeConversation?.id; + + if (convId && chatStore.isChatLoading(convId)) void chatStore.stopGeneration(); + + const turn = currentTurn; + + if (turn) turn.done = true; + } + /** Stops the reply where it is, and listens. */ function interrupt() { + tentative = false; + clearTimeout(tentativeTimer); reader.stop(); const convId = conversationsStore.activeConversation?.id; @@ -403,6 +464,8 @@ export function useVoiceConversation() { } function close() { + tentative = false; + clearTimeout(tentativeTimer); generation++; stopRecorder(); stopMonitor(); diff --git a/ui/apps/chat/src/lib/voice/pauses.ts b/ui/apps/chat/src/lib/voice/pauses.ts index 44083a7ee758..69815fe47c1b 100644 --- a/ui/apps/chat/src/lib/voice/pauses.ts +++ b/ui/apps/chat/src/lib/voice/pauses.ts @@ -17,6 +17,8 @@ export interface PauseDetectorOptions { * is ever quiet */ maxSegmentMs?: number; + /** The room's noise level when it is known already, so speech from the first level counts */ + floor?: number; } /** How far above the noise floor a level is speech, and below which it is quiet. */ @@ -47,6 +49,7 @@ export class PauseDetector { this.pauseMs = options.pauseMs ?? 500; this.minSpeechMs = options.minSpeechMs ?? 300; this.maxSegmentMs = options.maxSegmentMs ?? 10_000; + this.floor = options.floor ?? 1; } /** Takes the newest level; true when this is a place to cut the recording. */ diff --git a/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts b/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts index 33479959038c..f630ce6b604e 100644 --- a/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts +++ b/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts @@ -1,4 +1,5 @@ import { type ReadAloudPiece, readAloudPieces } from './read-aloud'; +import { trimSilence } from './trim-silence'; // one element plays every reply: unlocked once inside a click, it may play later (iOS asks for that) let sharedAudio: HTMLAudioElement | null = null; @@ -51,6 +52,7 @@ export class ReplyReader { private frame: number | null = null; private idle = true; private isMuted = false; + private isPaused = false; private readonly options: ReplyReaderOptions; private playing = false; // bookkeeping the UI never reads @@ -58,12 +60,18 @@ export class ReplyReader { private prefetched = new Map>(); private queue: ReadAloudPiece[] = []; private queuedTo = 0; + private resumeWaiters: (() => void)[] = []; private text = ''; get muted(): boolean { return this.isMuted; } + /** Whether reading is paused, see pause(). */ + get paused(): boolean { + return this.isPaused; + } + set muted(value: boolean) { this.isMuted = value; @@ -74,6 +82,15 @@ export class ReplyReader { this.options = options; } + /** Holds the voice where it is, e.g. to hear whether the user speaks; resume() goes on from there. */ + pause(): void { + if (this.isPaused) return; + + this.isPaused = true; + + if (this.audio && !this.audio.paused) this.audio.pause(); + } + /** Starts a new reply. */ reset(): void { this.stop(); @@ -85,8 +102,25 @@ export class ReplyReader { this.idle = false; } + /** Goes on reading from where pause() held it. */ + resume(): void { + if (!this.isPaused) return; + + this.isPaused = false; + + const audio = this.audio; + + if (this.playing && audio?.src && !audio.ended && audio.paused) audio.play().catch(() => {}); + + for (const wake of this.resumeWaiters.splice(0)) wake(); + } + /** Stops reading: the current piece, and what was still to come. */ stop(): void { + this.isPaused = false; + + for (const wake of this.resumeWaiters.splice(0)) wake(); + this.controller.abort(); this.queue = []; this.prefetched.clear(); @@ -120,7 +154,10 @@ export class ReplyReader { let pending = this.prefetched.get(piece); if (!pending) { - pending = this.options.synthesize(piece.speech, this.controller.signal); + // the model's silence around each piece is cut short, so the voice runs on + pending = this.options + .synthesize(piece.speech, this.controller.signal) + .then((blob) => trimSilence(blob)); // a failed fetch is handled where it is played pending.catch(() => {}); this.prefetched.set(piece, pending); @@ -214,6 +251,11 @@ export class ReplyReader { try { const blob = await this.fetchPiece(piece); + // paused while fetching: the piece waits to be played + while (this.isPaused && !signal.aborted) { + await new Promise((resolve) => this.resumeWaiters.push(resolve)); + } + if (!signal.aborted && !this.isMuted) await this.play(piece, blob, signal); } catch (error) { if (!signal.aborted) console.warn('Reading the reply aloud failed:', error); diff --git a/ui/apps/chat/src/lib/voice/trim-silence.ts b/ui/apps/chat/src/lib/voice/trim-silence.ts new file mode 100644 index 000000000000..83bc7ed93647 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/trim-silence.ts @@ -0,0 +1,83 @@ +import { encodeWav } from './wav'; + +/** Silence kept before the voice starts and after it ends, so pieces still breathe. */ +const LEAD_MS = 40; +const TAIL_MS = 90; +/** A sample under this share of the clip's loudest one is silence. */ +const SILENCE_SHARE = 0.03; + +/** + * A WAV with the silence a speech model puts around what it says cut down to a short + * breath, so pieces read one after another (a sentence, then the paragraph after an + * empty line) follow without long gaps. Anything but 16-bit PCM is returned as it is. + */ +export async function trimSilence(blob: Blob): Promise { + const bytes = await blob.arrayBuffer(); + const pcm = readPcm16(bytes); + + if (!pcm || pcm.channels !== 1) return blob; + + const { sampleRate, samples } = pcm; + + let loudest = 0; + + for (let i = 0; i < samples.length; i++) loudest = Math.max(loudest, Math.abs(samples[i])); + + if (loudest === 0) return blob; + + const threshold = loudest * SILENCE_SHARE; + + let first = 0; + let last = samples.length - 1; + + while (first < samples.length && Math.abs(samples[first]) < threshold) first++; + while (last > first && Math.abs(samples[last]) < threshold) last--; + + const start = Math.max(0, first - Math.round((sampleRate * LEAD_MS) / 1000)); + const end = Math.min(samples.length, last + 1 + Math.round((sampleRate * TAIL_MS) / 1000)); + + if (start === 0 && end === samples.length) return blob; + + return encodeWav(samples.slice(start, end), sampleRate); +} + +/** The samples of a 16-bit PCM WAV, or null for any other format. */ +function readPcm16( + bytes: ArrayBuffer +): { channels: number; sampleRate: number; samples: Int16Array } | null { + const view = new DataView(bytes); + const tag = (offset: number) => + String.fromCharCode(...new Uint8Array(bytes, offset, Math.min(4, bytes.byteLength - offset))); + + if (bytes.byteLength < 12 || tag(0) !== 'RIFF' || tag(8) !== 'WAVE') return null; + + let channels = 0; + let sampleRate = 0; + let bits = 0; + let offset = 12; + + while (offset + 8 <= bytes.byteLength) { + const id = tag(offset); + // a streamed WAV may leave the data size unknown (0xFFFFFFFF): it runs to the end + const size = Math.min(view.getUint32(offset + 4, true), bytes.byteLength - offset - 8); + const body = offset + 8; + + if (id === 'fmt ') { + if (view.getUint16(body, true) !== 1) return null; + + channels = view.getUint16(body + 2, true); + sampleRate = view.getUint32(body + 4, true); + bits = view.getUint16(body + 14, true); + } else if (id === 'data') { + if (bits !== 16 || !sampleRate) return null; + + const samples = new Int16Array(bytes.slice(body, body + size - (size % 2))); + + return { channels, sampleRate, samples }; + } + + offset = body + size + (size % 2); + } + + return null; +} diff --git a/ui/apps/chat/tests/unit/voice-pauses.test.ts b/ui/apps/chat/tests/unit/voice-pauses.test.ts index 9b2bfeda600b..188288958c6e 100644 --- a/ui/apps/chat/tests/unit/voice-pauses.test.ts +++ b/ui/apps/chat/tests/unit/voice-pauses.test.ts @@ -42,6 +42,19 @@ describe('PauseDetector', () => { expect(hold(detector, 0.45, 600)).toBe(true); }); + it('hears speech from the first level when the room is known', () => { + // without a known floor, the first level is taken as the room's noise + const unknown = new PauseDetector({ intervalMs: INTERVAL }); + + unknown.push(0.6); + expect(unknown.heardSpeech).toBe(false); + + const known = new PauseDetector({ floor: 0.1, intervalMs: INTERVAL }); + + known.push(0.6); + expect(known.heardSpeech).toBe(true); + }); + it('cuts a long stretch of speech at its next quiet moment', () => { const detector = new PauseDetector({ intervalMs: INTERVAL, maxSegmentMs: 2000 }); diff --git a/ui/apps/chat/tests/unit/voice-trim-silence.test.ts b/ui/apps/chat/tests/unit/voice-trim-silence.test.ts new file mode 100644 index 000000000000..5cc4cb98cc13 --- /dev/null +++ b/ui/apps/chat/tests/unit/voice-trim-silence.test.ts @@ -0,0 +1,46 @@ +// Guards the silence cut from read-aloud clips: the speech model's quiet lead and tail +// shrink to a short breath, so pieces follow each other without long gaps. + +import { encodeWav } from '#lib/voice/index.ts'; +import { trimSilence } from '#lib/voice/trim-silence.ts'; +import { describe, expect, it } from 'vitest'; + +const RATE = 24_000; + +/** `silenceMs` of silence, `toneMs` of a tone, then `silenceMs` of silence again. */ +function clip(silenceMs: number, toneMs: number): Blob { + const silence = Math.round((RATE * silenceMs) / 1000); + const tone = Math.round((RATE * toneMs) / 1000); + const samples = new Int16Array(silence * 2 + tone); + + for (let i = 0; i < tone; i++) { + samples[silence + i] = Math.round(Math.sin((2 * Math.PI * 440 * i) / RATE) * 12_000); + } + + return encodeWav(samples, RATE); +} + +/** Length of a 16-bit mono WAV in ms. */ +async function lengthMs(blob: Blob): Promise { + return ((blob.size - 44) / 2 / RATE) * 1000; +} + +describe('trimSilence', () => { + it('cuts the silence around the voice down to a short breath', async () => { + const trimmed = await trimSilence(clip(300, 1000)); + + // 1000 ms of tone, with 40 ms kept before and 90 ms after it + expect(await lengthMs(trimmed)).toBeGreaterThan(1100); + expect(await lengthMs(trimmed)).toBeLessThan(1160); + }); + + it('leaves a clip without silence, a silent one and anything not WAV alone', async () => { + const tight = clip(0, 500); + const silent = encodeWav(new Int16Array(RATE), RATE); + const other = new Blob(['not a wav'], { type: 'audio/mpeg' }); + + expect(await trimSilence(tight)).toBe(tight); + expect(await trimSilence(silent)).toBe(silent); + expect(await trimSilence(other)).toBe(other); + }); +}); From 3c450042168925f599ed1a9afcb1f2cff67e45eb Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 16 Nov 2026 11:27:46 +0100 Subject: [PATCH 05/10] ui: follow the voice, not the text, when voice mode scrolls The voice mode screen no longer scrolls to the bottom as a reply streams. A new exchange scrolls into view as it starts; then, while the reply is read aloud, the page keeps the word being read in view, moving only when it nears the bottom. With the speaker off, or no model that speaks, it stays where it is. Assisted-by: Claude (Opus 5.5 Medium) (cherry picked from commit b64182c323ed540c41cb41691198ad03161c6b3b) --- .../app/chat/VoiceMode/VoiceModeScreen.svelte | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte index b90db50b10de..13a1be751122 100644 --- a/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte @@ -45,12 +45,44 @@ onMount(() => conversation.open()); - // the newest exchange stays in view as it grows + /** Room kept under the word being read, so the next line shows before it is reached. */ + const READ_MARGIN_PX = 96; + + // a new exchange comes into view as it starts $effect(() => { - void conversation.turns.at(-1)?.reply; void conversation.turns.length; - tick().then(() => scroller?.scrollTo({ behavior: 'smooth', top: scroller.scrollHeight })); + tick().then(() => { + const turn = scroller?.querySelector( + '[data-slot="voice-mode-turn"]:last-of-type' + ); + + turn?.scrollIntoView({ behavior: 'smooth', block: 'start' }); + }); + }); + + // then the page follows the voice, not the text: a reply growing below does not scroll it, + // the word being read stays in view + $effect(() => { + void conversation.reader.spoken; + + // nothing read aloud (speaker off, or no model that speaks): nothing to follow + if (!scroller || !conversation.canSpeak || conversation.speakerMuted) return; + + const reply = scroller.querySelector( + '[data-slot="voice-mode-turn"]:last-of-type [data-slot="voice-mode-reply"]' + ); + const words = reply?.querySelectorAll('[data-spoken]'); + const word = words?.[words.length - 1]; + + if (!word) return; + + const view = scroller.getBoundingClientRect(); + const box = word.getBoundingClientRect(); + + if (box.bottom > view.bottom - READ_MARGIN_PX || box.top < view.top) { + scroller.scrollBy({ behavior: 'smooth', top: box.top - view.top - view.height / 3 }); + } }); function close() { @@ -95,7 +127,7 @@ {:else} {#each conversation.turns as turn (turn.id)} -
+

From ca44edfe145969dc2389aa9ea556034792469dce Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 16 Nov 2026 11:52:11 +0100 Subject: [PATCH 06/10] ui: ask for tool approval in voice mode Voice mode shows no chat messages, so a reply that wanted to use a tool, or an agentic loop asking to go on, waited on a card nobody could see. Both cards now show above the microphone: they rise a little from below as they fade in, and sink back as they fade out once answered (no motion with reduced motion). Escape in the card's allow menu closes the menu, not voice mode. Assisted-by: Claude (Opus 5.5 Medium) (cherry picked from commit 91d1dc079c2b952d2324722f1f35d413c8ecc841) --- .../app/chat/VoiceMode/VoiceModeScreen.svelte | 87 +++++++++++++++++-- 1 file changed, 82 insertions(+), 5 deletions(-) diff --git a/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte index 13a1be751122..e4abbad5c14c 100644 --- a/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte @@ -2,25 +2,75 @@ Voice mode: a full-screen spoken conversation with the chat model. A greeting until the first words, what is said shown as it is transcribed, then each exchange: the message, and the reply lit word by word as it is read aloud. The big microphone mutes and unmutes listening (and, while a reply plays, interrupts it); below it the model, the - speaker and the way out. + speaker and the way out. A tool the reply wants to use, or an agentic loop that asks to go on, waits on a card that + rises above the microphone and sinks away once answered. --> @@ -157,6 +208,32 @@

+ {#if askPermission} + {#key askPermission} +
+ +
+ {/key} + {:else if askContinue} +
+ +
+ {/if} +

Date: Mon, 16 Nov 2026 12:00:53 +0100 Subject: [PATCH 07/10] ui: let an interrupted reply go on when the microphone is muted An interruption, by voice or by tapping the microphone, now holds the reply where it paused instead of stopping it. Muting the microphone, or a turn in which nothing is understood, lets it go on from there; a turn with words drops it, stopping its reading and generation, and is sent. Assisted-by: Claude (Opus 5.5 Medium) (cherry picked from commit 704dd5f53dc1decd491140640d130de1c5336e10) --- .../hooks/use-voice-conversation.svelte.ts | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts index 19c144eb0e4b..2a9333edef34 100644 --- a/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts +++ b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts @@ -139,6 +139,9 @@ export function useVoiceConversation() { // the reply is paused over a sound, until speech confirms the interruption or a second passes let tentative = false; let tentativeTimer: ReturnType | undefined; + // a reply paused by an interruption: it goes on if the microphone is muted or nothing is + // said, and is dropped once the user's turn is sent + let heldReply = false; const reader = new ReplyReader({ onIdle: () => { @@ -336,13 +339,17 @@ export function useVoiceConversation() { text = text.trim(); - // nothing understood: listen again + // nothing understood: a reply held for this turn goes on, else listen again if (!text) { - void listen(); + if (heldReply) resumeReading(); + else void listen(); return; } + // the user had something to say: the reply they interrupted gives way + dropHeldReply(); + const turn: VoiceTurn = { done: false, id: ++turnId, reply: '', sent: false, user: text }; turns.push(turn); @@ -407,12 +414,14 @@ export function useVoiceConversation() { /** A sound over the reply: it pauses, and the next second tells whether it was speech. */ function bargeIn(floor: number) { reader.pause(); + heldReply = true; tentative = true; void listen({ floor, tentative: true }); } - /** No speech came: the reply goes on from where it paused. */ + /** No speech came, or the microphone was muted: the reply goes on from where it paused. */ function resumeReading() { + heldReply = false; tentative = false; clearTimeout(tentativeTimer); generation++; @@ -424,10 +433,17 @@ export function useVoiceConversation() { void watchForBargeIn(generation); } - /** Speech came: the reply stops for good, and the turn that started goes on. */ + /** Speech came: the turn that started goes on, over the reply held where it paused. */ function confirmInterrupt() { tentative = false; clearTimeout(tentativeTimer); + } + + /** The interrupted reply is given up: its reading and its generation stop. */ + function dropHeldReply() { + if (!heldReply) return; + + heldReply = false; reader.stop(); const convId = conversationsStore.activeConversation?.id; @@ -439,20 +455,12 @@ export function useVoiceConversation() { if (turn) turn.done = true; } - /** Stops the reply where it is, and listens. */ + /** Holds the reply where it is, and listens; muting the microphone lets it go on. */ function interrupt() { tentative = false; clearTimeout(tentativeTimer); - reader.stop(); - - const convId = conversationsStore.activeConversation?.id; - - if (convId && chatStore.isChatLoading(convId)) void chatStore.stopGeneration(); - - const turn = currentTurn; - - if (turn) turn.done = true; - + reader.pause(); + heldReply = true; void listen(); } @@ -464,6 +472,7 @@ export function useVoiceConversation() { } function close() { + heldReply = false; tentative = false; clearTimeout(tentativeTimer); generation++; @@ -478,6 +487,13 @@ export function useVoiceConversation() { micMuted = !micMuted; if (micMuted) { + // an interrupted reply goes on from where it paused + if (heldReply) { + resumeReading(); + + return; + } + if (state === 'listening' || state === 'starting') { generation++; stopRecorder(); From d0473c38cb7a3f77638dca74c946040a5e58ee5d Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 16 Nov 2026 12:10:40 +0100 Subject: [PATCH 08/10] ui: give voice mode turns more room and fade the unread reply Space turns and the user's words from the reply further apart, most on phones. A fade from the background over the bottom fifth of the screen hides what is generated ahead of the reading, and the scroll keeps the spoken word above it. Assisted-by: Claude (Opus 5.5 Medium) (cherry picked from commit e673c841866cd25a1566a85a8b3b3bab0c86d149) --- .../app/chat/VoiceMode/VoiceModeScreen.svelte | 94 +++++++++++-------- 1 file changed, 55 insertions(+), 39 deletions(-) diff --git a/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte index e4abbad5c14c..f643c314a9f7 100644 --- a/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte @@ -95,8 +95,10 @@ onMount(() => conversation.open()); - /** Room kept under the word being read, so the next line shows before it is reached. */ - const READ_MARGIN_PX = 96; + /** Room kept between the word being read and the fade at the bottom. */ + const READ_MARGIN_PX = 24; + + let fade = $state(); // a new exchange comes into view as it starts $effect(() => { @@ -129,8 +131,10 @@ const view = scroller.getBoundingClientRect(); const box = word.getBoundingClientRect(); + // the word stays above the fade, where text is still clear + const clear = view.bottom - (fade?.offsetHeight ?? 0) - READ_MARGIN_PX; - if (box.bottom > view.bottom - READ_MARGIN_PX || box.top < view.top) { + if (box.bottom > clear || box.top < view.top) { scroller.scrollBy({ behavior: 'smooth', top: box.top - view.top - view.height / 3 }); } }); @@ -166,45 +170,57 @@ data-slot="voice-mode" role="dialog" > -

-
- {#if conversation.turns.length === 0} -
- - -

- {VOICE_CONVERSATION_MESSAGE.greeting} -

-
- {:else} - {#each conversation.turns as turn (turn.id)} -
-

- {turn.user} -

+
+
+ +
+ {#if conversation.turns.length === 0} +
+ - {#if turn.reply} -

- -

- {:else if turn.id === lastTurnId && !turn.done} - - {/if} +

+ {VOICE_CONVERSATION_MESSAGE.greeting} +

- {/each} - {/if} + {:else} + {#each conversation.turns as turn (turn.id)} +
+

+ {turn.user} +

+ + {#if turn.reply} +

+ +

+ {:else if turn.id === lastTurnId && !turn.done} + + {/if} +
+ {/each} + {/if} +
+ + +
From 788f17a8afa2234c51967834a73a908ea8c1f8a1 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 16 Nov 2026 13:03:10 +0100 Subject: [PATCH 09/10] ui: read on from a word tapped in voice mode While a reply is read aloud, a click or tap on one of its words moves the voice there, back or ahead, and reading goes on from it. A sentence already fetched plays at once, from about where the word falls in it: the reply's speech is kept until the next one. A reply held for an interruption goes on from the word too. Assisted-by: Claude (Opus 5.5 Medium) (cherry picked from commit 95d93e62ac297c577017d23ab39ae8822a10dd7e) --- .../app/chat/VoiceMode/VoiceModeScreen.svelte | 3 + .../hooks/use-voice-conversation.svelte.ts | 19 +- .../chat/src/lib/voice/reply-reader.svelte.ts | 116 ++++++++++--- .../client/voice-reply-reader.svelte.test.ts | 163 ++++++++++++++++++ .../client/voice-spoken-text.svelte.test.ts | 26 +++ .../components/voice/VoiceSpokenText.svelte | 29 +++- 6 files changed, 329 insertions(+), 27 deletions(-) create mode 100644 ui/apps/chat/tests/client/voice-reply-reader.svelte.test.ts create mode 100644 ui/apps/chat/tests/client/voice-spoken-text.svelte.test.ts diff --git a/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte index f643c314a9f7..e0cfe55614cf 100644 --- a/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte @@ -196,6 +196,9 @@ {#if turn.reply}

| undefined; // a reply paused by an interruption: it goes on if the microphone is muted or nothing is // said, and is dropped once the user's turn is sent - let heldReply = false; + let heldReply = $state(false); const reader = new ReplyReader({ onIdle: () => { @@ -464,6 +464,18 @@ export function useVoiceConversation() { void listen(); } + /** + * Reads the reply on from `offset`, e.g. a word tapped while it is read; a reply held for + * an interruption goes on from there too, as the tap says the user wants to hear it. + */ + function readFrom(offset: number) { + if (!reader.speaking && !heldReply) return; + + if (heldReply) resumeReading(); + + reader.seek(offset); + } + /** Starts the conversation; the click that opened voice mode unlocked playback (unlockVoicePlayback). */ function open() { error = null; @@ -540,6 +552,11 @@ export function useVoiceConversation() { get reader() { return reader; }, + readFrom, + /** Whether the reply is being read, or held for an interruption: readFrom() can move it. */ + get reading() { + return reader.speaking || heldReply; + }, get speakerMuted() { return speakerMuted; }, diff --git a/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts b/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts index f630ce6b604e..2e084b2b044b 100644 --- a/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts +++ b/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts @@ -38,7 +38,8 @@ export interface ReplyReaderOptions { * Reads a reply aloud while it streams: give it the reply as it grows with update(), * and it speaks each complete sentence in order, fetching the next one while the * current one plays. `spoken` follows the voice through the reply as written, for - * VoiceSpokenText. Muted, it reads nothing and `spoken` keeps up with the text. + * VoiceSpokenText, and seek() moves the voice to another place in it. Muted, it reads + * nothing and `spoken` keeps up with the text. */ export class ReplyReader { /** Whether a piece is being fetched or played */ @@ -47,20 +48,25 @@ export class ReplyReader { spoken = $state(0); private audio: HTMLAudioElement | null = null; - private controller = new AbortController(); private done = false; + // the reply's pieces and their audio, kept so seek() can go back; bookkeeping the UI never reads + // eslint-disable-next-line svelte/prefer-svelte-reactivity + private fetched = new Map>(); + // the reply's speech requests, and the reading under way, which seek() starts anew + private fetches = new AbortController(); private frame: number | null = null; private idle = true; private isMuted = false; private isPaused = false; private readonly options: ReplyReaderOptions; + private pieces: ReadAloudPiece[] = []; private playing = false; - // bookkeeping the UI never reads - // eslint-disable-next-line svelte/prefer-svelte-reactivity - private prefetched = new Map>(); private queue: ReadAloudPiece[] = []; private queuedTo = 0; private resumeWaiters: (() => void)[] = []; + private run = new AbortController(); + // where in its piece the next one to play starts, after seek() + private startAt: { piece: ReadAloudPiece; share: number } | null = null; private text = ''; get muted(): boolean { @@ -94,7 +100,9 @@ export class ReplyReader { /** Starts a new reply. */ reset(): void { this.stop(); - this.controller = new AbortController(); + this.fetches = new AbortController(); + this.run = new AbortController(); + this.pieces = []; this.text = ''; this.done = false; this.queuedTo = 0; @@ -115,15 +123,52 @@ export class ReplyReader { for (const wake of this.resumeWaiters.splice(0)) wake(); } + /** + * Reads on from `offset` in the reply as written, e.g. a word tapped on screen, before + * or after where the voice is; a piece already fetched plays at once, from about where + * the word falls in it. False when there is nothing to read there: the reply was read + * to its end, the voice is muted, or that sentence is not complete yet. + */ + seek(offset: number): boolean { + if (this.idle || this.isMuted) return false; + + const index = this.pieces.findIndex((piece) => offset < piece.end); + + if (index === -1) return false; + + const piece = this.pieces[index]; + const from = Math.max(offset, piece.start); + + this.run.abort(); + this.run = new AbortController(); + + // a reading that waited on a pause or a fetch sees it was aborted and lets go + for (const wake of this.resumeWaiters.splice(0)) wake(); + + this.stopAudio(); + this.queue = this.pieces.slice(index); + // a piece opens with the space after the one before: a word right after it starts the piece + this.startAt = this.text.slice(piece.start, from).trim() + ? { piece, share: (from - piece.start) / Math.max(piece.end - piece.start, 1) } + : null; + this.spoken = from; + this.playing = false; + void this.pump(); + + return true; + } + /** Stops reading: the current piece, and what was still to come. */ stop(): void { this.isPaused = false; for (const wake of this.resumeWaiters.splice(0)) wake(); - this.controller.abort(); + this.fetches.abort(); + this.run.abort(); this.queue = []; - this.prefetched.clear(); + this.startAt = null; + this.fetched.clear(); this.stopAudio(); this.playing = false; this.speaking = false; @@ -137,6 +182,7 @@ export class ReplyReader { // the first piece may end at a clause, so the voice starts sooner for (const piece of readAloudPieces(text, this.queuedTo, done, this.queuedTo === 0)) { + this.pieces.push(piece); this.queue.push(piece); this.queuedTo = piece.end; } @@ -151,16 +197,16 @@ export class ReplyReader { } private fetchPiece(piece: ReadAloudPiece): Promise { - let pending = this.prefetched.get(piece); + let pending = this.fetched.get(piece); if (!pending) { // the model's silence around each piece is cut short, so the voice runs on pending = this.options - .synthesize(piece.speech, this.controller.signal) + .synthesize(piece.speech, this.fetches.signal) .then((blob) => trimSilence(blob)); - // a failed fetch is handled where it is played - pending.catch(() => {}); - this.prefetched.set(piece, pending); + // a failed fetch is handled where it is played, and fetched again next time + pending.catch(() => this.fetched.delete(piece)); + this.fetched.set(piece, pending); } return pending; @@ -180,13 +226,21 @@ export class ReplyReader { } } - private async play(piece: ReadAloudPiece, blob: Blob, signal: AbortSignal): Promise { + private async play( + piece: ReadAloudPiece, + blob: Blob, + signal: AbortSignal, + startShare = 0 + ): Promise { const audio = this.element(); const url = URL.createObjectURL(blob); try { await new Promise((resolve) => { const follow = () => { + // a piece left behind by seek() or stop() no longer moves the text + if (signal.aborted) return; + if (audio.duration > 0 && Number.isFinite(audio.duration)) { const share = Math.min(audio.currentTime / audio.duration, 1); @@ -198,7 +252,23 @@ export class ReplyReader { this.frame = requestAnimationFrame(follow); }; + const start = () => { + if (signal.aborted) return; + + // after seek(): from about where the word falls in the piece + if (startShare > 0 && Number.isFinite(audio.duration)) { + audio.currentTime = audio.duration * startShare; + } + + audio.play().then( + () => { + if (!signal.aborted) this.frame = requestAnimationFrame(follow); + }, + () => end() + ); + }; const end = () => { + audio.removeEventListener('loadedmetadata', start); audio.onended = null; audio.onerror = null; signal.removeEventListener('abort', end); @@ -213,10 +283,12 @@ export class ReplyReader { audio.preservesPitch = true; audio.playbackRate = this.options.playbackRate?.() ?? 1; audio.defaultPlaybackRate = audio.playbackRate; - audio.play().then( - () => (this.frame = requestAnimationFrame(follow)), - () => end() - ); + + if (startShare > 0) { + audio.addEventListener('loadedmetadata', start, { once: true }); + } else { + start(); + } }); } finally { if (this.frame !== null) cancelAnimationFrame(this.frame); @@ -237,8 +309,10 @@ export class ReplyReader { return; } - const { signal } = this.controller; + const { signal } = this.run; + const startShare = this.startAt?.piece === piece ? this.startAt.share : 0; + this.startAt = null; this.playing = true; this.speaking = true; @@ -256,14 +330,12 @@ export class ReplyReader { await new Promise((resolve) => this.resumeWaiters.push(resolve)); } - if (!signal.aborted && !this.isMuted) await this.play(piece, blob, signal); + if (!signal.aborted && !this.isMuted) await this.play(piece, blob, signal, startShare); } catch (error) { if (!signal.aborted) console.warn('Reading the reply aloud failed:', error); } } - this.prefetched.delete(piece); - if (signal.aborted) return; this.spoken = Math.max(this.spoken, piece.end); diff --git a/ui/apps/chat/tests/client/voice-reply-reader.svelte.test.ts b/ui/apps/chat/tests/client/voice-reply-reader.svelte.test.ts new file mode 100644 index 000000000000..3149b5bbccd3 --- /dev/null +++ b/ui/apps/chat/tests/client/voice-reply-reader.svelte.test.ts @@ -0,0 +1,163 @@ +// Guards reading a reply aloud from a word the user picks: the voice moves there, back or +// ahead, plays audio it already fetched without asking for it again, and starts inside a +// sentence at about where the word falls in it. + +import { ReplyReader } from '#lib/voice/reply-reader.svelte.ts'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; + +const DURATION = 2; + +/** An audio element that "plays" a clip of DURATION seconds until finish() is called. */ +class FakeAudio extends EventTarget { + currentTime = 0; + defaultPlaybackRate = 1; + duration = Number.NaN; + static last: FakeAudio | null = null; + onended: (() => void) | null = null; + onerror: (() => void) | null = null; + paused = true; + playbackRate = 1; + preservesPitch = true; + /** What each play() started on: the clip and where in it */ + starts: { src: string; time: number }[] = []; + private source = ''; + + get src() { + return this.source; + } + + set src(value: string) { + this.source = value; + this.currentTime = 0; + this.duration = Number.NaN; + // metadata comes after the source is set, as in a browser + queueMicrotask(() => { + this.duration = DURATION; + this.dispatchEvent(new Event('loadedmetadata')); + }); + } + + constructor() { + super(); + FakeAudio.last = this; + } + + override dispatchEvent(event: Event): boolean { + if (event.type === 'ended') this.onended?.(); + + return super.dispatchEvent(event); + } + + /** The clip plays to its end. */ + finish() { + this.currentTime = this.duration; + this.paused = true; + this.onended?.(); + } + + pause() { + this.paused = true; + } + + play() { + this.paused = false; + this.starts.push({ src: this.source, time: this.currentTime }); + + return Promise.resolve(); + } +} + +const REPLY = 'One two three. Four five six. Seven eight nine.'; +const at = (word: string) => REPLY.indexOf(word); +const settle = () => new Promise((resolve) => setTimeout(resolve, 20)); + +function reader() { + const synthesize = vi.fn(async (text: string) => new Blob([text], { type: 'audio/test' })); + const replyReader = new ReplyReader({ synthesize }); + + replyReader.reset(); + replyReader.update(REPLY, true); + + return { replyReader, synthesize }; +} + +/** The text of the clip that last started, and where in it. */ +function lastStart() { + const start = FakeAudio.last!.starts.at(-1)!; + + return { text: clips.get(start.src), time: start.time }; +} + +const clips = new Map(); + +beforeAll(() => { + vi.stubGlobal('Audio', FakeAudio); + vi.spyOn(URL, 'createObjectURL').mockImplementation((blob) => { + const url = `blob:test/${clips.size}`; + + void (blob as Blob).text().then((text) => clips.set(url, text)); + + return url; + }); + vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}); +}); + +afterAll(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe('ReplyReader.seek', () => { + it('jumps ahead to a later sentence', async () => { + const { replyReader, synthesize } = reader(); + + await settle(); + expect(lastStart().text).toBe('One two three.'); + + expect(replyReader.seek(at('Seven'))).toBe(true); + expect(replyReader.spoken).toBe(at('Seven')); + await settle(); + + expect(lastStart()).toEqual({ text: 'Seven eight nine.', time: 0 }); + // the later sentences were fetched while the first played: nothing is asked for twice + expect(synthesize).toHaveBeenCalledTimes(3); + replyReader.stop(); + }); + + it('goes back into a sentence already read, from about where the word is', async () => { + const { replyReader, synthesize } = reader(); + + await settle(); + FakeAudio.last!.finish(); + await settle(); + expect(lastStart().text).toBe('Four five six.'); + + expect(replyReader.seek(at('three'))).toBe(true); + expect(replyReader.spoken).toBe(at('three')); + await settle(); + + const { text, time } = lastStart(); + + expect(text).toBe('One two three.'); + expect(time).toBeGreaterThan(0.8); + expect(time).toBeLessThan(DURATION); + expect(synthesize).toHaveBeenCalledTimes(3); + + // and reads on from there + FakeAudio.last!.finish(); + await settle(); + expect(lastStart()).toEqual({ text: 'Four five six.', time: 0 }); + replyReader.stop(); + }); + + it('has nowhere to go once muted or read to the end', async () => { + const { replyReader } = reader(); + + replyReader.muted = true; + expect(replyReader.seek(at('Four'))).toBe(false); + + replyReader.muted = false; + replyReader.stop(); + expect(replyReader.seek(at('Four'))).toBe(false); + }); +}); diff --git a/ui/apps/chat/tests/client/voice-spoken-text.svelte.test.ts b/ui/apps/chat/tests/client/voice-spoken-text.svelte.test.ts new file mode 100644 index 000000000000..d78a69f1e35a --- /dev/null +++ b/ui/apps/chat/tests/client/voice-spoken-text.svelte.test.ts @@ -0,0 +1,26 @@ +// Guards picking a word of a reply being read aloud: a click gives where the word starts, +// and without a listener the words are plain text. + +import { VoiceSpokenText } from '@ggml-org/llama-ui/voice'; +import { describe, expect, it, vi } from 'vitest'; +import { page, userEvent } from 'vitest/browser'; +import { render } from 'vitest-browser-svelte'; + +const TEXT = 'Read this. Then that.'; + +describe('VoiceSpokenText word picks', () => { + it('gives where a clicked word starts', async () => { + const onWordSelect = vi.fn(); + + render(VoiceSpokenText, { onWordSelect, spoken: 4, text: TEXT }); + + await userEvent.click(page.getByText('Then')); + expect(onWordSelect).toHaveBeenCalledWith(TEXT.indexOf(' Then') + 1); + }); + + it('leaves the words alone without a listener', () => { + const { container } = render(VoiceSpokenText, { spoken: 4, text: TEXT }); + + expect(container.querySelector('[data-start]')).toBeNull(); + }); +}); diff --git a/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte b/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte index f9299cc1eac1..9e2aa8e31592 100644 --- a/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte +++ b/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte @@ -3,6 +3,7 @@ muted. Used for a transcript followed during playback (`spoken`, e.g. from spokenLength()), for a reply read aloud, and for text that streams in while someone speaks (`live`: each new word brightens as it arrives). Without `spoken` or `live` all of it reads as spoken. The words are plain text for screen readers. + With `onWordSelect`, a click or tap on a word gives where it starts, e.g. to read on from there. --> - + +{#each words as { start, word }, index (live ? index : start)}{word}{word}{/each} From 3bc7bd2e9ad5cbdd3c85218deaa90fe7b89e2dd5 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 16 Nov 2026 13:05:40 +0100 Subject: [PATCH 10/10] ui: show a reasoning effort that is not the default on the model selector Next to the model on the closed selector, a set level shows as its word, and reasoning that is off as a small crossed-out lightbulb. At the default, which leaves it to the model, nothing more shows. (cherry picked from commits 44b368f0f0 and f91f4a9457 on develop) Assisted-by: Claude (Opus 5.5 Medium) --- .../app/chat/VoiceMode/VoiceModeScreen.svelte | 2 +- .../ModelsSelector/ModelsSelector.svelte | 6 ++++ .../ModelsSelectorDropdown.svelte | 6 ++++ .../ModelsSelectorReasoningBadge.svelte | 34 +++++++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorReasoningBadge.svelte diff --git a/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte index e0cfe55614cf..c2db460fcdd9 100644 --- a/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte @@ -6,6 +6,7 @@ rises above the microphone and sinks away once answered. --> + +{#if reasoning.isOff} + + +{:else if reasoning.currentEffort !== ReasoningEffort.DEFAULT} + + + + Reasoning {REASONING_EFFORT_LABELS[reasoning.currentEffort]} + +{/if}