diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte index 5b296c62dced..12b66d18e57c 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -64,7 +64,7 @@ parseClipboardContent, uuid } from '#lib/utils/index.ts'; - import { VoiceRecorder, warmUpVoiceCapture } from '#lib/voice/index.ts'; + import { unlockVoicePlayback, VoiceRecorder, warmUpVoiceCapture } from '#lib/voice/index.ts'; import { onMount, tick, untrack } from 'svelte'; import type { ClassValue } from 'svelte/elements'; import { toast } from 'svelte-sonner'; @@ -99,6 +99,8 @@ /** Records voice input; tests and stories pass a fake, the app a VoiceRecorder. */ voiceRecorder?: VoiceRecorderLike; + /** Offers voice mode, a spoken conversation, as the main button of an empty message. */ + voiceMode?: boolean; } let { @@ -121,6 +123,7 @@ singleRow = false, uploadedFiles = $bindable([]), value = $bindable(''), + voiceMode = false, voiceRecorder }: Props = $props(); @@ -668,6 +671,16 @@ onSubmit?.(); } + // voice mode leads only when what is said can be transcribed: the chat model hears, or a + // transcription model is set or loaded; else the microphone leads, saying why it cannot + let voiceModeAvailable = $derived(voiceMode && !voice.unavailableReason); + + // inside the click: a browser that plays sound only after a gesture lets the replies play later + function handleVoiceModeClick() { + unlockVoicePlayback(); + uiStore.voiceModeOpen = true; + } + // Escape drops a recording, wherever the focus is function handleWindowKeydown(event: KeyboardEvent) { if (event.key !== KeyboardKey.ESCAPE || !voice.isActive || event.defaultPrevented) return; @@ -862,6 +875,7 @@ {onStop} onSystemPromptClick={() => onSystemPromptClick?.({ files: uploadedFiles, message: value })} + onVoiceModeClick={voiceModeAvailable ? handleVoiceModeClick : undefined} onWorkingDirectoryClick={isWorkingDirectoryInAddMenu ? pickers.handleWorkingDirectoryOpen : undefined} diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte index 73069301474e..439e4f98cea3 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte @@ -17,7 +17,7 @@ micWarning?: string | null; /** Why no model can take the recording; disables the button. */ unavailableReason?: string | null; - /** A quieter button, beside the main one (stop or send). */ + /** A quieter button, beside the main one: stop, send or voice mode. */ secondary?: boolean; /** With `secondary`: filled like the stop button it stands beside, not bare. */ filled?: boolean; diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionVoiceMode.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionVoiceMode.svelte new file mode 100644 index 000000000000..67bba80b724c --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionVoiceMode.svelte @@ -0,0 +1,46 @@ + + + +
+ + + + + + +

{unavailableReason ?? VOICE_CONVERSATION_MESSAGE.open}

+
+
+
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte index a523300286a8..44368c396ca2 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte @@ -2,10 +2,10 @@ import { Button } from '@ggml-org/llama-ui/ui/button'; import { LoaderCircle, SkipForward, Square } from '@lucide/svelte'; import { - ChatFormActionReasoning, ChatFormActionRecord, ChatFormActionsAdd, ChatFormActionSubmit, + ChatFormActionVoiceMode, ChatFormContextGauge, ModelsSelector } from '#lib/components/app/index.ts'; @@ -50,6 +50,10 @@ workingDirectory?: string | null; onFileUpload?: () => void; onMicClick?: () => void; + /** Opens voice mode; left out, the microphone is the main button. */ + onVoiceModeClick?: () => void; + /** Why voice mode cannot listen. */ + voiceModeUnavailableReason?: string | null; onStop?: () => void; onSystemPromptClick?: () => void; onWorkingDirectoryClick?: () => void; @@ -69,11 +73,13 @@ onMicClick, onStop, onSystemPromptClick, + onVoiceModeClick, onWorkingDirectoryClick, showAddButton = true, showModelSelector = true, uploadedFiles = [], voice, + voiceModeUnavailableReason = null, workingDirectory = null }: Props = $props(); @@ -171,12 +177,12 @@ {/if} {#if showModelSelector} - - + {/if} @@ -236,6 +242,7 @@ /> {:else if shouldShowRecordButton} + + + {#if onVoiceModeClick} + + {/if} {:else} {/if} +{#if uiStore.voiceModeOpen} + +{/if} + diff --git a/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte new file mode 100644 index 000000000000..c2db460fcdd9 --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/chat/VoiceMode/VoiceModeScreen.svelte @@ -0,0 +1,348 @@ + + + + + + diff --git a/ui/apps/chat/src/lib/components/app/chat/index.ts b/ui/apps/chat/src/lib/components/app/chat/index.ts index f7626c73a4f5..554d77333a0f 100644 --- a/ui/apps/chat/src/lib/components/app/chat/index.ts +++ b/ui/apps/chat/src/lib/components/app/chat/index.ts @@ -185,6 +185,16 @@ export { default as ChatFormActionsAdd } from './ChatForm/ChatFormActions/ChatFo */ export { default as ChatFormActionRecord } from './ChatForm/ChatFormActions/ChatFormActionRecord.svelte'; +/** + * **ChatFormActionVoiceMode** - Opens voice mode, the chat form's main button while the message is empty + */ +export { default as ChatFormActionVoiceMode } from './ChatForm/ChatFormActions/ChatFormActionVoiceMode.svelte'; + +/** + * **VoiceModeScreen** - Full-screen spoken conversation with the chat model + */ +export { default as VoiceModeScreen } from './VoiceMode/VoiceModeScreen.svelte'; + /** * The actions row while the chat form records: cancel, the live waveform and the * elapsed time, and stop, with a voice message's streaming text under the waveform diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte index 3b24b83a0472..7138a07f91cb 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte @@ -1,8 +1,11 @@ + +{#snippet header()} + {#if reasoning.isOff} + + {:else} + + {/if} + + Reasoning + + {current} +{/snippet} + +{#snippet levelLabel(level: ReasoningEffortLevel)} + {@const tokenLabel = reasoning.tokenLabel(level)} + + {#if reasoning.isSelected(level)} + + {:else} +
+ {/if} + + {level.label} + + {#if tokenLabel} + {tokenLabel} + {/if} +{/snippet} + +{#if variant === 'menu'} + + + +
+ {#each reasoning.levels as level (level.value)} + reasoning.select(level)}> + {@render levelLabel(level)} + + {/each} +
+
+
+{:else} +
+ +
+ {#each reasoning.levels as level (level.value)} + + {/each} +
+
+
+{/if} diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorReasoningBadge.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorReasoningBadge.svelte new file mode 100644 index 000000000000..fabc65c5425c --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorReasoningBadge.svelte @@ -0,0 +1,34 @@ + + + +{#if reasoning.isOff} + + +{:else if reasoning.currentEffort !== ReasoningEffort.DEFAULT} + + + + Reasoning {REASONING_EFFORT_LABELS[reasoning.currentEffort]} + +{/if} diff --git a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts index 7b067c468095..dcaed3da880a 100644 --- a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts +++ b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts @@ -94,6 +94,7 @@ export const SETTINGS_KEYS = { VOICE_INPUT_MODE: 'voiceInputMode', VOICE_MICROPHONE: 'voiceMicrophone', VOICE_NOISE_SUPPRESSION: 'voiceNoiseSuppression', + VOICE_READING_SPEED: 'voiceReadingSpeed', VOICE_TRANSCRIBE_MESSAGES: 'voiceTranscribeMessages', VOICE_TRANSCRIPTION_LANGUAGE: 'voiceTranscriptionLanguage', VOICE_TRANSCRIPTION_MODEL: 'voiceTranscriptionModel', diff --git a/ui/apps/chat/src/lib/constants/settings.constants.ts b/ui/apps/chat/src/lib/constants/settings.constants.ts index 4544a88df01f..ebf19c82715f 100644 --- a/ui/apps/chat/src/lib/constants/settings.constants.ts +++ b/ui/apps/chat/src/lib/constants/settings.constants.ts @@ -12,6 +12,7 @@ import { Database, File, Funnel, + Gauge, GlassWater, ListRestart, Mic, @@ -481,6 +482,20 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [ label: 'Automatic gain', type: SettingsFieldType.CHECKBOX }, + { + defaultValue: '1.25', + help: 'How fast voice mode reads replies aloud. The voice keeps its pitch.', + key: SETTINGS_KEYS.VOICE_READING_SPEED, + label: 'Reading speed', + options: [ + { icon: Gauge, label: '1×', value: '1' }, + { icon: Gauge, label: '1.25×', value: '1.25' }, + { icon: Gauge, label: '1.5×', value: '1.5' }, + { icon: Gauge, label: '1.75×', value: '1.75' }, + { icon: Gauge, label: '2×', value: '2' } + ], + type: SettingsFieldType.SELECT + }, { defaultValue: '', help: 'The language you speak, as a two-letter code such as en or de, passed to the transcription. Leave it empty to let the model tell.', diff --git a/ui/apps/chat/src/lib/constants/voice.constants.ts b/ui/apps/chat/src/lib/constants/voice.constants.ts index 7c49dcfff478..b152017e11e1 100644 --- a/ui/apps/chat/src/lib/constants/voice.constants.ts +++ b/ui/apps/chat/src/lib/constants/voice.constants.ts @@ -16,6 +16,25 @@ export const VOICE_MODE_MESSAGE = { voiceMessage: 'Send a voice message' } as const; +/** What voice mode says: its states, buttons and the reason it cannot read replies aloud. */ +export const VOICE_CONVERSATION_MESSAGE = { + close: 'Leave voice mode', + greeting: "Let's talk", + interrupt: 'Interrupt the reply', + listening: 'Listening...', + micOff: 'Microphone off', + mute: 'Turn the microphone off', + noSpeechModel: + 'No model that speaks is available, so replies are shown, not read aloud. Load a model that generates audio to hear them.', + open: 'Voice mode', + speakerOff: 'Stop reading replies aloud', + speakerOn: 'Read replies aloud', + starting: 'Starting...', + thinking: 'Thinking...', + transcribing: 'Transcribing...', + unmute: 'Turn the microphone on' +} as const; + /** Where the chat form remembers that it showed the hint about a separate transcription model. */ export const VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.voiceSeparateModelHintSeen`; diff --git a/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts new file mode 100644 index 000000000000..a99eaa39640b --- /dev/null +++ b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts @@ -0,0 +1,576 @@ +/** + * Voice mode: a spoken conversation with the chat model, hands-free. + * + * Each turn listens (echo cancellation on, as the reply plays from the same device), + * shows what is said as it is transcribed, and ends after a pause in speech: the text + * goes to the chat as a message. The reply streams on screen and, when a model that + * speaks is available, is read aloud sentence by sentence; then the next turn listens. + * Speaking while the reply plays stops it and starts a new turn. + */ + +import { SETTINGS_KEYS } from '#lib/constants/index.ts'; +import { MessageRole, VoiceInputMode, VoiceMode } from '#lib/enums/index.ts'; +import { + chatStore, + conversationsStore, + modelsStore, + serverStore, + settingsStore +} from '#lib/stores/index.ts'; +import { getAuthHeaders } from '#lib/utils/api-headers.ts'; +import { describeMicrophoneError } from '#lib/utils/audio-recording.ts'; +import { withBase } from '#lib/utils/base-path.ts'; +import { resolveVoiceMode } from '#lib/utils/voice-mode.ts'; +import { + PauseDetector, + ReplyReader, + resolveMicrophone, + synthesizeSpeech, + VOICE_LEVELS_INTERVAL_MS, + VOICE_MICROPHONE_AUTO, + VoiceCapture, + VoiceCaptureError, + VoiceRecorder, + type VoiceServerContext +} from '#lib/voice/index.ts'; +import { onDestroy, untrack } from 'svelte'; + +/** Where a voice conversation is. */ +export type VoiceConversationState = + | 'idle' + | 'starting' + | 'listening' + | 'transcribing' + | 'thinking' + | 'speaking' + | 'paused'; + +/** One exchange: what was said, and the reply. */ +export interface VoiceTurn { + id: number; + user: string; + reply: string; + /** The message went to the chat, so its reply is followed */ + sent: boolean; + done: boolean; +} + +// voice mode talks to a device that also plays the reply: the browser's clean-up removes its echo +const CONVERSATION_CONSTRAINTS: MediaTrackConstraints = { + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true +}; +/** Silence after speech that ends a turn. */ +const TURN_PAUSE_MS = 900; +/** Sound over the reply that pauses it, to hear whether it is speech. */ +const BARGE_IN_MS = 300; +/** How long a paused reply waits for speech before it goes on: no speech, it was a noise or its own echo. */ +const BARGE_IN_CONFIRM_MS = 1000; + +function serverContext(): { context: VoiceServerContext; realtime: boolean } { + const headers = getAuthHeaders(); + const hasAuth = Object.keys(headers).some((key) => key.toLowerCase() === 'authorization'); + + return { + context: { + getHeaders: () => getAuthHeaders(), + resolveUrl: (path) => withBase(path.replace(/^\./, '')) + }, + realtime: !hasAuth + }; +} + +export function useVoiceConversation() { + const config = $derived(settingsStore.config); + const isRouter = $derived(serverStore.isRouterMode); + const chatModel = $derived(modelsStore.activeModelId); + const hasAudio = (model: string | null) => { + void modelsStore.props.cacheVersion; + + return model ? modelsStore.props.modelSupportsAudio(model) : false; + }; + const transcriptionSetting = $derived( + String(config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL] ?? '') + ); + // what is said always becomes text here, so it is resolved as dictation + const resolution = $derived.by(() => { + void modelsStore.props.cacheVersion; + + const loadedAudioModel = isRouter + ? (modelsStore.models.find( + (option) => hasAudio(option.model) && modelsStore.isModelLoaded(option.model) + )?.model ?? null) + : null; + + return resolveVoiceMode({ + chatModel, + chatModelHasAudio: hasAudio(chatModel), + isRouter, + loadedAudioModel, + setting: VoiceInputMode.DICTATION, + transcriptionModel: transcriptionSetting, + transcriptionModelHasAudio: hasAudio(transcriptionSetting || null) + }); + }); + // the chat model reads its own replies when it can speak, else another model that can + const speechModel = $derived.by(() => { + const options = modelsStore.models; + const chat = options.find((option) => option.model === chatModel); + + if (chat?.speaks) return chat.model; + + return isRouter ? (options.find((option) => option.speaks)?.model ?? null) : null; + }); + + let state = $state('idle'); + let micMuted = $state(false); + let speakerMuted = $state(false); + let liveText = $state(''); + let levels = $state([]); + let error = $state(null); + let turns = $state([]); + let recorder: VoiceRecorder | null = null; + let monitor: VoiceCapture | null = null; + let generation = 0; + let turnId = 0; + // the current reply was seen loading: until then, not loading does not mean done + let replyStarted = false; + // the reply is paused over a sound, until speech confirms the interruption or a second passes + let tentative = false; + let tentativeTimer: ReturnType | undefined; + // a reply paused by an interruption: it goes on if the microphone is muted or nothing is + // said, and is dropped once the user's turn is sent + let heldReply = $state(false); + + const reader = new ReplyReader({ + onIdle: () => { + if (state === 'speaking' || state === 'thinking') void listen(); + }, + playbackRate: () => Number(config[SETTINGS_KEYS.VOICE_READING_SPEED]) || 1.25, + synthesize: (text, signal) => { + const { context } = serverContext(); + + return synthesizeSpeech(context, text, { + model: isRouter && speechModel ? speechModel : undefined, + signal + }); + } + }); + const currentTurn = $derived(turns[turns.length - 1]); + + // the reply as it streams, then as stored once done; the conversation is read here, as a + // first message creates it while being sent + $effect(() => { + const turn = currentTurn; + const convId = conversationsStore.activeConversation?.id; + + if (!turn || !turn.sent || turn.done || !convId) return; + + const streaming = chatStore.getChatStreaming(convId); + const loading = chatStore.isChatLoading(convId); + + untrack(() => { + if (loading) { + replyStarted = true; + + if (!streaming) return; + + turn.reply = streaming.response; + reader.update(streaming.response, false); + + return; + } + + if (replyStarted) finishReply(turn); + }); + }); + + /** The reply is complete: as stored, read to its end. */ + function finishReply(turn: VoiceTurn) { + if (turn.done) return; + + const last = conversationsStore.activeMessages.at(-1); + const text = last?.role === MessageRole.ASSISTANT ? last.content : turn.reply; + + turn.reply = text; + turn.done = true; + reader.update(text, true); + } + + // the reply plays: speaking over it may interrupt it + $effect(() => { + if (reader.speaking && state === 'thinking') { + untrack(() => { + state = 'speaking'; + void watchForBargeIn(generation); + }); + } + }); + + async function microphone(): Promise { + return resolveMicrophone( + String(config[SETTINGS_KEYS.VOICE_MICROPHONE] ?? VOICE_MICROPHONE_AUTO) + ); + } + + function stopRecorder() { + recorder?.cancel(); + recorder = null; + } + + function stopMonitor() { + monitor?.cancel(); + monitor = null; + } + + /** + * Starts a turn: listens until the speaker pauses, then sends what was said. `floor` is + * the room's noise level when known; `tentative` listens over a paused reply, which + * goes on unless speech is heard within a second. + */ + async function listen(options: { floor?: number; tentative?: boolean } = {}) { + stopMonitor(); + stopRecorder(); + liveText = ''; + levels = []; + + if (micMuted) { + state = 'paused'; + + return; + } + + if (resolution.mode === VoiceMode.UNAVAILABLE) { + error = resolution.reason; + state = 'paused'; + + return; + } + + const run = ++generation; + const model = resolution.transcriptionModel; + const { context, realtime } = serverContext(); + const language = String(config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE] ?? '').trim(); + const pauses = new PauseDetector({ + floor: options.floor, + intervalMs: VOICE_LEVELS_INTERVAL_MS, + maxSegmentMs: Number.POSITIVE_INFINITY, + pauseMs: TURN_PAUSE_MS + }); + const active = new VoiceRecorder(); + + recorder = active; + state = 'starting'; + + try { + const deviceId = await microphone(); + + await active.start({ + capture: { constraints: CONVERSATION_CONSTRAINTS, ...(deviceId ? { deviceId } : {}) }, + onLevels: (next) => { + levels = next as number[]; + + if (run !== generation) return; + + const cut = pauses.push(next.at(-1) ?? 0); + + // speech over the paused reply: the interruption was meant + if (tentative && pauses.heardSpeech) confirmInterrupt(); + + if (state === 'listening' && cut) void endTurn(run); + }, + onText: (text) => { + if (run === generation) liveText = text; + }, + realtime, + transcribe: { + context, + language: language || undefined, + model: isRouter && model ? model : undefined + } + }); + } catch (cause) { + if (run !== generation) return; + + error = cause instanceof VoiceCaptureError ? cause.message : describeMicrophoneError(cause); + state = 'paused'; + + return; + } + + if (run !== generation) { + active.cancel(); + + return; + } + + error = null; + state = 'listening'; + + if (options.tentative && tentative) { + clearTimeout(tentativeTimer); + tentativeTimer = setTimeout(() => { + if (run === generation && tentative && !pauses.heardSpeech) resumeReading(); + }, BARGE_IN_CONFIRM_MS); + } + } + + /** The speaker paused: what they said goes to the chat. */ + async function endTurn(run: number) { + const active = recorder; + + if (!active || run !== generation) return; + + state = 'transcribing'; + recorder = null; + + let text = liveText; + + try { + const result = await active.stop(); + + text = (await (result.transcript ?? Promise.resolve(text))) || result.liveText || text; + } catch (cause) { + console.warn('Voice mode: the turn could not be transcribed:', cause); + } + + if (run !== generation) return; + + text = text.trim(); + + // nothing understood: a reply held for this turn goes on, else listen again + if (!text) { + if (heldReply) resumeReading(); + else void listen(); + + return; + } + + // the user had something to say: the reply they interrupted gives way + dropHeldReply(); + + const turn: VoiceTurn = { done: false, id: ++turnId, reply: '', sent: false, user: text }; + + turns.push(turn); + liveText = ''; + reader.reset(); + reader.muted = speakerMuted || !speechModel; + replyStarted = false; + state = 'thinking'; + + const sending = chatStore.sendMessage(text); + + // followed from here: the message is on its way + turns[turns.length - 1].sent = true; + + try { + await sending; + + // a reply that was never seen streaming (an error, a very short one) ends here + if (run === generation) finishReply(turns[turns.length - 1]); + } catch (cause) { + if (run !== generation) return; + + error = cause instanceof Error ? cause.message : String(cause); + void listen(); + } + } + + /** While the reply plays, speech over it stops it and starts a new turn. */ + async function watchForBargeIn(run: number) { + if (micMuted || speakerMuted || !speechModel || monitor) return; + + const capture = new VoiceCapture({ constraints: CONVERSATION_CONSTRAINTS }); + + let speechMs = 0; + let floor = 1; + + monitor = capture; + capture.onLevels((next) => { + const level = next.at(-1) ?? 0; + + floor = Math.min(level, floor + 0.002); + speechMs = level > floor + 0.2 ? speechMs + VOICE_LEVELS_INTERVAL_MS : 0; + + if (speechMs >= BARGE_IN_MS && run === generation && reader.speaking && !reader.paused) { + bargeIn(floor); + } + }); + + try { + await capture.start(); + } catch { + // no barge-in without a microphone; the turn still follows the reply + } + + if (run !== generation || monitor !== capture || state !== 'speaking') { + capture.cancel(); + + if (monitor === capture) monitor = null; + } + } + + /** A sound over the reply: it pauses, and the next second tells whether it was speech. */ + function bargeIn(floor: number) { + reader.pause(); + heldReply = true; + tentative = true; + void listen({ floor, tentative: true }); + } + + /** No speech came, or the microphone was muted: the reply goes on from where it paused. */ + function resumeReading() { + heldReply = false; + tentative = false; + clearTimeout(tentativeTimer); + generation++; + stopRecorder(); + liveText = ''; + levels = []; + state = 'speaking'; + reader.resume(); + void watchForBargeIn(generation); + } + + /** Speech came: the turn that started goes on, over the reply held where it paused. */ + function confirmInterrupt() { + tentative = false; + clearTimeout(tentativeTimer); + } + + /** The interrupted reply is given up: its reading and its generation stop. */ + function dropHeldReply() { + if (!heldReply) return; + + heldReply = false; + reader.stop(); + + const convId = conversationsStore.activeConversation?.id; + + if (convId && chatStore.isChatLoading(convId)) void chatStore.stopGeneration(); + + const turn = currentTurn; + + if (turn) turn.done = true; + } + + /** Holds the reply where it is, and listens; muting the microphone lets it go on. */ + function interrupt() { + tentative = false; + clearTimeout(tentativeTimer); + reader.pause(); + heldReply = true; + void listen(); + } + + /** + * Reads the reply on from `offset`, e.g. a word tapped while it is read; a reply held for + * an interruption goes on from there too, as the tap says the user wants to hear it. + */ + function readFrom(offset: number) { + if (!reader.speaking && !heldReply) return; + + if (heldReply) resumeReading(); + + reader.seek(offset); + } + + /** Starts the conversation; the click that opened voice mode unlocked playback (unlockVoicePlayback). */ + function open() { + error = null; + turns = []; + void listen(); + } + + function close() { + heldReply = false; + tentative = false; + clearTimeout(tentativeTimer); + generation++; + stopRecorder(); + stopMonitor(); + reader.stop(); + state = 'idle'; + liveText = ''; + } + + function toggleMic() { + micMuted = !micMuted; + + if (micMuted) { + // an interrupted reply goes on from where it paused + if (heldReply) { + resumeReading(); + + return; + } + + if (state === 'listening' || state === 'starting') { + generation++; + stopRecorder(); + state = 'paused'; + } + + stopMonitor(); + + return; + } + + if (state === 'paused') void listen(); + } + + function toggleSpeaker() { + speakerMuted = !speakerMuted; + reader.muted = speakerMuted || !speechModel; + + if (speakerMuted) stopMonitor(); + } + + onDestroy(close); + + return { + /** Whether replies can be read aloud: a model that speaks is available. */ + get canSpeak() { + return !!speechModel; + }, + close, + get error() { + return error; + }, + interrupt, + get levels() { + return levels; + }, + get liveText() { + return liveText; + }, + get micMuted() { + return micMuted; + }, + open, + get reader() { + return reader; + }, + readFrom, + /** Whether the reply is being read, or held for an interruption: readFrom() can move it. */ + get reading() { + return reader.speaking || heldReply; + }, + get speakerMuted() { + return speakerMuted; + }, + get state() { + return state; + }, + toggleMic, + toggleSpeaker, + get turns() { + return turns; + }, + /** Why voice mode cannot listen, or null. */ + get unavailableReason() { + return resolution.mode === VoiceMode.UNAVAILABLE ? resolution.reason : null; + } + }; +} diff --git a/ui/apps/chat/src/lib/stores/models/index.svelte.ts b/ui/apps/chat/src/lib/stores/models/index.svelte.ts index 67518bf8bddf..7928defaad30 100644 --- a/ui/apps/chat/src/lib/stores/models/index.svelte.ts +++ b/ui/apps/chat/src/lib/stores/models/index.svelte.ts @@ -690,6 +690,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { model: modelId, name: this.toDisplayName(displayNameSource), parsedId: ModelsService.parseModelId(modelId), + speaks: item.architecture?.output_modalities?.includes('audio') ?? false, tags: item.tags ?? [] }; }) diff --git a/ui/apps/chat/src/lib/stores/ui.svelte.ts b/ui/apps/chat/src/lib/stores/ui.svelte.ts index 707648d06de7..1ba36014c014 100644 --- a/ui/apps/chat/src/lib/stores/ui.svelte.ts +++ b/ui/apps/chat/src/lib/stores/ui.svelte.ts @@ -107,6 +107,8 @@ class UiStore { sidebarSectionOrder = $state(loadSidebarSectionOrder()); /** Expanded desktop sidebar width in px; null keeps the stylesheet default. */ sidebarWidth = $state(this.loadSidebarWidth()); + /** Whether the full-screen voice mode is open: a spoken conversation with the chat model. */ + voiceModeOpen = $state(false); #isSidebarExpanded = $state(this.loadSidebarExpanded()); diff --git a/ui/apps/chat/src/lib/types/api.d.ts b/ui/apps/chat/src/lib/types/api.d.ts index abdd1b3de36e..371913fc4c26 100644 --- a/ui/apps/chat/src/lib/types/api.d.ts +++ b/ui/apps/chat/src/lib/types/api.d.ts @@ -121,6 +121,8 @@ export interface ApiModelDataEntry { export interface ApiModelArchitecture { /** Accepted input modalities, always contains "text" */ input_modalities: string[]; + /** Generated output modalities: "audio" when the model can speak */ + output_modalities?: string[]; } /** diff --git a/ui/apps/chat/src/lib/types/models.d.ts b/ui/apps/chat/src/lib/types/models.d.ts index 9d0594073406..9d4a7c88882b 100644 --- a/ui/apps/chat/src/lib/types/models.d.ts +++ b/ui/apps/chat/src/lib/types/models.d.ts @@ -23,6 +23,8 @@ export interface ModelOption { /** Draft sidecars downloaded alongside the model, e.g. `mtp` or `dflash`. */ draftSidecars?: ModelSidecarBadge[]; modalities?: ModelModalities; + /** The model can generate speech, e.g. to read replies aloud. */ + speaks?: boolean; details?: ApiModelDetails['details']; meta?: ApiModelDataEntry['meta']; parsedId?: ParsedModelId; diff --git a/ui/apps/chat/src/lib/voice/constants.ts b/ui/apps/chat/src/lib/voice/constants.ts index 7004254429d8..d38d050300f6 100644 --- a/ui/apps/chat/src/lib/voice/constants.ts +++ b/ui/apps/chat/src/lib/voice/constants.ts @@ -73,6 +73,7 @@ export const VOICE_BASE64_SLICE = 0x8000; /** Server routes, relative so a server under a subpath still works. */ export const VOICE_API_PATHS = { REALTIME: './v1/realtime', + SPEECH: './v1/audio/speech', TRANSCRIPTIONS: './v1/audio/transcriptions' } as const; diff --git a/ui/apps/chat/src/lib/voice/index.ts b/ui/apps/chat/src/lib/voice/index.ts index ed54f2db83b6..28f157753798 100644 --- a/ui/apps/chat/src/lib/voice/index.ts +++ b/ui/apps/chat/src/lib/voice/index.ts @@ -33,7 +33,11 @@ export { VOICE_MICROPHONE_DEFAULT } from './microphone'; export { computePeaks, computePeaksFromAudio, decodeAudio, mixToMono } from './peaks'; +export { PauseDetector, type PauseDetectorOptions } from './pauses'; +export { type ReadAloudPiece, readAloudPieces, toSpeech } from './read-aloud'; export { RealtimeTranscriber, realtimeUrl } from './realtime'; +export { ReplyReader, type ReplyReaderOptions, unlockVoicePlayback } from './reply-reader.svelte'; +export { synthesizeSpeech, type SynthesizeSpeechOptions } from './speech'; export { startVoiceSession } from './session'; export { transcribeClip } from './transcribe'; export type * from './types'; diff --git a/ui/apps/chat/src/lib/voice/pauses.ts b/ui/apps/chat/src/lib/voice/pauses.ts index 44083a7ee758..69815fe47c1b 100644 --- a/ui/apps/chat/src/lib/voice/pauses.ts +++ b/ui/apps/chat/src/lib/voice/pauses.ts @@ -17,6 +17,8 @@ export interface PauseDetectorOptions { * is ever quiet */ maxSegmentMs?: number; + /** The room's noise level when it is known already, so speech from the first level counts */ + floor?: number; } /** How far above the noise floor a level is speech, and below which it is quiet. */ @@ -47,6 +49,7 @@ export class PauseDetector { this.pauseMs = options.pauseMs ?? 500; this.minSpeechMs = options.minSpeechMs ?? 300; this.maxSegmentMs = options.maxSegmentMs ?? 10_000; + this.floor = options.floor ?? 1; } /** Takes the newest level; true when this is a place to cut the recording. */ diff --git a/ui/apps/chat/src/lib/voice/read-aloud.ts b/ui/apps/chat/src/lib/voice/read-aloud.ts new file mode 100644 index 000000000000..cf375edcfd65 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/read-aloud.ts @@ -0,0 +1,79 @@ +/** + * Cuts a reply into pieces to read aloud while it streams: whole sentences (or lines), + * each read as soon as it is complete, and the rest once the reply is done. Markdown + * marks are left out of what is spoken; the offsets point into the reply as written, + * so the text on screen can follow the voice. + */ + +export interface ReadAloudPiece { + /** Where the piece starts and ends in the reply as written */ + start: number; + end: number; + /** What is spoken: the piece without markdown marks */ + speech: string; +} + +// a sentence ends at . ! ? (with closing quotes or brackets) before whitespace, or a line ends; +// a list number such as "1." does not end one +const PIECE_END = /(?|[-*+]|\d+[.)])\s+/gm, '') + .replace(/[*_~]{1,3}([^*_~]+)[*_~]{1,3}/g, '$1') + .replace(/\s+/g, ' ') + .trim(); +} diff --git a/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts b/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts new file mode 100644 index 000000000000..2e084b2b044b --- /dev/null +++ b/ui/apps/chat/src/lib/voice/reply-reader.svelte.ts @@ -0,0 +1,363 @@ +import { type ReadAloudPiece, readAloudPieces } from './read-aloud'; +import { trimSilence } from './trim-silence'; + +// one element plays every reply: unlocked once inside a click, it may play later (iOS asks for that) +let sharedAudio: HTMLAudioElement | null = null; + +function playbackElement(): HTMLAudioElement { + sharedAudio ??= new Audio(); + + return sharedAudio; +} + +/** + * Plays nothing, inside a click: browsers that let a page play sound only after a gesture + * (iOS) then let replies play later. Call it in the handler of the click that opens voice mode. + */ +export function unlockVoicePlayback(): void { + if (typeof Audio === 'undefined') return; + + const audio = playbackElement(); + + audio.muted = true; + audio.play().catch(() => {}); + audio.pause(); + audio.muted = false; +} + +export interface ReplyReaderOptions { + /** Speaks a piece of text, e.g. with synthesizeSpeech(); the WAV or other audio to play */ + synthesize: (text: string, signal: AbortSignal) => Promise; + /** Called once the reply is done and all of it has been read (or skipped while muted) */ + onIdle?: () => void; + /** Speed of the voice, e.g. 1.25; the pitch is kept. Read before each piece. */ + playbackRate?: () => number; +} + +/** + * Reads a reply aloud while it streams: give it the reply as it grows with update(), + * and it speaks each complete sentence in order, fetching the next one while the + * current one plays. `spoken` follows the voice through the reply as written, for + * VoiceSpokenText, and seek() moves the voice to another place in it. Muted, it reads + * nothing and `spoken` keeps up with the text. + */ +export class ReplyReader { + /** Whether a piece is being fetched or played */ + speaking = $state(false); + /** Characters of the reply read so far */ + spoken = $state(0); + + private audio: HTMLAudioElement | null = null; + private done = false; + // the reply's pieces and their audio, kept so seek() can go back; bookkeeping the UI never reads + // eslint-disable-next-line svelte/prefer-svelte-reactivity + private fetched = new Map>(); + // the reply's speech requests, and the reading under way, which seek() starts anew + private fetches = new AbortController(); + private frame: number | null = null; + private idle = true; + private isMuted = false; + private isPaused = false; + private readonly options: ReplyReaderOptions; + private pieces: ReadAloudPiece[] = []; + private playing = false; + private queue: ReadAloudPiece[] = []; + private queuedTo = 0; + private resumeWaiters: (() => void)[] = []; + private run = new AbortController(); + // where in its piece the next one to play starts, after seek() + private startAt: { piece: ReadAloudPiece; share: number } | null = null; + private text = ''; + + get muted(): boolean { + return this.isMuted; + } + + /** Whether reading is paused, see pause(). */ + get paused(): boolean { + return this.isPaused; + } + + set muted(value: boolean) { + this.isMuted = value; + + if (value) this.silence(); + } + + constructor(options: ReplyReaderOptions) { + this.options = options; + } + + /** Holds the voice where it is, e.g. to hear whether the user speaks; resume() goes on from there. */ + pause(): void { + if (this.isPaused) return; + + this.isPaused = true; + + if (this.audio && !this.audio.paused) this.audio.pause(); + } + + /** Starts a new reply. */ + reset(): void { + this.stop(); + this.fetches = new AbortController(); + this.run = new AbortController(); + this.pieces = []; + this.text = ''; + this.done = false; + this.queuedTo = 0; + this.spoken = 0; + this.idle = false; + } + + /** Goes on reading from where pause() held it. */ + resume(): void { + if (!this.isPaused) return; + + this.isPaused = false; + + const audio = this.audio; + + if (this.playing && audio?.src && !audio.ended && audio.paused) audio.play().catch(() => {}); + + for (const wake of this.resumeWaiters.splice(0)) wake(); + } + + /** + * Reads on from `offset` in the reply as written, e.g. a word tapped on screen, before + * or after where the voice is; a piece already fetched plays at once, from about where + * the word falls in it. False when there is nothing to read there: the reply was read + * to its end, the voice is muted, or that sentence is not complete yet. + */ + seek(offset: number): boolean { + if (this.idle || this.isMuted) return false; + + const index = this.pieces.findIndex((piece) => offset < piece.end); + + if (index === -1) return false; + + const piece = this.pieces[index]; + const from = Math.max(offset, piece.start); + + this.run.abort(); + this.run = new AbortController(); + + // a reading that waited on a pause or a fetch sees it was aborted and lets go + for (const wake of this.resumeWaiters.splice(0)) wake(); + + this.stopAudio(); + this.queue = this.pieces.slice(index); + // a piece opens with the space after the one before: a word right after it starts the piece + this.startAt = this.text.slice(piece.start, from).trim() + ? { piece, share: (from - piece.start) / Math.max(piece.end - piece.start, 1) } + : null; + this.spoken = from; + this.playing = false; + void this.pump(); + + return true; + } + + /** Stops reading: the current piece, and what was still to come. */ + stop(): void { + this.isPaused = false; + + for (const wake of this.resumeWaiters.splice(0)) wake(); + + this.fetches.abort(); + this.run.abort(); + this.queue = []; + this.startAt = null; + this.fetched.clear(); + this.stopAudio(); + this.playing = false; + this.speaking = false; + this.idle = true; + } + + /** The reply so far; `done` once it is complete. */ + update(text: string, done: boolean): void { + this.text = text; + this.done = done; + + // the first piece may end at a clause, so the voice starts sooner + for (const piece of readAloudPieces(text, this.queuedTo, done, this.queuedTo === 0)) { + this.pieces.push(piece); + this.queue.push(piece); + this.queuedTo = piece.end; + } + + void this.pump(); + } + + private element(): HTMLAudioElement { + this.audio ??= playbackElement(); + + return this.audio; + } + + private fetchPiece(piece: ReadAloudPiece): Promise { + let pending = this.fetched.get(piece); + + if (!pending) { + // the model's silence around each piece is cut short, so the voice runs on + pending = this.options + .synthesize(piece.speech, this.fetches.signal) + .then((blob) => trimSilence(blob)); + // a failed fetch is handled where it is played, and fetched again next time + pending.catch(() => this.fetched.delete(piece)); + this.fetched.set(piece, pending); + } + + return pending; + } + + private finishIfDone(): void { + if (this.playing || this.queue.length > 0 || !this.done || this.queuedTo < this.text.length) { + return; + } + + this.speaking = false; + this.spoken = this.text.length; + + if (!this.idle) { + this.idle = true; + this.options.onIdle?.(); + } + } + + private async play( + piece: ReadAloudPiece, + blob: Blob, + signal: AbortSignal, + startShare = 0 + ): Promise { + const audio = this.element(); + const url = URL.createObjectURL(blob); + + try { + await new Promise((resolve) => { + const follow = () => { + // a piece left behind by seek() or stop() no longer moves the text + if (signal.aborted) return; + + if (audio.duration > 0 && Number.isFinite(audio.duration)) { + const share = Math.min(audio.currentTime / audio.duration, 1); + + this.spoken = Math.max( + this.spoken, + piece.start + Math.round((piece.end - piece.start) * share) + ); + } + + this.frame = requestAnimationFrame(follow); + }; + const start = () => { + if (signal.aborted) return; + + // after seek(): from about where the word falls in the piece + if (startShare > 0 && Number.isFinite(audio.duration)) { + audio.currentTime = audio.duration * startShare; + } + + audio.play().then( + () => { + if (!signal.aborted) this.frame = requestAnimationFrame(follow); + }, + () => end() + ); + }; + const end = () => { + audio.removeEventListener('loadedmetadata', start); + audio.onended = null; + audio.onerror = null; + signal.removeEventListener('abort', end); + resolve(); + }; + + audio.onended = end; + audio.onerror = end; + signal.addEventListener('abort', end); + audio.src = url; + // set after src: loading a source resets the rate + audio.preservesPitch = true; + audio.playbackRate = this.options.playbackRate?.() ?? 1; + audio.defaultPlaybackRate = audio.playbackRate; + + if (startShare > 0) { + audio.addEventListener('loadedmetadata', start, { once: true }); + } else { + start(); + } + }); + } finally { + if (this.frame !== null) cancelAnimationFrame(this.frame); + + this.frame = null; + URL.revokeObjectURL(url); + } + } + + private async pump(): Promise { + if (this.playing) return; + + const piece = this.queue.shift(); + + if (!piece) { + this.finishIfDone(); + + return; + } + + const { signal } = this.run; + const startShare = this.startAt?.piece === piece ? this.startAt.share : 0; + + this.startAt = null; + this.playing = true; + this.speaking = true; + + if (piece.speech && !this.isMuted) { + // the next pieces are fetched while this one plays + for (const next of this.queue.slice(0, 2)) { + if (next.speech) this.fetchPiece(next); + } + + try { + const blob = await this.fetchPiece(piece); + + // paused while fetching: the piece waits to be played + while (this.isPaused && !signal.aborted) { + await new Promise((resolve) => this.resumeWaiters.push(resolve)); + } + + if (!signal.aborted && !this.isMuted) await this.play(piece, blob, signal, startShare); + } catch (error) { + if (!signal.aborted) console.warn('Reading the reply aloud failed:', error); + } + } + + if (signal.aborted) return; + + this.spoken = Math.max(this.spoken, piece.end); + this.playing = false; + void this.pump(); + } + + /** Muting mid-piece: the voice stops, the text keeps up. */ + private silence(): void { + this.stopAudio(); + this.spoken = Math.max(this.spoken, this.queuedTo); + } + + private stopAudio(): void { + if (this.frame !== null) cancelAnimationFrame(this.frame); + + this.frame = null; + + const audio = this.audio; + + if (audio && !audio.paused) audio.pause(); + + audio?.dispatchEvent(new Event('ended')); + } +} diff --git a/ui/apps/chat/src/lib/voice/speech.ts b/ui/apps/chat/src/lib/voice/speech.ts new file mode 100644 index 000000000000..0cf584965550 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/speech.ts @@ -0,0 +1,49 @@ +import { VOICE_API_PATHS } from './constants'; +import { VoiceTranscriptionError } from './errors'; +import type { VoiceServerContext } from './types'; + +export interface SynthesizeSpeechOptions { + /** Model id, required by a router server */ + model?: string; + /** One of the model's voices (`voices` of GET /props); left out, its default */ + voice?: string; + signal?: AbortSignal; +} + +/** + * Speaks `text` with llama-server's POST /v1/audio/speech and returns the WAV. + * @throws VoiceTranscriptionError with the server's message when it refuses + */ +export async function synthesizeSpeech( + context: VoiceServerContext, + text: string, + options: SynthesizeSpeechOptions = {} +): Promise { + const response = await (context.fetch ?? fetch)(context.resolveUrl(VOICE_API_PATHS.SPEECH), { + body: JSON.stringify({ + input: text, + response_format: 'wav', + ...(options.model ? { model: options.model } : {}), + ...(options.voice ? { voice: options.voice } : {}) + }), + headers: { ...context.getHeaders?.(), 'Content-Type': 'application/json' }, + method: 'POST', + signal: options.signal + }); + + if (!response.ok) { + let message = `Speech failed (${response.status})`; + + try { + const data = await response.json(); + + if (typeof data?.error?.message === 'string') message = data.error.message; + } catch { + // not JSON: keep the status + } + + throw new VoiceTranscriptionError(message, response.status); + } + + return await response.blob(); +} diff --git a/ui/apps/chat/src/lib/voice/trim-silence.ts b/ui/apps/chat/src/lib/voice/trim-silence.ts new file mode 100644 index 000000000000..83bc7ed93647 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/trim-silence.ts @@ -0,0 +1,83 @@ +import { encodeWav } from './wav'; + +/** Silence kept before the voice starts and after it ends, so pieces still breathe. */ +const LEAD_MS = 40; +const TAIL_MS = 90; +/** A sample under this share of the clip's loudest one is silence. */ +const SILENCE_SHARE = 0.03; + +/** + * A WAV with the silence a speech model puts around what it says cut down to a short + * breath, so pieces read one after another (a sentence, then the paragraph after an + * empty line) follow without long gaps. Anything but 16-bit PCM is returned as it is. + */ +export async function trimSilence(blob: Blob): Promise { + const bytes = await blob.arrayBuffer(); + const pcm = readPcm16(bytes); + + if (!pcm || pcm.channels !== 1) return blob; + + const { sampleRate, samples } = pcm; + + let loudest = 0; + + for (let i = 0; i < samples.length; i++) loudest = Math.max(loudest, Math.abs(samples[i])); + + if (loudest === 0) return blob; + + const threshold = loudest * SILENCE_SHARE; + + let first = 0; + let last = samples.length - 1; + + while (first < samples.length && Math.abs(samples[first]) < threshold) first++; + while (last > first && Math.abs(samples[last]) < threshold) last--; + + const start = Math.max(0, first - Math.round((sampleRate * LEAD_MS) / 1000)); + const end = Math.min(samples.length, last + 1 + Math.round((sampleRate * TAIL_MS) / 1000)); + + if (start === 0 && end === samples.length) return blob; + + return encodeWav(samples.slice(start, end), sampleRate); +} + +/** The samples of a 16-bit PCM WAV, or null for any other format. */ +function readPcm16( + bytes: ArrayBuffer +): { channels: number; sampleRate: number; samples: Int16Array } | null { + const view = new DataView(bytes); + const tag = (offset: number) => + String.fromCharCode(...new Uint8Array(bytes, offset, Math.min(4, bytes.byteLength - offset))); + + if (bytes.byteLength < 12 || tag(0) !== 'RIFF' || tag(8) !== 'WAVE') return null; + + let channels = 0; + let sampleRate = 0; + let bits = 0; + let offset = 12; + + while (offset + 8 <= bytes.byteLength) { + const id = tag(offset); + // a streamed WAV may leave the data size unknown (0xFFFFFFFF): it runs to the end + const size = Math.min(view.getUint32(offset + 4, true), bytes.byteLength - offset - 8); + const body = offset + 8; + + if (id === 'fmt ') { + if (view.getUint16(body, true) !== 1) return null; + + channels = view.getUint16(body + 2, true); + sampleRate = view.getUint32(body + 4, true); + bits = view.getUint16(body + 14, true); + } else if (id === 'data') { + if (bits !== 16 || !sampleRate) return null; + + const samples = new Int16Array(bytes.slice(body, body + size - (size % 2))); + + return { channels, sampleRate, samples }; + } + + offset = body + size + (size % 2); + } + + return null; +} diff --git a/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts b/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts index 395bde627cc5..058c7e02ed98 100644 --- a/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts +++ b/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts @@ -2,11 +2,15 @@ // what the microphone offers per model, the recording row, dictated text written into the // message around what the user types, and a voice message whose transcript comes later. -import { modelServerProps, seedStores } from '../stories/chat/form/fixtures'; +import { modelServerProps, routerModels, seedStores } from '../stories/chat/form/fixtures'; import { deferred, FakeVoiceRecorder } from '../stories/voice/fake-voice-recorder.svelte'; import ChatFormVoiceTestWrapper from './components/ChatFormVoiceTestWrapper.svelte'; import { EditorView } from '@codemirror/view'; -import { SETTINGS_KEYS, VOICE_MODE_MESSAGE } from '#lib/constants/index.ts'; +import { + SETTINGS_KEYS, + VOICE_CONVERSATION_MESSAGE, + VOICE_MODE_MESSAGE +} from '#lib/constants/index.ts'; import { ServerRole, VoiceInputMode } from '#lib/enums/index.ts'; import { voiceTranscriptsStore } from '#lib/stores/index.ts'; import { parseFilesToMessageExtras } from '#lib/utils/browser-only.ts'; @@ -39,10 +43,37 @@ function seed( }); } -async function renderForm({ isLoading = false } = {}) { +/** A router whose chat model (Qwen3) can't hear while another (Gemma) can, none of them loaded. */ +function seedRouter(transcriptionModel = '') { + const [chat, hearing] = routerModels; + + restore = seedStores({ + config: { + autoMicOnEmpty: true, + [SETTINGS_KEYS.VOICE_INPUT_MODE]: VoiceInputMode.AUTO, + [SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL]: transcriptionModel + }, + modelProps: { + [chat.model]: modelServerProps(), + [hearing.model]: modelServerProps({ + modalities: { audio: true, video: false, vision: false } + }) + }, + models: routerModels, + selectedModelId: chat.id, + serverRole: ServerRole.ROUTER + }); +} + +async function renderForm(options: { isLoading?: boolean; voiceMode?: boolean } = {}) { const recorder = new FakeVoiceRecorder(); const state = $state({ files: [] as ChatUploadedFile[], sent: 0, value: '' }); - const screen = render(ChatFormVoiceTestWrapper, { isLoading, state, voiceRecorder: recorder }); + const screen = render(ChatFormVoiceTestWrapper, { + isLoading: options.isLoading, + state, + voiceMode: options.voiceMode, + voiceRecorder: recorder + }); await tick(); @@ -141,6 +172,40 @@ describe('the microphone', () => { }); }); +describe('voice mode', () => { + it('is the main button when the model hears', async () => { + seed(true); + + const { screen } = await renderForm({ voiceMode: true }); + + await expect + .element(screen.getByRole('button', { name: VOICE_CONVERSATION_MESSAGE.open }).last()) + .toBeInTheDocument(); + }); + + it('leaves the lead to the microphone when nothing can transcribe what is said', async () => { + seedRouter(); + + const { screen } = await renderForm({ voiceMode: true }); + + // the microphone stays, saying why it can't record + await expect + .element(screen.getByRole('button', { name: VOICE_MODE_MESSAGE.unavailable }).last()) + .toBeInTheDocument(); + expect(screen.container.innerHTML).not.toContain(VOICE_CONVERSATION_MESSAGE.open); + }); + + it('is the main button again once a transcription model is chosen', async () => { + seedRouter(routerModels[1].model); + + const { screen } = await renderForm({ voiceMode: true }); + + await expect + .element(screen.getByRole('button', { name: VOICE_CONVERSATION_MESSAGE.open }).last()) + .toBeInTheDocument(); + }); +}); + describe('recording', () => { it('asks for the raw microphone, and for the processing that is switched on', async () => { seed(true, VoiceInputMode.DICTATION); diff --git a/ui/apps/chat/tests/client/components/ChatFormVoiceTestWrapper.svelte b/ui/apps/chat/tests/client/components/ChatFormVoiceTestWrapper.svelte index 34dfe4ce77ef..a286a23b6689 100644 --- a/ui/apps/chat/tests/client/components/ChatFormVoiceTestWrapper.svelte +++ b/ui/apps/chat/tests/client/components/ChatFormVoiceTestWrapper.svelte @@ -9,9 +9,11 @@ /** The form's message and files, kept in step with it for the test to read, and how often it sent. */ state: { files: ChatUploadedFile[]; sent?: number; value: string }; voiceRecorder: VoiceRecorderLike; + /** Offer voice mode, as the chat screen does. */ + voiceMode?: boolean; } - let { isLoading = false, state, voiceRecorder }: Props = $props(); + let { isLoading = false, state, voiceMode = false, voiceRecorder }: Props = $props(); @@ -20,6 +22,7 @@ bind:value={state.value} {isLoading} onSubmit={() => (state.sent = (state.sent ?? 0) + 1)} + {voiceMode} {voiceRecorder} /> diff --git a/ui/apps/chat/tests/client/voice-reply-reader.svelte.test.ts b/ui/apps/chat/tests/client/voice-reply-reader.svelte.test.ts new file mode 100644 index 000000000000..3149b5bbccd3 --- /dev/null +++ b/ui/apps/chat/tests/client/voice-reply-reader.svelte.test.ts @@ -0,0 +1,163 @@ +// Guards reading a reply aloud from a word the user picks: the voice moves there, back or +// ahead, plays audio it already fetched without asking for it again, and starts inside a +// sentence at about where the word falls in it. + +import { ReplyReader } from '#lib/voice/reply-reader.svelte.ts'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; + +const DURATION = 2; + +/** An audio element that "plays" a clip of DURATION seconds until finish() is called. */ +class FakeAudio extends EventTarget { + currentTime = 0; + defaultPlaybackRate = 1; + duration = Number.NaN; + static last: FakeAudio | null = null; + onended: (() => void) | null = null; + onerror: (() => void) | null = null; + paused = true; + playbackRate = 1; + preservesPitch = true; + /** What each play() started on: the clip and where in it */ + starts: { src: string; time: number }[] = []; + private source = ''; + + get src() { + return this.source; + } + + set src(value: string) { + this.source = value; + this.currentTime = 0; + this.duration = Number.NaN; + // metadata comes after the source is set, as in a browser + queueMicrotask(() => { + this.duration = DURATION; + this.dispatchEvent(new Event('loadedmetadata')); + }); + } + + constructor() { + super(); + FakeAudio.last = this; + } + + override dispatchEvent(event: Event): boolean { + if (event.type === 'ended') this.onended?.(); + + return super.dispatchEvent(event); + } + + /** The clip plays to its end. */ + finish() { + this.currentTime = this.duration; + this.paused = true; + this.onended?.(); + } + + pause() { + this.paused = true; + } + + play() { + this.paused = false; + this.starts.push({ src: this.source, time: this.currentTime }); + + return Promise.resolve(); + } +} + +const REPLY = 'One two three. Four five six. Seven eight nine.'; +const at = (word: string) => REPLY.indexOf(word); +const settle = () => new Promise((resolve) => setTimeout(resolve, 20)); + +function reader() { + const synthesize = vi.fn(async (text: string) => new Blob([text], { type: 'audio/test' })); + const replyReader = new ReplyReader({ synthesize }); + + replyReader.reset(); + replyReader.update(REPLY, true); + + return { replyReader, synthesize }; +} + +/** The text of the clip that last started, and where in it. */ +function lastStart() { + const start = FakeAudio.last!.starts.at(-1)!; + + return { text: clips.get(start.src), time: start.time }; +} + +const clips = new Map(); + +beforeAll(() => { + vi.stubGlobal('Audio', FakeAudio); + vi.spyOn(URL, 'createObjectURL').mockImplementation((blob) => { + const url = `blob:test/${clips.size}`; + + void (blob as Blob).text().then((text) => clips.set(url, text)); + + return url; + }); + vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}); +}); + +afterAll(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe('ReplyReader.seek', () => { + it('jumps ahead to a later sentence', async () => { + const { replyReader, synthesize } = reader(); + + await settle(); + expect(lastStart().text).toBe('One two three.'); + + expect(replyReader.seek(at('Seven'))).toBe(true); + expect(replyReader.spoken).toBe(at('Seven')); + await settle(); + + expect(lastStart()).toEqual({ text: 'Seven eight nine.', time: 0 }); + // the later sentences were fetched while the first played: nothing is asked for twice + expect(synthesize).toHaveBeenCalledTimes(3); + replyReader.stop(); + }); + + it('goes back into a sentence already read, from about where the word is', async () => { + const { replyReader, synthesize } = reader(); + + await settle(); + FakeAudio.last!.finish(); + await settle(); + expect(lastStart().text).toBe('Four five six.'); + + expect(replyReader.seek(at('three'))).toBe(true); + expect(replyReader.spoken).toBe(at('three')); + await settle(); + + const { text, time } = lastStart(); + + expect(text).toBe('One two three.'); + expect(time).toBeGreaterThan(0.8); + expect(time).toBeLessThan(DURATION); + expect(synthesize).toHaveBeenCalledTimes(3); + + // and reads on from there + FakeAudio.last!.finish(); + await settle(); + expect(lastStart()).toEqual({ text: 'Four five six.', time: 0 }); + replyReader.stop(); + }); + + it('has nowhere to go once muted or read to the end', async () => { + const { replyReader } = reader(); + + replyReader.muted = true; + expect(replyReader.seek(at('Four'))).toBe(false); + + replyReader.muted = false; + replyReader.stop(); + expect(replyReader.seek(at('Four'))).toBe(false); + }); +}); diff --git a/ui/apps/chat/tests/client/voice-spoken-text.svelte.test.ts b/ui/apps/chat/tests/client/voice-spoken-text.svelte.test.ts new file mode 100644 index 000000000000..d78a69f1e35a --- /dev/null +++ b/ui/apps/chat/tests/client/voice-spoken-text.svelte.test.ts @@ -0,0 +1,26 @@ +// Guards picking a word of a reply being read aloud: a click gives where the word starts, +// and without a listener the words are plain text. + +import { VoiceSpokenText } from '@ggml-org/llama-ui/voice'; +import { describe, expect, it, vi } from 'vitest'; +import { page, userEvent } from 'vitest/browser'; +import { render } from 'vitest-browser-svelte'; + +const TEXT = 'Read this. Then that.'; + +describe('VoiceSpokenText word picks', () => { + it('gives where a clicked word starts', async () => { + const onWordSelect = vi.fn(); + + render(VoiceSpokenText, { onWordSelect, spoken: 4, text: TEXT }); + + await userEvent.click(page.getByText('Then')); + expect(onWordSelect).toHaveBeenCalledWith(TEXT.indexOf(' Then') + 1); + }); + + it('leaves the words alone without a listener', () => { + const { container } = render(VoiceSpokenText, { spoken: 4, text: TEXT }); + + expect(container.querySelector('[data-start]')).toBeNull(); + }); +}); diff --git a/ui/apps/chat/tests/unit/voice-mode.test.ts b/ui/apps/chat/tests/unit/voice-mode.test.ts index 7d2f09225f66..a5c39085f901 100644 --- a/ui/apps/chat/tests/unit/voice-mode.test.ts +++ b/ui/apps/chat/tests/unit/voice-mode.test.ts @@ -158,6 +158,7 @@ describe('voice settings', () => { SETTINGS_KEYS.VOICE_NOISE_SUPPRESSION, SETTINGS_KEYS.VOICE_ECHO_CANCELLATION, SETTINGS_KEYS.VOICE_AUTO_GAIN, + SETTINGS_KEYS.VOICE_READING_SPEED, SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE ]); expect(section?.fields?.[0].type).toBe(SettingsFieldType.SELECT); diff --git a/ui/apps/chat/tests/unit/voice-pauses.test.ts b/ui/apps/chat/tests/unit/voice-pauses.test.ts index 9b2bfeda600b..188288958c6e 100644 --- a/ui/apps/chat/tests/unit/voice-pauses.test.ts +++ b/ui/apps/chat/tests/unit/voice-pauses.test.ts @@ -42,6 +42,19 @@ describe('PauseDetector', () => { expect(hold(detector, 0.45, 600)).toBe(true); }); + it('hears speech from the first level when the room is known', () => { + // without a known floor, the first level is taken as the room's noise + const unknown = new PauseDetector({ intervalMs: INTERVAL }); + + unknown.push(0.6); + expect(unknown.heardSpeech).toBe(false); + + const known = new PauseDetector({ floor: 0.1, intervalMs: INTERVAL }); + + known.push(0.6); + expect(known.heardSpeech).toBe(true); + }); + it('cuts a long stretch of speech at its next quiet moment', () => { const detector = new PauseDetector({ intervalMs: INTERVAL, maxSegmentMs: 2000 }); diff --git a/ui/apps/chat/tests/unit/voice-read-aloud.test.ts b/ui/apps/chat/tests/unit/voice-read-aloud.test.ts new file mode 100644 index 000000000000..1df51a68a375 --- /dev/null +++ b/ui/apps/chat/tests/unit/voice-read-aloud.test.ts @@ -0,0 +1,59 @@ +// Guards how voice mode reads a streaming reply aloud: whole sentences as soon as they are +// complete, the rest once the reply is done, without markdown marks, with offsets into the +// reply as written so the text on screen can follow the voice. + +import { readAloudPieces, toSpeech } from '#lib/voice/index.ts'; +import { describe, expect, it } from 'vitest'; + +describe('readAloudPieces', () => { + it('reads complete sentences while the reply streams', () => { + const text = 'Sure, here you go. The first one is'; + const pieces = readAloudPieces(text, 0, false); + + expect(pieces.map((piece) => piece.speech)).toEqual(['Sure, here you go.']); + expect(text.slice(pieces[0].start, pieces[0].end)).toBe('Sure, here you go.'); + }); + + it('reads the rest once the reply is done, from where it left off', () => { + const text = 'Sure, here you go. The first one is short'; + const [first] = readAloudPieces(text, 0, false); + const rest = readAloudPieces(text, first.end, true); + + expect(rest.map((piece) => piece.speech)).toEqual(['The first one is short']); + expect(rest[0].end).toBe(text.length); + }); + + it('does not end a sentence at a list number, and reads lines on their own', () => { + const text = 'Jokes:\n1. Why are fish clever? They live in schools.\n2. Next'; + // a bare line break is a piece with nothing to say: it only moves the reading on + const pieces = readAloudPieces(text, 0, true) + .map((piece) => piece.speech) + .filter(Boolean); + + expect(pieces).toEqual(['Jokes:', 'Why are fish clever?', 'They live in schools.', 'Next']); + }); +}); + +describe('the first piece', () => { + it('may end at a clause, so the voice starts before the sentence is done', () => { + const text = 'Quantum physics describes how very small things, like electrons and photons'; + const [first] = readAloudPieces(text, 0, false, true); + + expect(first.speech).toBe('Quantum physics describes how very small things,'); + // later pieces wait for whole sentences + expect(readAloudPieces(text, first.end, false)).toEqual([]); + }); + + it('waits for the sentence when its first clause is short', () => { + expect(readAloudPieces('Sure, here are ten jokes about', 0, false, true)).toEqual([]); + }); +}); + +describe('toSpeech', () => { + it('leaves out markdown marks', () => { + expect(toSpeech('## **Quantization.** Some `values`, see [the docs](https://x.y).')).toBe( + 'Quantization. Some values, see the docs.' + ); + expect(toSpeech('```js\nconst a = 1;\n```')).toBe(''); + }); +}); diff --git a/ui/apps/chat/tests/unit/voice-trim-silence.test.ts b/ui/apps/chat/tests/unit/voice-trim-silence.test.ts new file mode 100644 index 000000000000..5cc4cb98cc13 --- /dev/null +++ b/ui/apps/chat/tests/unit/voice-trim-silence.test.ts @@ -0,0 +1,46 @@ +// Guards the silence cut from read-aloud clips: the speech model's quiet lead and tail +// shrink to a short breath, so pieces follow each other without long gaps. + +import { encodeWav } from '#lib/voice/index.ts'; +import { trimSilence } from '#lib/voice/trim-silence.ts'; +import { describe, expect, it } from 'vitest'; + +const RATE = 24_000; + +/** `silenceMs` of silence, `toneMs` of a tone, then `silenceMs` of silence again. */ +function clip(silenceMs: number, toneMs: number): Blob { + const silence = Math.round((RATE * silenceMs) / 1000); + const tone = Math.round((RATE * toneMs) / 1000); + const samples = new Int16Array(silence * 2 + tone); + + for (let i = 0; i < tone; i++) { + samples[silence + i] = Math.round(Math.sin((2 * Math.PI * 440 * i) / RATE) * 12_000); + } + + return encodeWav(samples, RATE); +} + +/** Length of a 16-bit mono WAV in ms. */ +async function lengthMs(blob: Blob): Promise { + return ((blob.size - 44) / 2 / RATE) * 1000; +} + +describe('trimSilence', () => { + it('cuts the silence around the voice down to a short breath', async () => { + const trimmed = await trimSilence(clip(300, 1000)); + + // 1000 ms of tone, with 40 ms kept before and 90 ms after it + expect(await lengthMs(trimmed)).toBeGreaterThan(1100); + expect(await lengthMs(trimmed)).toBeLessThan(1160); + }); + + it('leaves a clip without silence, a silent one and anything not WAV alone', async () => { + const tight = clip(0, 500); + const silent = encodeWav(new Int16Array(RATE), RATE); + const other = new Blob(['not a wav'], { type: 'audio/mpeg' }); + + expect(await trimSilence(tight)).toBe(tight); + expect(await trimSilence(silent)).toBe(silent); + expect(await trimSilence(other)).toBe(other); + }); +}); diff --git a/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte b/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte index f9299cc1eac1..9e2aa8e31592 100644 --- a/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte +++ b/ui/lib/ui/src/lib/components/voice/VoiceSpokenText.svelte @@ -3,6 +3,7 @@ muted. Used for a transcript followed during playback (`spoken`, e.g. from spokenLength()), for a reply read aloud, and for text that streams in while someone speaks (`live`: each new word brightens as it arrives). Without `spoken` or `live` all of it reads as spoken. The words are plain text for screen readers. + With `onWordSelect`, a click or tap on a word gives where it starts, e.g. to read on from there. --> - + +{#each words as { start, word }, index (live ? index : start)}{word}{word}{/each}