From f8486a94af27e9e080c1d1ed98c75e64c91caf02 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Sun, 15 Nov 2026 21:02:51 +0100 Subject: [PATCH 01/19] ui: add a voice engine for capture, peaks and transcription Voice input needs a live waveform, a clip that can be sent and drawn, and a transcript, whether the model streams text while the user speaks or not. This adds the engine for it in src/lib/voice, with no imports of app stores or components, so it can move into a package later. - VoiceCapture: one microphone start gives live levels (RMS on a dB curve, fixed history, newest last), PCM16 mono chunks at 16 kHz from an inline AudioWorklet (ScriptProcessor fallback), and on stop a 16-bit mono WAV clip with its duration and peaks. Tracks, nodes and the context are released on stop, cancel, a failed start and page hide. - computePeaks / computePeaksFromAudio: normalised peaks of samples, a buffer, or a stored base64 or Blob file. - transcribeClip: POST /v1/audio/transcriptions, with the server's own error message on failure. - RealtimeTranscriber: the /v1/realtime?intent=transcription socket, queueing audio until session.created, a final commit, and a typed RealtimeUnsupportedError when the model cannot stream or the socket is refused (a browser websocket cannot send the API key header). - startVoiceSession and VoiceRecorder: try realtime, fall back to one HTTP transcription of the clip, and expose levels and live text. Assisted-by: Claude (Opus 5.5 Medium) --- ui/apps/chat/src/lib/voice/capture.ts | 297 +++++++++++++++ ui/apps/chat/src/lib/voice/constants.ts | 83 +++++ ui/apps/chat/src/lib/voice/errors.ts | 41 ++ ui/apps/chat/src/lib/voice/index.ts | 32 ++ ui/apps/chat/src/lib/voice/levels.ts | 49 +++ ui/apps/chat/src/lib/voice/pcm-tap.ts | 148 ++++++++ ui/apps/chat/src/lib/voice/pcm.ts | 140 +++++++ ui/apps/chat/src/lib/voice/peaks.ts | 75 ++++ ui/apps/chat/src/lib/voice/realtime.ts | 350 ++++++++++++++++++ ui/apps/chat/src/lib/voice/session.ts | 132 +++++++ ui/apps/chat/src/lib/voice/transcribe.ts | 83 +++++ ui/apps/chat/src/lib/voice/types.ts | 122 ++++++ .../src/lib/voice/voice-recorder.svelte.ts | 99 +++++ ui/apps/chat/src/lib/voice/wav.ts | 36 ++ .../tests/client/voice-capture.svelte.test.ts | 163 ++++++++ ui/apps/chat/tests/unit/voice-audio.test.ts | 197 ++++++++++ .../chat/tests/unit/voice-realtime.test.ts | 321 ++++++++++++++++ ui/apps/chat/tests/unit/voice-session.test.ts | 215 +++++++++++ .../chat/tests/unit/voice-transcribe.test.ts | 116 ++++++ 19 files changed, 2699 insertions(+) create mode 100644 ui/apps/chat/src/lib/voice/capture.ts create mode 100644 ui/apps/chat/src/lib/voice/constants.ts create mode 100644 ui/apps/chat/src/lib/voice/errors.ts create mode 100644 ui/apps/chat/src/lib/voice/index.ts create mode 100644 ui/apps/chat/src/lib/voice/levels.ts create mode 100644 ui/apps/chat/src/lib/voice/pcm-tap.ts create mode 100644 ui/apps/chat/src/lib/voice/pcm.ts create mode 100644 ui/apps/chat/src/lib/voice/peaks.ts create mode 100644 ui/apps/chat/src/lib/voice/realtime.ts create mode 100644 ui/apps/chat/src/lib/voice/session.ts create mode 100644 ui/apps/chat/src/lib/voice/transcribe.ts create mode 100644 ui/apps/chat/src/lib/voice/types.ts create mode 100644 ui/apps/chat/src/lib/voice/voice-recorder.svelte.ts create mode 100644 ui/apps/chat/src/lib/voice/wav.ts create mode 100644 ui/apps/chat/tests/client/voice-capture.svelte.test.ts create mode 100644 ui/apps/chat/tests/unit/voice-audio.test.ts create mode 100644 ui/apps/chat/tests/unit/voice-realtime.test.ts create mode 100644 ui/apps/chat/tests/unit/voice-session.test.ts create mode 100644 ui/apps/chat/tests/unit/voice-transcribe.test.ts diff --git a/ui/apps/chat/src/lib/voice/capture.ts b/ui/apps/chat/src/lib/voice/capture.ts new file mode 100644 index 000000000000..87da54e8e4a8 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/capture.ts @@ -0,0 +1,297 @@ +import { + VOICE_ANALYSER_FFT_SIZE, + VOICE_CHUNK_MS, + VOICE_LEVELS_INTERVAL_MS, + VOICE_LEVELS_LENGTH, + VOICE_MICROPHONE_CONSTRAINTS, + VOICE_PEAKS_COUNT, + VOICE_SAMPLE_RATE +} from './constants'; +import { VoiceCaptureError } from './errors'; +import { LevelHistory, rms, rmsToLevel } from './levels'; +import { concatPcm16, createResampler, floatToPcm16, type Resampler } from './pcm'; +import { createPcmTap, type PcmTap } from './pcm-tap'; +import { computePeaks } from './peaks'; +import type { + VoiceCaptureOptions, + VoiceCaptureState, + VoiceClip, + VoiceLevelsListener, + VoicePcmListener +} from './types'; +import { encodeWav } from './wav'; +import { describeMicrophoneError } from '#lib/utils/audio-recording.ts'; + +/** Graph and device handles of a running capture, released together. */ +interface CaptureGraph { + analyser: AnalyserNode; + context: AudioContext; + mute: GainNode; + source: MediaStreamAudioSourceNode; + stream: MediaStream; + tap: PcmTap; +} + +/** + * Microphone capture for voice input. One `start()` gives, at once: a live level + * history for the waveform, PCM16 mono chunks for streaming, and the recording, + * returned by `stop()` as a WAV clip with its peaks. Every path (stop, cancel, a + * failed start, the page going away) releases the tracks, the nodes and the context. + */ +export class VoiceCapture { + readonly sampleRate: number; + + private chunks: Int16Array[] = []; + private readonly chunkSamples: number; + private currentState: VoiceCaptureState = 'idle'; + private frame: number | null = null; + // bumped by every start, stop and cancel, so a start still awaiting the microphone can tell it was dropped + private generation = 0; + + private graph: CaptureGraph | null = null; + private handlePageHide = () => this.cancel(); + private readonly history: LevelHistory; + private levelListeners = new Set(); + private readonly levelsIntervalMs: number; + private readonly options: VoiceCaptureOptions; + private partial: number[] = []; + private pcmListeners = new Set(); + private readonly peaksCount: number; + private recordedSamples = 0; + + private resampler: Resampler | null = null; + + /** Length of the audio captured so far. */ + get durationMs(): number { + return ((this.recordedSamples + this.partial.length) / this.sampleRate) * 1000; + } + + /** The level history now, 0..1, newest last. */ + get levels(): number[] { + return this.history.snapshot(); + } + + get state(): VoiceCaptureState { + return this.currentState; + } + + /** Drops the capture, including one still asking for the microphone. */ + cancel(): void { + this.generation++; + this.teardown(); + this.chunks = []; + this.partial = []; + this.recordedSamples = 0; + this.currentState = 'idle'; + } + + constructor(options: VoiceCaptureOptions = {}) { + this.options = options; + this.sampleRate = options.sampleRate ?? VOICE_SAMPLE_RATE; + this.chunkSamples = Math.max( + 1, + Math.round((this.sampleRate * (options.chunkMs ?? VOICE_CHUNK_MS)) / 1000) + ); + this.history = new LevelHistory(options.levelsLength ?? VOICE_LEVELS_LENGTH); + this.levelsIntervalMs = options.levelsIntervalMs ?? VOICE_LEVELS_INTERVAL_MS; + this.peaksCount = options.peaksCount ?? VOICE_PEAKS_COUNT; + } + + /** Listens to the level history; returns the unsubscribe function. */ + onLevels(listener: VoiceLevelsListener): () => void { + this.levelListeners.add(listener); + + return () => this.levelListeners.delete(listener); + } + + /** Listens to the PCM16 chunks; returns the unsubscribe function. */ + onPcm(listener: VoicePcmListener): () => void { + this.pcmListeners.add(listener); + + return () => this.pcmListeners.delete(listener); + } + + /** + * Asks for the microphone and starts capturing. + * @throws VoiceCaptureError with a message for the user when the microphone does not start + */ + async start(): Promise { + if (this.currentState === 'starting' || this.currentState === 'recording') { + throw new Error('Voice capture already started'); + } + + const generation = ++this.generation; + const dropped = () => generation !== this.generation; + + this.currentState = 'starting'; + this.chunks = []; + this.partial = []; + this.recordedSamples = 0; + this.history.reset(); + + let stream: MediaStream | null = this.options.stream ?? null; + let context: AudioContext | null = null; + let started = false; + + try { + stream ??= await navigator.mediaDevices.getUserMedia({ + audio: { ...VOICE_MICROPHONE_CONSTRAINTS, ...this.options.constraints } + }); + + if (dropped()) throw new DOMException('Voice capture cancelled', 'AbortError'); + + context = new AudioContext(); + + // a context made outside a user gesture starts suspended; resume() resolves once allowed + if (context.state === 'suspended') context.resume().catch(() => {}); + + const source = context.createMediaStreamSource(stream); + const analyser = context.createAnalyser(); + const mute = context.createGain(); + + analyser.fftSize = VOICE_ANALYSER_FFT_SIZE; + mute.gain.value = 0; + this.resampler = createResampler(context.sampleRate, this.sampleRate); + + const tap = await createPcmTap(context, (block) => this.handleBlock(block)); + + if (dropped()) { + tap.disconnect(); + + throw new DOMException('Voice capture cancelled', 'AbortError'); + } + + // the tap only runs while it is pulled by the destination, muted so nothing is heard + source.connect(analyser); + source.connect(tap.node); + tap.node.connect(mute); + mute.connect(context.destination); + + this.graph = { analyser, context, mute, source, stream, tap }; + started = true; + this.currentState = 'recording'; + this.watchLevels(analyser); + addEventListener('pagehide', this.handlePageHide); + } catch (error) { + if (!started) releaseDevice(stream, context); + + if (!dropped()) { + this.currentState = 'idle'; + } + + if (error instanceof DOMException && error.name === 'AbortError' && dropped()) throw error; + + throw new VoiceCaptureError(describeMicrophoneError(error), { cause: error }); + } + } + + /** Ends the capture and returns the recording. */ + async stop(): Promise { + const graph = this.graph; + + if (!graph || this.currentState !== 'recording') { + throw new Error('No voice capture to stop'); + } + + this.generation++; + await graph.tap.flush(); + + if (this.graph !== graph) throw new Error('Voice capture cancelled'); + + this.flushPartial(); + this.teardown(); + + const samples = concatPcm16(this.chunks); + + this.chunks = []; + this.currentState = 'stopped'; + + return { + blob: encodeWav(samples, this.sampleRate), + durationMs: (samples.length / this.sampleRate) * 1000, + peaks: computePeaks(samples, this.peaksCount), + sampleRate: this.sampleRate + }; + } + + private flushPartial(): void { + if (this.partial.length === 0) return; + + const chunk = floatToPcm16(Float32Array.from(this.partial)); + + this.partial = []; + this.chunks.push(chunk); + this.recordedSamples += chunk.length; + + for (const listener of this.pcmListeners) listener(chunk); + } + + private handleBlock(block: Float32Array): void { + if (!this.resampler || this.currentState !== 'recording') return; + + const resampled = this.resampler.process(block); + + for (let i = 0; i < resampled.length; i++) { + this.partial.push(resampled[i]); + + if (this.partial.length === this.chunkSamples) this.flushPartial(); + } + } + + private teardown(): void { + removeEventListener('pagehide', this.handlePageHide); + + if (this.frame !== null) { + cancelAnimationFrame(this.frame); + this.frame = null; + } + + const graph = this.graph; + + this.graph = null; + this.resampler = null; + + if (!graph) return; + + graph.tap.disconnect(); + graph.source.disconnect(); + graph.analyser.disconnect(); + graph.mute.disconnect(); + releaseDevice(graph.stream, graph.context); + } + + private watchLevels(analyser: AnalyserNode): void { + const buffer = new Float32Array(analyser.fftSize); + + let loudest = 0; + let last = performance.now(); + + const tick = (now: number) => { + if (!this.graph) return; + + analyser.getFloatTimeDomainData(buffer); + loudest = Math.max(loudest, rmsToLevel(rms(buffer))); + + if (now - last >= this.levelsIntervalMs) { + const levels = this.history.push(loudest); + + loudest = 0; + last = now; + + for (const listener of this.levelListeners) listener(levels); + } + + this.frame = requestAnimationFrame(tick); + }; + + this.frame = requestAnimationFrame(tick); + } +} + +function releaseDevice(stream: MediaStream | null, context: AudioContext | null): void { + for (const track of stream?.getTracks() ?? []) track.stop(); + + if (context && context.state !== 'closed') { + context.close().catch(() => {}); + } +} diff --git a/ui/apps/chat/src/lib/voice/constants.ts b/ui/apps/chat/src/lib/voice/constants.ts new file mode 100644 index 000000000000..ea40bdb39e9b --- /dev/null +++ b/ui/apps/chat/src/lib/voice/constants.ts @@ -0,0 +1,83 @@ +/** + * Voice engine defaults. Kept next to the engine (not in `#lib/constants/index.ts`) so the + * folder can move into its own package without pulling the app's constants along. + */ + +/** Rate of the PCM chunks and of the recorded WAV, what speech encoders expect. */ +export const VOICE_SAMPLE_RATE = 16_000; + +/** Length of the PCM chunks handed to the stream. */ +export const VOICE_CHUNK_MS = 100; + +/** Number of values in the live level history, newest last. */ +export const VOICE_LEVELS_LENGTH = 48; + +/** Time between two values of the live level history. */ +export const VOICE_LEVELS_INTERVAL_MS = 50; + +/** Level floor in dBFS: quieter input reads as 0. */ +export const VOICE_LEVELS_FLOOR_DB = -60; + +/** Analyser window, about 20 ms at 48 kHz. */ +export const VOICE_ANALYSER_FFT_SIZE = 1024; + +/** Number of peaks of a clip's static waveform. */ +export const VOICE_PEAKS_COUNT = 64; + +/** Microphone constraints: the same clean-up the chat form's recorder asks for. */ +export const VOICE_MICROPHONE_CONSTRAINTS: MediaTrackConstraints = { + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true +}; + +/** Name of the PCM tap worklet processor. */ +export const VOICE_WORKLET_PROCESSOR = 'llama-voice-pcm-tap'; + +/** Frames the worklet gathers before it posts them, about 43 ms at 48 kHz. */ +export const VOICE_WORKLET_BATCH_FRAMES = 2048; + +/** Buffer size of the ScriptProcessor fallback. */ +export const VOICE_SCRIPT_PROCESSOR_FRAMES = 2048; + +/** Bytes per String.fromCharCode call when encoding base64, below the argument limit. */ +export const VOICE_BASE64_SLICE = 0x8000; + +/** Server routes, relative so a server under a subpath still works. */ +export const VOICE_API_PATHS = { + REALTIME: './v1/realtime', + TRANSCRIPTIONS: './v1/audio/transcriptions' +} as const; + +/** Query parameters of the realtime socket. */ +export const VOICE_REALTIME_QUERY = { + AUTOLOAD: 'autoload', + INTENT: 'intent', + INTENT_TRANSCRIPTION: 'transcription', + MODEL: 'model' +} as const; + +/** Realtime events this client sends and reads, see tools/server/README.md. */ +export const VOICE_REALTIME_EVENTS = { + APPEND: 'input_audio_buffer.append', + COMMIT: 'input_audio_buffer.commit', + COMPLETED: 'conversation.item.input_audio_transcription.completed', + DELTA: 'conversation.item.input_audio_transcription.delta', + ERROR: 'error', + SESSION_CREATED: 'session.created', + SESSION_UPDATE: 'session.update' +} as const; + +/** The session type that transcribes; a conversation session is switched to it. */ +export const VOICE_REALTIME_SESSION_TRANSCRIPTION = 'transcription'; + +/** What the server says when the model has no streaming audio encoder. */ +export const VOICE_REALTIME_UNSUPPORTED_PATTERN = /does not support realtime transcription/i; + +/** Deadlines of a realtime session. */ +export const VOICE_REALTIME_TIMEOUTS = { + /** Socket open until session.created; a busy slot can hold it back */ + CONNECT_MS: 10_000, + /** Final commit until the last completed event */ + FINISH_MS: 20_000 +} as const; diff --git a/ui/apps/chat/src/lib/voice/errors.ts b/ui/apps/chat/src/lib/voice/errors.ts new file mode 100644 index 000000000000..b37f919d7d8d --- /dev/null +++ b/ui/apps/chat/src/lib/voice/errors.ts @@ -0,0 +1,41 @@ +/** A realtime session failed; the caller can transcribe the clip over HTTP instead. */ +export class RealtimeError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = 'RealtimeError'; + } +} + +/** + * Realtime transcription is not available: the model has no streaming audio encoder + * (`model`), or the socket could not be opened (`connection`), as with a server that + * checks an API key, which a browser websocket cannot send. + */ +export class RealtimeUnsupportedError extends RealtimeError { + readonly reason: 'model' | 'connection'; + + constructor(message: string, reason: 'model' | 'connection', options?: { cause?: unknown }) { + super(message, options); + this.name = 'RealtimeUnsupportedError'; + this.reason = reason; + } +} + +/** The transcription endpoint answered with an error; `message` is the server's own. */ +export class VoiceTranscriptionError extends Error { + readonly status: number; + + constructor(message: string, status: number) { + super(message); + this.name = 'VoiceTranscriptionError'; + this.status = status; + } +} + +/** The microphone did not start; `message` says why in words for the user. */ +export class VoiceCaptureError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = 'VoiceCaptureError'; + } +} diff --git a/ui/apps/chat/src/lib/voice/index.ts b/ui/apps/chat/src/lib/voice/index.ts new file mode 100644 index 000000000000..96bd82fe5171 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/index.ts @@ -0,0 +1,32 @@ +/** + * Voice engine: microphone capture with live levels and PCM streaming, clip peaks, + * and transcription over HTTP or llama-server's realtime socket. It imports no app + * stores or components; the app passes in how to reach a server (VoiceServerContext). + */ + +export { VoiceCapture } from './capture'; +export * from './constants'; +export { + RealtimeError, + RealtimeUnsupportedError, + VoiceCaptureError, + VoiceTranscriptionError +} from './errors'; +export { LevelHistory, rms, rmsToLevel } from './levels'; +export { + base64ToBytes, + bytesToBase64, + concatPcm16, + createResampler, + floatToPcm16, + pcm16ToBase64, + pcm16ToFloat, + type Resampler +} from './pcm'; +export { computePeaks, computePeaksFromAudio, decodeAudio, mixToMono } from './peaks'; +export { RealtimeTranscriber, realtimeUrl } from './realtime'; +export { startVoiceSession } from './session'; +export { transcribeClip } from './transcribe'; +export type * from './types'; +export { VoiceRecorder, type VoiceRecorderStatus } from './voice-recorder.svelte'; +export { encodeWav } from './wav'; diff --git a/ui/apps/chat/src/lib/voice/levels.ts b/ui/apps/chat/src/lib/voice/levels.ts new file mode 100644 index 000000000000..17e83804d09f --- /dev/null +++ b/ui/apps/chat/src/lib/voice/levels.ts @@ -0,0 +1,49 @@ +import { VOICE_LEVELS_FLOOR_DB } from './constants'; + +/** Root mean square of a block of samples. */ +export function rms(samples: Float32Array): number { + if (samples.length === 0) return 0; + + let sum = 0; + + for (let i = 0; i < samples.length; i++) sum += samples[i] * samples[i]; + + return Math.sqrt(sum / samples.length); +} + +/** + * RMS to a 0..1 level that follows loudness as heard: decibels mapped linearly from + * the floor (0) to full scale (1), so quiet speech still moves the waveform. + */ +export function rmsToLevel(value: number, floorDb = VOICE_LEVELS_FLOOR_DB): number { + if (value <= 0) return 0; + + const db = 20 * Math.log10(value); + + return Math.max(0, Math.min(1, 1 - db / floorDb)); +} + +/** Fixed-length level history, newest last. */ +export class LevelHistory { + private values: number[]; + + constructor(readonly length: number) { + this.values = new Array(length).fill(0); + } + + /** Adds a value, drops the oldest, and returns a fresh copy for listeners. */ + push(level: number): number[] { + this.values.push(level); + this.values.shift(); + + return this.values.slice(); + } + + reset(): void { + this.values.fill(0); + } + + snapshot(): number[] { + return this.values.slice(); + } +} diff --git a/ui/apps/chat/src/lib/voice/pcm-tap.ts b/ui/apps/chat/src/lib/voice/pcm-tap.ts new file mode 100644 index 000000000000..a144a9026528 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/pcm-tap.ts @@ -0,0 +1,148 @@ +import { + VOICE_SCRIPT_PROCESSOR_FRAMES, + VOICE_WORKLET_BATCH_FRAMES, + VOICE_WORKLET_PROCESSOR +} from './constants'; + +/** Message the tap answers once it has posted the samples it still held. */ +const FLUSH = 'flush'; +const FLUSHED = 'flushed'; +const FLUSH_TIMEOUT_MS = 250; +// runs on the audio thread; inline so no extra file has to be served next to the app +const WORKLET_SOURCE = ` +class PcmTap extends AudioWorkletProcessor { + constructor(options) { + super(); + this.size = options.processorOptions.batchFrames; + this.buffer = new Float32Array(this.size); + this.filled = 0; + this.port.onmessage = (event) => { + if (event.data !== '${FLUSH}') return; + this.post(); + this.port.postMessage('${FLUSHED}'); + }; + } + + post() { + if (this.filled === 0) return; + const block = this.buffer.slice(0, this.filled); + this.port.postMessage(block, [block.buffer]); + this.filled = 0; + } + + process(inputs) { + const channel = inputs[0] && inputs[0][0]; + if (!channel) return true; + let offset = 0; + while (offset < channel.length) { + const n = Math.min(channel.length - offset, this.size - this.filled); + this.buffer.set(channel.subarray(offset, offset + n), this.filled); + this.filled += n; + offset += n; + if (this.filled === this.size) this.post(); + } + return true; + } +} +registerProcessor('${VOICE_WORKLET_PROCESSOR}', PcmTap); +`; + +/** A node that hands mono float blocks of its input to the main thread. */ +export interface PcmTap { + node: AudioNode; + /** Posts what is still buffered, then resolves */ + flush(): Promise; + disconnect(): void; +} + +/** + * Taps mono float samples from the graph: an AudioWorklet loaded from a Blob URL + * (fine under COOP/COEP, a blob inherits the page's origin), or a ScriptProcessor + * where AudioWorklet is missing. + */ +export async function createPcmTap( + context: AudioContext, + onBlock: (block: Float32Array) => void +): Promise { + if (context.audioWorklet && typeof AudioWorkletNode !== 'undefined') { + try { + return await createWorkletTap(context, onBlock); + } catch (error) { + console.warn('AudioWorklet unavailable, using ScriptProcessor:', error); + } + } + + return createScriptProcessorTap(context, onBlock); +} + +async function createWorkletTap( + context: AudioContext, + onBlock: (block: Float32Array) => void +): Promise { + const url = URL.createObjectURL(new Blob([WORKLET_SOURCE], { type: 'text/javascript' })); + + try { + await context.audioWorklet.addModule(url); + } finally { + URL.revokeObjectURL(url); + } + + const node = new AudioWorkletNode(context, VOICE_WORKLET_PROCESSOR, { + channelCount: 1, + channelCountMode: 'explicit', + numberOfInputs: 1, + numberOfOutputs: 1, + processorOptions: { batchFrames: VOICE_WORKLET_BATCH_FRAMES } + }); + + let flushed: (() => void) | null = null; + + node.port.onmessage = (event: MessageEvent) => { + if (event.data === FLUSHED) { + flushed?.(); + } else if (event.data instanceof Float32Array) { + onBlock(event.data); + } + }; + + return { + disconnect: () => { + node.port.onmessage = null; + node.port.close(); + node.disconnect(); + }, + flush: () => + new Promise((resolve) => { + const timer = setTimeout(resolve, FLUSH_TIMEOUT_MS); + + flushed = () => { + clearTimeout(timer); + flushed = null; + resolve(); + }; + node.port.postMessage(FLUSH); + }), + node + }; +} + +function createScriptProcessorTap( + context: AudioContext, + onBlock: (block: Float32Array) => void +): PcmTap { + const node = context.createScriptProcessor(VOICE_SCRIPT_PROCESSOR_FRAMES, 1, 1); + + node.onaudioprocess = (event) => { + onBlock(event.inputBuffer.getChannelData(0).slice()); + }; + + return { + disconnect: () => { + node.onaudioprocess = null; + node.disconnect(); + }, + // a ScriptProcessor holds nothing back: each block is posted as soon as it is full + flush: () => Promise.resolve(), + node + }; +} diff --git a/ui/apps/chat/src/lib/voice/pcm.ts b/ui/apps/chat/src/lib/voice/pcm.ts new file mode 100644 index 000000000000..95b092b38288 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/pcm.ts @@ -0,0 +1,140 @@ +import { VOICE_BASE64_SLICE } from './constants'; + +/** Float samples (-1..1) to PCM16, clipped. */ +export function floatToPcm16(samples: Float32Array): Int16Array { + const out = new Int16Array(samples.length); + + for (let i = 0; i < samples.length; i++) { + const s = Math.max(-1, Math.min(1, samples[i])); + + out[i] = s < 0 ? s * 0x8000 : s * 0x7fff; + } + + return out; +} + +/** PCM16 samples back to floats (-1..1). */ +export function pcm16ToFloat(samples: Int16Array): Float32Array { + const out = new Float32Array(samples.length); + + for (let i = 0; i < samples.length; i++) { + out[i] = samples[i] / 0x8000; + } + + return out; +} + +/** Joins PCM16 chunks into one array. */ +export function concatPcm16(chunks: readonly Int16Array[]): Int16Array { + let length = 0; + + for (const chunk of chunks) length += chunk.length; + + const out = new Int16Array(length); + + let offset = 0; + + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + + return out; +} + +/** Bytes to base64, in slices so long inputs do not overflow the call stack. */ +export function bytesToBase64(bytes: Uint8Array): string { + let binary = ''; + + for (let i = 0; i < bytes.length; i += VOICE_BASE64_SLICE) { + binary += String.fromCharCode(...bytes.subarray(i, i + VOICE_BASE64_SLICE)); + } + + return btoa(binary); +} + +/** Base64 to bytes. */ +export function base64ToBytes(base64: string): Uint8Array { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + + return bytes; +} + +/** PCM16 samples as base64, little-endian as the realtime API expects. */ +export function pcm16ToBase64(samples: Int16Array): string { + // typed arrays use the platform byte order, which is little-endian on every browser target + return bytesToBase64(new Uint8Array(samples.buffer, samples.byteOffset, samples.byteLength)); +} + +/** Streaming resampler: feed blocks of any size, get the samples ready so far. */ +export interface Resampler { + process(input: Float32Array): Float32Array; +} + +/** + * Streaming mono resampler. Going down, each output sample is a triangle-weighted mean + * of the input around it, two output periods wide: a cheap low-pass that keeps most + * aliasing out of speech. Going up, it interpolates linearly. Same rates pass through. + */ +export function createResampler(fromRate: number, toRate: number): Resampler { + if (fromRate === toRate) { + return { process: (input) => input }; + } + + const ratio = fromRate / toRate; + // input reach on each side of an output sample + const reach = Math.max(1, ratio); + + // input samples kept for the next block, and the position of the next output in them + let pending = new Float32Array(0); + let position = 0; + + return { + process(input: Float32Array): Float32Array { + const buffer = new Float32Array(pending.length + input.length); + + buffer.set(pending); + buffer.set(input, pending.length); + + const out: number[] = []; + + if (ratio > 1) { + while (position + reach < buffer.length) { + let sum = 0; + let weight = 0; + + for (let i = Math.max(0, Math.ceil(position - reach)); i <= position + reach; i++) { + const w = 1 - Math.abs(i - position) / reach; + + sum += buffer[i] * w; + weight += w; + } + + out.push(weight > 0 ? sum / weight : 0); + position += ratio; + } + } else { + while (position + 1 < buffer.length) { + const i = Math.floor(position); + const t = position - i; + + out.push(buffer[i] * (1 - t) + buffer[i + 1] * t); + position += ratio; + } + } + + // keep what the next output still reaches back to + const keep = Math.max(0, Math.floor(position - (ratio > 1 ? reach : 0))); + + pending = buffer.slice(keep); + position -= keep; + + return Float32Array.from(out); + } + }; +} diff --git a/ui/apps/chat/src/lib/voice/peaks.ts b/ui/apps/chat/src/lib/voice/peaks.ts new file mode 100644 index 000000000000..52a9252ba878 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/peaks.ts @@ -0,0 +1,75 @@ +import { VOICE_PEAKS_COUNT } from './constants'; +import { base64ToBytes } from './pcm'; + +/** Mono samples of a buffer: the mean of its channels. */ +export function mixToMono(buffer: AudioBuffer): Float32Array { + if (buffer.numberOfChannels === 1) return buffer.getChannelData(0); + + const out = new Float32Array(buffer.length); + + for (let c = 0; c < buffer.numberOfChannels; c++) { + const channel = buffer.getChannelData(c); + + for (let i = 0; i < out.length; i++) out[i] += channel[i] / buffer.numberOfChannels; + } + + return out; +} + +/** + * Peaks of a clip for a static waveform: the loudest sample of each of `count` + * equal spans, scaled so the loudest span is 1. Silence gives all zeros. + */ +export function computePeaks( + source: Float32Array | Int16Array | AudioBuffer, + count = VOICE_PEAKS_COUNT +): number[] { + const samples = + source instanceof Float32Array || source instanceof Int16Array ? source : mixToMono(source); + const scale = samples instanceof Int16Array ? 1 / 0x8000 : 1; + const peaks = new Array(count).fill(0); + + if (samples.length === 0 || count <= 0) return peaks; + + for (let p = 0; p < count; p++) { + const start = Math.floor((p * samples.length) / count); + // a clip shorter than `count` repeats samples rather than leaving gaps + const end = Math.max(start + 1, Math.floor(((p + 1) * samples.length) / count)); + + let max = 0; + + for (let i = start; i < end && i < samples.length; i++) { + const value = Math.abs(samples[i] * scale); + + if (value > max) max = value; + } + + peaks[p] = max; + } + + const loudest = Math.max(...peaks); + + return loudest > 0 ? peaks.map((value) => value / loudest) : peaks; +} + +/** Decodes an audio file (Blob, bytes, or base64 such as a stored attachment) to a buffer. */ +export async function decodeAudio(source: Blob | ArrayBuffer | string): Promise { + const bytes = + typeof source === 'string' + ? base64ToBytes(source.replace(/^data:[^,]*,/, '')).buffer + : source instanceof ArrayBuffer + ? source + : await source.arrayBuffer(); + // an offline context decodes without a running audio device; the rate is a placeholder + const context = new OfflineAudioContext(1, 1, 44_100); + + return context.decodeAudioData(bytes as ArrayBuffer); +} + +/** Peaks of an encoded audio file, for stored clips that carry none. */ +export async function computePeaksFromAudio( + source: Blob | ArrayBuffer | string, + count = VOICE_PEAKS_COUNT +): Promise { + return computePeaks(await decodeAudio(source), count); +} diff --git a/ui/apps/chat/src/lib/voice/realtime.ts b/ui/apps/chat/src/lib/voice/realtime.ts new file mode 100644 index 000000000000..ff17e52b0a0a --- /dev/null +++ b/ui/apps/chat/src/lib/voice/realtime.ts @@ -0,0 +1,350 @@ +import { + VOICE_API_PATHS, + VOICE_REALTIME_EVENTS, + VOICE_REALTIME_QUERY, + VOICE_REALTIME_SESSION_TRANSCRIPTION, + VOICE_REALTIME_TIMEOUTS, + VOICE_REALTIME_UNSUPPORTED_PATTERN +} from './constants'; +import { RealtimeError, RealtimeUnsupportedError } from './errors'; +import { pcm16ToBase64 } from './pcm'; +import type { RealtimeTranscriberOptions, VoiceServerContext } from './types'; + +type RealtimeState = 'idle' | 'connecting' | 'open' | 'finishing' | 'closed'; + +interface Pending { + resolve: (value: T) => void; + reject: (error: Error) => void; +} + +/** A server event, only the fields this client reads. */ +interface RealtimeServerEvent { + type?: string; + delta?: unknown; + transcript?: unknown; + error?: { message?: unknown }; + session?: { type?: unknown; audio?: { input?: { format?: { rate?: unknown } } } }; +} + +/** + * URL of the realtime socket for a server: the same target as its HTTP routes, with + * ws(s) for http(s), `intent=transcription`, and the model for a router. + * No API key goes in it: llama-server only reads the key from headers. + */ +export function realtimeUrl( + context: VoiceServerContext, + options: Pick = {} +): string { + const url = new URL( + context.resolveUrl(VOICE_API_PATHS.REALTIME), + context.baseHref ?? globalThis.location?.href + ); + + url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:'; + url.searchParams.set(VOICE_REALTIME_QUERY.INTENT, VOICE_REALTIME_QUERY.INTENT_TRANSCRIPTION); + + if (options.model) url.searchParams.set(VOICE_REALTIME_QUERY.MODEL, options.model); + + if (options.autoload !== undefined) { + url.searchParams.set(VOICE_REALTIME_QUERY.AUTOLOAD, String(options.autoload)); + } + + return url.toString(); +} + +/** + * Live transcription over llama-server's `/v1/realtime` socket. Audio given to + * `append` before the session starts is queued and sent once it does. + * + * A model with no streaming audio encoder gets an `error` event and the socket + * closes: `connect()` then rejects with RealtimeUnsupportedError and the caller + * transcribes the clip over HTTP instead. A model that holds realtime conversations + * is switched to a transcription session: its transcript comes after the final commit. + */ +export class RealtimeTranscriber { + /** True when the session streams text while the audio comes in */ + live = false; + /** Input rate of the model, from session.created */ + modelSampleRate: number | null = null; + + private completedText = ''; + private connecting: Promise | null = null; + + private readonly context: VoiceServerContext; + private itemText = ''; + private readonly options: RealtimeTranscriberOptions; + private pendingConnect: Pending | null = null; + private pendingFinish: Pending | null = null; + private queue: string[] = []; + private sentAudio = false; + private socket: WebSocket | null = null; + private state: RealtimeState = 'idle'; + private timer: ReturnType | null = null; + + /** True between session.created and finish() or close(). */ + get ready(): boolean { + return this.state === 'open'; + } + + /** The whole transcript so far, live text of the current item included. */ + get text(): string { + return this.completedText + this.itemText; + } + + /** Streams a PCM16 mono chunk at `options.sampleRate`. */ + append(chunk: Int16Array): void { + if (chunk.length === 0 || this.state === 'finishing' || this.state === 'closed') return; + + const message = JSON.stringify({ + audio: pcm16ToBase64(chunk), + type: VOICE_REALTIME_EVENTS.APPEND + }); + + this.sentAudio = true; + + if (this.state === 'open') { + this.socket?.send(message); + } else { + this.queue.push(message); + } + } + + /** Closes the socket; pending connect or finish calls reject. */ + close(): void { + if (this.state === 'closed') return; + + this.fail(new RealtimeError('The realtime session was closed.')); + } + + /** + * Opens the socket and resolves once the session is created. + * @throws RealtimeUnsupportedError when the model cannot stream or the socket does not open + */ + connect(): Promise { + if (this.connecting) return this.connecting; + + this.connecting = new Promise((resolve, reject) => { + this.pendingConnect = { reject, resolve }; + }); + + this.state = 'connecting'; + + let socket: WebSocket; + + try { + socket = new WebSocket(realtimeUrl(this.context, this.options)); + } catch (error) { + this.fail( + new RealtimeUnsupportedError('The realtime socket could not be opened.', 'connection', { + cause: error + }) + ); + + return this.connecting; + } + + this.socket = socket; + socket.onmessage = (event: MessageEvent) => this.handleMessage(event.data); + socket.onclose = (event: CloseEvent) => this.handleClose(event); + this.startTimer( + this.options.connectTimeoutMs ?? VOICE_REALTIME_TIMEOUTS.CONNECT_MS, + 'The realtime session did not start in time.' + ); + + return this.connecting; + } + + constructor(context: VoiceServerContext, options: RealtimeTranscriberOptions) { + this.context = context; + this.options = options; + } + + /** + * Ends the input (`input_audio_buffer.commit` with `final: true`) and resolves with + * the whole transcript once the last item completes. + * @throws RealtimeError when the session fails, closes or times out first + */ + async finish(): Promise { + if (this.state === 'connecting') await this.connecting; + + if (this.state !== 'open') throw new RealtimeError('The realtime session is not open.'); + + if (!this.sentAudio) { + this.close(); + + return ''; + } + + this.state = 'finishing'; + + const done = new Promise((resolve, reject) => { + this.pendingFinish = { reject, resolve }; + }); + + this.send({ final: true, type: VOICE_REALTIME_EVENTS.COMMIT }); + this.startTimer( + this.options.finishTimeoutMs ?? VOICE_REALTIME_TIMEOUTS.FINISH_MS, + 'The transcript did not complete in time.' + ); + + return done; + } + + private clearTimer(): void { + if (this.timer !== null) { + clearTimeout(this.timer); + this.timer = null; + } + } + + /** Rejects whatever is pending with `error` and releases the socket. */ + private fail(error: Error): void { + this.state = 'closed'; + this.clearTimer(); + this.queue = []; + this.pendingConnect?.reject(error); + this.pendingConnect = null; + this.pendingFinish?.reject(error); + this.pendingFinish = null; + + const socket = this.socket; + + this.socket = null; + + if (socket) { + socket.onmessage = null; + socket.onclose = null; + socket.close(); + } + } + + private handleClose(event: CloseEvent): void { + if (this.state === 'closed') return; + + // a refused upgrade (e.g. 401 from a server with an API key) closes before any session + const error = + this.state === 'connecting' + ? new RealtimeUnsupportedError( + `The realtime socket closed before the session started (code ${event.code}).`, + 'connection' + ) + : new RealtimeError('The realtime socket closed before the transcript completed.'); + + if (this.state === 'open') this.options.onError?.(error); + + this.fail(error); + } + + private handleCompleted(transcript: string): void { + this.completedText += transcript; + this.itemText = ''; + this.options.onCompleted?.(this.text); + + if (this.state === 'finishing' && this.pendingFinish) { + const text = this.text.trim(); + + this.pendingFinish.resolve(text); + this.pendingFinish = null; + this.close(); + } + } + + private handleError(message: string): void { + if (this.state === 'connecting') { + this.fail( + VOICE_REALTIME_UNSUPPORTED_PATTERN.test(message) + ? new RealtimeUnsupportedError(message, 'model') + : new RealtimeError(message) + ); + + return; + } + + const error = new RealtimeError(message); + + this.options.onError?.(error); + + if (this.state === 'finishing') this.fail(error); + } + + private handleMessage(data: unknown): void { + if (typeof data !== 'string') return; + + let event: RealtimeServerEvent; + + try { + event = JSON.parse(data); + } catch { + return; + } + + switch (event.type) { + case VOICE_REALTIME_EVENTS.SESSION_CREATED: + this.handleSessionCreated(event); + + break; + case VOICE_REALTIME_EVENTS.DELTA: + if (typeof event.delta === 'string' && event.delta) { + this.itemText += event.delta; + this.options.onDelta?.(event.delta, this.text); + } + + break; + case VOICE_REALTIME_EVENTS.COMPLETED: + this.handleCompleted( + typeof event.transcript === 'string' ? event.transcript : this.itemText + ); + + break; + case VOICE_REALTIME_EVENTS.ERROR: + this.handleError( + typeof event.error?.message === 'string' + ? event.error.message + : 'Realtime transcription failed.' + ); + + break; + } + } + + private handleSessionCreated(event: RealtimeServerEvent): void { + if (this.state !== 'connecting') return; + + const rate = event.session?.audio?.input?.format?.rate; + + this.modelSampleRate = typeof rate === 'number' ? rate : null; + this.live = event.session?.type === VOICE_REALTIME_SESSION_TRANSCRIPTION; + this.clearTimer(); + + // the rate of our chunks, and a transcription session for a model that would hold a conversation + this.send({ + session: { + audio: { + input: { + format: { rate: this.options.sampleRate, type: 'audio/pcm' }, + ...(this.options.language ? { transcription: { language: this.options.language } } : {}) + } + }, + type: VOICE_REALTIME_SESSION_TRANSCRIPTION + }, + type: VOICE_REALTIME_EVENTS.SESSION_UPDATE + }); + + this.state = 'open'; + + for (const message of this.queue) this.socket?.send(message); + + this.queue = []; + this.pendingConnect?.resolve(); + this.pendingConnect = null; + } + + private send(event: object): void { + this.socket?.send(JSON.stringify(event)); + } + + private startTimer(ms: number, message: string): void { + this.clearTimer(); + this.timer = setTimeout(() => this.fail(new RealtimeError(message)), ms); + } +} diff --git a/ui/apps/chat/src/lib/voice/session.ts b/ui/apps/chat/src/lib/voice/session.ts new file mode 100644 index 000000000000..de02e2dc0048 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/session.ts @@ -0,0 +1,132 @@ +import { VoiceCapture } from './capture'; +import { RealtimeUnsupportedError } from './errors'; +import { RealtimeTranscriber } from './realtime'; +import { transcribeClip } from './transcribe'; +import type { VoiceSession, VoiceSessionOptions, VoiceSessionResult } from './types'; + +/** + * Starts capturing and, when asked to transcribe, tries the realtime socket at the + * same time. Realtime gives live text; when the model cannot stream, the socket does + * not open, or the session fails, the clip is transcribed over HTTP after recording. + * @throws VoiceCaptureError when the microphone does not start + */ +export async function startVoiceSession(options: VoiceSessionOptions = {}): Promise { + const capture = isCapture(options.capture) ? options.capture : new VoiceCapture(options.capture); + const unsubscribers: (() => void)[] = []; + + if (options.onLevels) unsubscribers.push(capture.onLevels(options.onLevels)); + + try { + await capture.start(); + } catch (error) { + for (const unsubscribe of unsubscribers) unsubscribe(); + + throw error; + } + + const transcribe = options.transcribe; + + let transcriber: RealtimeTranscriber | null = null; + let liveText: string | null = null; + let ended = false; + + if (transcribe && options.realtime !== false) { + const realtime = new RealtimeTranscriber(transcribe.context, { + autoload: transcribe.autoload, + language: transcribe.language, + model: transcribe.model, + onCompleted: (text) => { + liveText = text; + options.onText?.(text); + }, + onDelta: (_delta, text) => { + liveText = text; + options.onText?.(text); + }, + sampleRate: capture.sampleRate + }); + const unsubscribePcm = capture.onPcm((chunk) => realtime.append(chunk)); + + transcriber = realtime; + unsubscribers.push(unsubscribePcm); + realtime.connect().then( + () => { + if (!ended) options.onRealtime?.(realtime.live); + }, + (error) => { + unsubscribePcm(); + + if (ended) return; + + if (!(error instanceof RealtimeUnsupportedError)) { + console.warn('Realtime transcription unavailable, transcribing after recording:', error); + } + + options.onRealtime?.(false); + } + ); + } else if (transcribe) { + options.onRealtime?.(false); + } + + const release = () => { + ended = true; + + for (const unsubscribe of unsubscribers) unsubscribe(); + + unsubscribers.length = 0; + }; + + return { + cancel() { + release(); + capture.cancel(); + transcriber?.close(); + }, + capture, + async stop(): Promise { + let clip; + + try { + // the last PCM chunk reaches the socket before stop() resolves + clip = await capture.stop(); + } catch (error) { + release(); + transcriber?.close(); + + throw error; + } + + release(); + + if (!transcribe) return { clip, liveText: null, transcript: null }; + + const overHttp = () => + transcribeClip(transcribe.context, clip.blob, { + language: transcribe.language, + model: transcribe.model + }); + const realtime = transcriber; + + let transcript: Promise; + + if (realtime?.ready) { + transcript = realtime.finish().catch((error) => { + console.warn('Realtime transcript failed, transcribing over HTTP:', error); + + return overHttp(); + }); + } else { + // still connecting at the end: the HTTP request is the shorter wait + realtime?.close(); + transcript = overHttp(); + } + + return { clip, liveText: liveText?.trim() || null, transcript }; + } + }; +} + +function isCapture(value: VoiceSessionOptions['capture']): value is VoiceCapture { + return typeof (value as VoiceCapture | undefined)?.start === 'function'; +} diff --git a/ui/apps/chat/src/lib/voice/transcribe.ts b/ui/apps/chat/src/lib/voice/transcribe.ts new file mode 100644 index 000000000000..27d54df604f3 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/transcribe.ts @@ -0,0 +1,83 @@ +import { VOICE_API_PATHS } from './constants'; +import { VoiceTranscriptionError } from './errors'; +import type { TranscribeClipOptions, VoiceServerContext } from './types'; + +/** + * Transcribes a recorded clip with `POST /v1/audio/transcriptions`: any model with + * audio input can do it, the server turns it into a chat completion. + * @returns the transcript, trimmed + * @throws VoiceTranscriptionError carrying the server's own message + */ +export async function transcribeClip( + context: VoiceServerContext, + blob: Blob, + options: TranscribeClipOptions = {} +): Promise { + const form = new FormData(); + + form.append('file', blob, options.filename ?? 'audio.wav'); + + if (options.model) form.append('model', options.model); + + if (options.language) form.append('language', options.language); + + if (options.prompt) form.append('prompt', options.prompt); + + form.append('response_format', 'json'); + + // no Content-Type: the browser sets the multipart boundary itself + const headers = { ...context.getHeaders?.() }; + + for (const key of Object.keys(headers)) { + if (key.toLowerCase() === 'content-type') delete headers[key]; + } + + const response = await (context.fetch ?? fetch)( + context.resolveUrl(VOICE_API_PATHS.TRANSCRIPTIONS), + { + body: form, + headers, + method: 'POST', + signal: options.signal + } + ); + + if (!response.ok) { + throw new VoiceTranscriptionError(await readErrorMessage(response), response.status); + } + + const data = (await response.json()) as { text?: unknown }; + + if (typeof data?.text !== 'string') { + throw new VoiceTranscriptionError('The server returned no transcript.', response.status); + } + + return data.text.trim(); +} + +async function readErrorMessage(response: Response): Promise { + const fallback = `Transcription failed (${response.status}${response.statusText ? ` ${response.statusText}` : ''})`; + + try { + const text = await response.text(); + + try { + const data = JSON.parse(text); + + if (typeof data?.error?.message === 'string') return data.error.message; + + if (typeof data?.error === 'string') return data.error; + + if (typeof data?.message === 'string') return data.message; + } catch { + // not JSON: a proxy's plain text error + } + + const plain = text.trim(); + + // an HTML error page says nothing useful in a toast + return plain && !plain.startsWith('<') ? plain : fallback; + } catch { + return fallback; + } +} diff --git a/ui/apps/chat/src/lib/voice/types.ts b/ui/apps/chat/src/lib/voice/types.ts new file mode 100644 index 000000000000..4210d6bb2e09 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/types.ts @@ -0,0 +1,122 @@ +import type { VoiceCapture } from './capture'; + +/** + * Voice engine types. Kept next to the engine so the folder can move into its own + * package; the app re-exports nothing from here and imports `#lib/voice/index.ts` directly. + */ + +/** A finished recording, ready to send and to draw. */ +export interface VoiceClip { + /** WAV, 16-bit PCM, mono */ + blob: Blob; + durationMs: number; + /** Normalised 0..1, fixed count, for the static waveform */ + peaks: number[]; + sampleRate: number; +} + +/** Called with the live level history: 0..1, fixed length, newest last. */ +export type VoiceLevelsListener = (levels: readonly number[]) => void; + +/** Called with each PCM16 mono chunk at the capture rate. */ +export type VoicePcmListener = (chunk: Int16Array) => void; + +export interface VoiceCaptureOptions { + /** Rate of the PCM chunks and of the clip (default 16000) */ + sampleRate?: number; + /** Length of a PCM chunk (default 100 ms) */ + chunkMs?: number; + /** Values in the level history (default 48) */ + levelsLength?: number; + /** Time between two level values (default 50 ms) */ + levelsIntervalMs?: number; + /** Peaks of the clip (default 64) */ + peaksCount?: number; + /** Extra constraints merged over echo cancellation, noise suppression and auto gain */ + constraints?: MediaTrackConstraints; + /** Use this stream instead of asking for the microphone; the capture stops its tracks too */ + stream?: MediaStream; +} + +export type VoiceCaptureState = 'idle' | 'starting' | 'recording' | 'stopped'; + +/** + * How the engine reaches a server. The app wires its own URL resolution and + * auth in, so the engine never imports app stores. + */ +export interface VoiceServerContext { + /** Request target for an API path, e.g. `(path) => apiUrl(path, serverId)` */ + resolveUrl: (path: string) => string; + /** HTTP headers such as the API key; a browser websocket cannot carry them */ + getHeaders?: () => Record; + /** Fetch to use, defaults to the global one */ + fetch?: typeof fetch; + /** Base for relative URLs, defaults to `location.href` */ + baseHref?: string; +} + +export interface TranscribeClipOptions { + /** Model id, required by a router server */ + model?: string; + /** ISO-639-1 language hint */ + language?: string; + /** Replaces the model's own transcription prompt */ + prompt?: string; + /** File name sent with the audio (default `audio.wav`) */ + filename?: string; + signal?: AbortSignal; +} + +export interface RealtimeTranscriberOptions { + /** Model id, sent as `?model=` (the router routes on it) */ + model?: string; + /** Rate of the PCM16 chunks given to `append` */ + sampleRate: number; + /** ISO-639-1 language hint, used by a conversation model switched to transcription */ + language?: string; + /** Sent as `?autoload=`; false keeps a router from loading the model */ + autoload?: boolean; + connectTimeoutMs?: number; + finishTimeoutMs?: number; + /** New text of the current item, and the whole live text */ + onDelta?: (delta: string, text: string) => void; + /** The whole transcript so far, once an item completes */ + onCompleted?: (text: string) => void; + /** Errors after the session started; the session goes on unless finish() fails */ + onError?: (error: Error) => void; +} + +export interface VoiceSessionOptions { + /** Capture options, or a capture made by the caller */ + capture?: VoiceCaptureOptions | VoiceCapture; + /** Try the realtime socket first (default true); off for servers behind a tunnel */ + realtime?: boolean; + /** Where and with which model to transcribe; leave out to record only */ + transcribe?: { + context: VoiceServerContext; + model?: string; + language?: string; + autoload?: boolean; + }; + onLevels?: VoiceLevelsListener; + /** Live text while speaking, only when the realtime socket streams it */ + onText?: (text: string) => void; + /** Once known: true when text streams while speaking, false when it comes after recording */ + onRealtime?: (live: boolean) => void; +} + +export interface VoiceSessionResult { + clip: VoiceClip; + /** Text the realtime socket gave while speaking, null when it streamed none */ + liveText: string | null; + /** Resolves with the final transcript (realtime, else HTTP); null when not asked to transcribe */ + transcript: Promise | null; +} + +export interface VoiceSession { + readonly capture: VoiceCapture; + /** Ends the recording and resolves with the clip and the transcript (or its promise) */ + stop(): Promise; + /** Drops the recording and closes everything */ + cancel(): void; +} diff --git a/ui/apps/chat/src/lib/voice/voice-recorder.svelte.ts b/ui/apps/chat/src/lib/voice/voice-recorder.svelte.ts new file mode 100644 index 000000000000..66bb64e652f9 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/voice-recorder.svelte.ts @@ -0,0 +1,99 @@ +import { VOICE_LEVELS_LENGTH } from './constants'; +import { startVoiceSession } from './session'; +import type { VoiceSession, VoiceSessionOptions, VoiceSessionResult } from './types'; + +export type VoiceRecorderStatus = 'idle' | 'starting' | 'recording' | 'stopping'; + +/** + * Reactive voice session for components: `levels`, `liveText`, `live` and `status` + * are runes that follow the running session. + */ +export class VoiceRecorder { + /** Level history for the live waveform, 0..1, newest last */ + levels = $state.raw(new Array(VOICE_LEVELS_LENGTH).fill(0)); + /** True once realtime is known to stream text while speaking */ + live = $state(false); + /** Text the realtime socket streamed so far */ + liveText = $state(''); + status = $state('idle'); + + // bumped by cancel(), so a start still waiting for the microphone drops its session + private generation = 0; + private session: VoiceSession | null = null; + + /** Drops the recording. */ + cancel(): void { + this.generation++; + this.session?.cancel(); + this.session = null; + this.status = 'idle'; + } + + /** + * Starts a session; the callbacks in `options` still run. + * @throws VoiceCaptureError when the microphone does not start + */ + async start(options: VoiceSessionOptions = {}): Promise { + if (this.status !== 'idle') throw new Error('Voice recorder already running'); + + const capture = options.capture; + const length = + capture && 'levels' in capture + ? capture.levels.length + : (capture?.levelsLength ?? VOICE_LEVELS_LENGTH); + + this.levels = new Array(length).fill(0); + this.liveText = ''; + this.live = false; + this.status = 'starting'; + + const generation = ++this.generation; + + try { + const session = await startVoiceSession({ + ...options, + onLevels: (levels) => { + this.levels = levels as number[]; + options.onLevels?.(levels); + }, + onRealtime: (live) => { + this.live = live; + options.onRealtime?.(live); + }, + onText: (text) => { + this.liveText = text; + options.onText?.(text); + } + }); + + if (generation !== this.generation) { + session.cancel(); + + return; + } + + this.session = session; + this.status = 'recording'; + } catch (error) { + if (generation === this.generation) this.status = 'idle'; + + throw error; + } + } + + /** Ends the recording; see VoiceSessionResult. */ + async stop(): Promise { + const session = this.session; + + if (!session || this.status !== 'recording') throw new Error('No voice recording to stop'); + + this.status = 'stopping'; + + try { + return await session.stop(); + } finally { + this.session = null; + this.status = 'idle'; + } + } +} diff --git a/ui/apps/chat/src/lib/voice/wav.ts b/ui/apps/chat/src/lib/voice/wav.ts new file mode 100644 index 000000000000..3b89485461c3 --- /dev/null +++ b/ui/apps/chat/src/lib/voice/wav.ts @@ -0,0 +1,36 @@ +import { MimeTypeAudio } from '#lib/enums/index.ts'; + +const WAV_HEADER_BYTES = 44; + +/** + * Mono PCM16 samples as a WAV file: the format `/v1/audio/transcriptions` and the + * chat's `input_audio` parts accept. + */ +export function encodeWav(samples: Int16Array, sampleRate: number): Blob { + const dataSize = samples.length * 2; + const buffer = new ArrayBuffer(WAV_HEADER_BYTES + dataSize); + const view = new DataView(buffer); + const writeAscii = (offset: number, text: string) => { + for (let i = 0; i < text.length; i++) view.setUint8(offset + i, text.charCodeAt(i)); + }; + + writeAscii(0, 'RIFF'); + view.setUint32(4, WAV_HEADER_BYTES - 8 + dataSize, true); + writeAscii(8, 'WAVE'); + writeAscii(12, 'fmt '); + view.setUint32(16, 16, true); // fmt chunk size + view.setUint16(20, 1, true); // PCM + view.setUint16(22, 1, true); // mono + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * 2, true); // byte rate + view.setUint16(32, 2, true); // block align + view.setUint16(34, 16, true); // bits per sample + writeAscii(36, 'data'); + view.setUint32(40, dataSize, true); + + for (let i = 0; i < samples.length; i++) { + view.setInt16(WAV_HEADER_BYTES + i * 2, samples[i], true); + } + + return new Blob([buffer], { type: MimeTypeAudio.WAV }); +} diff --git a/ui/apps/chat/tests/client/voice-capture.svelte.test.ts b/ui/apps/chat/tests/client/voice-capture.svelte.test.ts new file mode 100644 index 000000000000..3abd04696ba9 --- /dev/null +++ b/ui/apps/chat/tests/client/voice-capture.svelte.test.ts @@ -0,0 +1,163 @@ +// Guards VoiceCapture in a real browser, fed by a synthesized microphone (an oscillator +// into a MediaStream, since the fake capture device flags are not available here): the +// levels move, the PCM worklet loads from its Blob URL and streams chunks, stop() returns +// a WAV clip with a plausible duration and peaks, and every path stops the tracks. + +import { MICROPHONE_UNAVAILABLE_MESSAGE } from '#lib/constants/index.ts'; +import { VoiceCapture, VoiceCaptureError } from '#lib/voice/index.ts'; +import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; +import { userEvent } from 'vitest/browser'; + +const sources: AudioContext[] = []; + +/** A 440 Hz tone as a microphone stream. */ +function toneStream(): MediaStream { + const context = new AudioContext(); + const oscillator = context.createOscillator(); + const destination = context.createMediaStreamDestination(); + + oscillator.frequency.value = 440; + oscillator.connect(destination); + oscillator.start(); + sources.push(context); + + return destination.stream; +} + +// audio contexts only run after a user gesture, as the mic button gives in the app +beforeAll(async () => { + const button = document.createElement('button'); + + button.textContent = 'Record'; + document.body.append(button); + await userEvent.click(button); + button.remove(); +}); + +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + + for (const context of sources.splice(0)) await context.close(); +}); + +describe('VoiceCapture', () => { + it('streams levels and PCM chunks, and stops with a WAV clip', async () => { + const warn = vi.spyOn(console, 'warn'); + const stream = toneStream(); + const capture = new VoiceCapture({ levelsIntervalMs: 20, levelsLength: 16, stream }); + const levels: (readonly number[])[] = []; + const chunks: Int16Array[] = []; + + capture.onLevels((value) => levels.push(value)); + capture.onPcm((chunk) => chunks.push(chunk)); + + await capture.start(); + expect(capture.state).toBe('recording'); + + await wait(700); + + const clip = await capture.stop(); + + // the worklet loaded from its Blob URL under COOP/COEP, as llama-server serves the UI, no fallback + expect(crossOriginIsolated).toBe(true); + expect(warn).not.toHaveBeenCalledWith( + expect.stringContaining('AudioWorklet unavailable'), + expect.anything() + ); + + expect(levels.length).toBeGreaterThan(5); + expect(levels.at(-1)).toHaveLength(16); + expect(Math.max(...levels.at(-1)!)).toBeGreaterThan(0.5); + + // 100 ms chunks at 16 kHz, the last one may be shorter + expect(chunks.length).toBeGreaterThan(3); + expect(chunks[0]).toHaveLength(1600); + + expect(clip.sampleRate).toBe(16_000); + expect(clip.durationMs).toBeGreaterThan(400); + expect(clip.durationMs).toBeLessThan(1500); + expect(clip.peaks).toHaveLength(64); + expect(Math.max(...clip.peaks)).toBe(1); + // a steady tone: no span of the clip is silent + expect(clip.peaks.filter((peak) => peak > 0.5).length).toBeGreaterThan(48); + + const header = new DataView(await clip.blob.slice(0, 44).arrayBuffer()); + const pcmBytes = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0); + + expect(clip.blob.type).toBe('audio/wav'); + expect(header.getUint16(22, true)).toBe(1); + expect(header.getUint32(24, true)).toBe(16_000); + expect(header.getUint16(34, true)).toBe(16); + expect(header.getUint32(40, true)).toBe(pcmBytes); + + // the clip decodes as audio + const decoded = await new OfflineAudioContext(1, 1, 16_000).decodeAudioData( + await clip.blob.arrayBuffer() + ); + + expect(decoded.duration * 1000).toBeCloseTo(clip.durationMs, 0); + expect(stream.getTracks().every((track) => track.readyState === 'ended')).toBe(true); + expect(capture.state).toBe('stopped'); + }); + + it('cancel stops the tracks and drops the recording', async () => { + const stream = toneStream(); + const capture = new VoiceCapture({ stream }); + + await capture.start(); + await wait(200); + capture.cancel(); + + expect(stream.getTracks().every((track) => track.readyState === 'ended')).toBe(true); + expect(capture.state).toBe('idle'); + await expect(capture.stop()).rejects.toThrow(); + }); + + it('cancel during start releases the microphone once it arrives', async () => { + const stream = toneStream(); + + let grant!: (value: MediaStream) => void; + + vi.spyOn(navigator.mediaDevices, 'getUserMedia').mockImplementation( + () => new Promise((resolve) => (grant = resolve)) + ); + + const capture = new VoiceCapture(); + const started = capture.start().catch((e) => e); + + capture.cancel(); + grant(stream); + + expect(await started).toBeInstanceOf(DOMException); + expect(stream.getTracks().every((track) => track.readyState === 'ended')).toBe(true); + }); + + it('a page hide tears the capture down', async () => { + const stream = toneStream(); + const capture = new VoiceCapture({ stream }); + + await capture.start(); + dispatchEvent(new PageTransitionEvent('pagehide')); + + expect(capture.state).toBe('idle'); + expect(stream.getTracks().every((track) => track.readyState === 'ended')).toBe(true); + }); + + it('asks for the microphone with the clean-up constraints and names a refusal', async () => { + const getUserMedia = vi + .spyOn(navigator.mediaDevices, 'getUserMedia') + .mockRejectedValue(new DOMException('denied', 'NotAllowedError')); + const capture = new VoiceCapture(); + const error = await capture.start().catch((e) => e); + + expect(getUserMedia).toHaveBeenCalledWith({ + audio: { autoGainControl: true, echoCancellation: true, noiseSuppression: true } + }); + expect(error).toBeInstanceOf(VoiceCaptureError); + expect(error.message).toBe(MICROPHONE_UNAVAILABLE_MESSAGE.denied); + expect(capture.state).toBe('idle'); + }); +}); diff --git a/ui/apps/chat/tests/unit/voice-audio.test.ts b/ui/apps/chat/tests/unit/voice-audio.test.ts new file mode 100644 index 000000000000..f4d24c75ba3e --- /dev/null +++ b/ui/apps/chat/tests/unit/voice-audio.test.ts @@ -0,0 +1,197 @@ +// Guards the voice engine's audio math: PCM16 conversion, the streaming resampler, +// the WAV the server accepts (16-bit PCM mono header fields), peaks and levels. + +import { + base64ToBytes, + computePeaks, + concatPcm16, + createResampler, + encodeWav, + floatToPcm16, + LevelHistory, + pcm16ToBase64, + pcm16ToFloat, + rms, + rmsToLevel +} from '#lib/voice/index.ts'; +import { describe, expect, it } from 'vitest'; + +function sine(rate: number, seconds: number, frequency: number, amplitude = 0.5): Float32Array { + const out = new Float32Array(Math.round(rate * seconds)); + + for (let i = 0; i < out.length; i++) { + out[i] = amplitude * Math.sin((2 * Math.PI * frequency * i) / rate); + } + + return out; +} + +describe('PCM16 conversion', () => { + it('maps full scale to the int16 range and clips beyond it', () => { + expect(Array.from(floatToPcm16(Float32Array.from([0, 1, -1, 2, -2, 0.5])))).toEqual([ + 0, 32767, -32768, 32767, -32768, 16383 + ]); + }); + + it('round-trips through floats within one step', () => { + const input = Float32Array.from([0.25, -0.75, 0.1]); + const back = pcm16ToFloat(floatToPcm16(input)); + + for (let i = 0; i < input.length; i++) expect(back[i]).toBeCloseTo(input[i], 4); + }); + + it('joins chunks in order', () => { + expect( + Array.from(concatPcm16([Int16Array.from([1, 2]), Int16Array.from([]), Int16Array.from([3])])) + ).toEqual([1, 2, 3]); + }); + + it('encodes little-endian base64', () => { + const bytes = base64ToBytes(pcm16ToBase64(Int16Array.from([1, -2, 0x1234]))); + + expect(Array.from(bytes)).toEqual([0x01, 0x00, 0xfe, 0xff, 0x34, 0x12]); + }); + + it('encodes long inputs without overflowing the call stack', () => { + const samples = new Int16Array(200_000).fill(7); + + expect(base64ToBytes(pcm16ToBase64(samples)).length).toBe(400_000); + }); +}); + +describe('createResampler', () => { + it('passes the same rate through', () => { + const input = Float32Array.from([0.1, 0.2]); + + expect(createResampler(16_000, 16_000).process(input)).toBe(input); + }); + + it('downsamples 48 kHz to 16 kHz in blocks, holding back at most the last sample', () => { + const resampler = createResampler(48_000, 16_000); + const input = sine(48_000, 1, 440); + + let total = 0; + + // odd block sizes, as the audio thread may deliver + for (let offset = 0; offset < input.length; offset += 1000) { + total += resampler.process(input.subarray(offset, offset + 1000)).length; + } + + // the last output waits for the input after it + expect(total).toBeGreaterThanOrEqual(15_999); + expect(total).toBeLessThanOrEqual(16_000); + }); + + it('downsamples 44.1 kHz to within one sample of the expected length', () => { + const resampler = createResampler(44_100, 16_000); + const out = resampler.process(sine(44_100, 1, 300)); + + expect(Math.abs(out.length - 16_000)).toBeLessThanOrEqual(2); + }); + + it('keeps a speech-band tone and its level', () => { + const out = createResampler(48_000, 16_000).process(sine(48_000, 0.5, 440, 0.5)); + + expect(rms(out)).toBeCloseTo(0.5 / Math.SQRT2, 2); + }); + + it('averages away a tone above the new Nyquist rate', () => { + // 12 kHz aliases into speech at 16 kHz unless it is filtered out + const out = createResampler(48_000, 16_000).process(sine(48_000, 0.5, 12_000, 0.5)); + + expect(rms(out)).toBeLessThan(0.05); + }); + + it('upsamples by linear interpolation', () => { + const out = createResampler(8_000, 16_000).process(Float32Array.from([0, 1, 0, -1, 0])); + + expect(Array.from(out)).toEqual([0, 0.5, 1, 0.5, 0, -0.5, -1, -0.5]); + }); +}); + +describe('encodeWav', () => { + it('writes a 16-bit PCM mono header the server accepts', async () => { + const samples = Int16Array.from([0, 1000, -1000, 32767]); + const blob = encodeWav(samples, 16_000); + const view = new DataView(await blob.arrayBuffer()); + const ascii = (offset: number) => + String.fromCharCode(...new Uint8Array(view.buffer, offset, 4)); + + expect(blob.type).toBe('audio/wav'); + expect(view.byteLength).toBe(44 + 8); + expect(ascii(0)).toBe('RIFF'); + expect(view.getUint32(4, true)).toBe(36 + 8); + expect(ascii(8)).toBe('WAVE'); + expect(ascii(12)).toBe('fmt '); + expect(view.getUint32(16, true)).toBe(16); + expect(view.getUint16(20, true)).toBe(1); // PCM + expect(view.getUint16(22, true)).toBe(1); // mono + expect(view.getUint32(24, true)).toBe(16_000); + expect(view.getUint32(28, true)).toBe(32_000); + expect(view.getUint16(32, true)).toBe(2); + expect(view.getUint16(34, true)).toBe(16); + expect(ascii(36)).toBe('data'); + expect(view.getUint32(40, true)).toBe(8); + expect([0, 1, 2, 3].map((i) => view.getInt16(44 + i * 2, true))).toEqual([ + 0, 1000, -1000, 32767 + ]); + }); +}); + +describe('computePeaks', () => { + it('gives the normalised loudest sample of each span', () => { + const samples = Float32Array.from([0.1, -0.2, 0.4, -0.1, 0, 0.05, -0.8, 0.2]); + + expect(computePeaks(samples, 4)).toEqual([0.25, 0.5, 0.0625, 1]); + }); + + it('reads PCM16 the same as floats', () => { + const floats = sine(16_000, 0.2, 200); + const fromPcm = computePeaks(floatToPcm16(floats), 16); + const fromFloats = computePeaks(floats, 16); + + fromPcm.forEach((value, i) => expect(value).toBeCloseTo(fromFloats[i], 3)); + }); + + it('mixes the channels of a buffer', () => { + const left = Float32Array.from([1, 0]); + const right = Float32Array.from([0, 0.5]); + const buffer = { + getChannelData: (c: number) => (c === 0 ? left : right), + length: 2, + numberOfChannels: 2 + } as unknown as AudioBuffer; + + expect(computePeaks(buffer, 2)).toEqual([1, 0.5]); + }); + + it('keeps the count for silence, empty and short clips', () => { + expect(computePeaks(new Float32Array(100), 8)).toEqual(new Array(8).fill(0)); + expect(computePeaks(new Float32Array(0), 8)).toEqual(new Array(8).fill(0)); + expect(computePeaks(Float32Array.from([0.5, -1]), 4)).toEqual([0.5, 0.5, 1, 1]); + }); +}); + +describe('levels', () => { + it('maps loudness in decibels from the floor to full scale', () => { + expect(rmsToLevel(0)).toBe(0); + expect(rmsToLevel(1)).toBe(1); + expect(rmsToLevel(2)).toBe(1); + expect(rmsToLevel(0.001)).toBe(0); // -60 dBFS + expect(rmsToLevel(0.0316)).toBeCloseTo(0.5, 2); // -30 dBFS + }); + + it('keeps a fixed-length history, newest last', () => { + const history = new LevelHistory(3); + + history.push(0.1); + history.push(0.2); + + const levels = history.push(0.3); + + expect(levels).toEqual([0.1, 0.2, 0.3]); + expect(history.push(0.4)).toEqual([0.2, 0.3, 0.4]); + // listeners get copies they can keep + expect(levels).toEqual([0.1, 0.2, 0.3]); + }); +}); diff --git a/ui/apps/chat/tests/unit/voice-realtime.test.ts b/ui/apps/chat/tests/unit/voice-realtime.test.ts new file mode 100644 index 000000000000..be6bfc6623b1 --- /dev/null +++ b/ui/apps/chat/tests/unit/voice-realtime.test.ts @@ -0,0 +1,321 @@ +// Guards the realtime transcription client against llama-server's /v1/realtime +// protocol (tools/server/README.md): the URL, the session flow, deltas and completed +// events, the final commit, the "does not support" error, refused sockets and timeouts. + +import { + base64ToBytes, + RealtimeError, + RealtimeTranscriber, + RealtimeUnsupportedError, + realtimeUrl, + type VoiceServerContext +} from '#lib/voice/index.ts'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +class FakeWebSocket { + closed = false; + + static instances: FakeWebSocket[] = []; + onclose: ((event: { code: number }) => void) | null = null; + onmessage: ((event: { data: string }) => void) | null = null; + sent: Record[] = []; + + close() { + this.closed = true; + } + + constructor(public url: string) { + FakeWebSocket.instances.push(this); + } + + /** The server (or the browser, on a refused upgrade) closes the socket. */ + drop(code = 1006) { + this.onclose?.({ code }); + } + + /** An event from the server. */ + emit(event: Record) { + this.onmessage?.({ data: JSON.stringify(event) }); + } + + send(data: string) { + this.sent.push(JSON.parse(data)); + } +} + +const context: VoiceServerContext = { + baseHref: 'http://localhost:8080/', + resolveUrl: (path) => new URL(path, 'http://localhost:8080/').toString() +}; +const sessionCreated = (type = 'transcription') => ({ + session: { audio: { input: { format: { rate: 24_000, type: 'audio/pcm' } } }, type }, + type: 'session.created' +}); + +function socket(): FakeWebSocket { + return FakeWebSocket.instances.at(-1)!; +} + +beforeEach(() => { + vi.stubGlobal('WebSocket', FakeWebSocket); +}); + +afterEach(() => { + FakeWebSocket.instances = []; + vi.unstubAllGlobals(); + vi.useRealTimers(); +}); + +describe('realtimeUrl', () => { + it('turns the HTTP target into a websocket URL with the intent and model', () => { + expect(realtimeUrl(context, { model: 'org/model:Q4' })).toBe( + 'ws://localhost:8080/v1/realtime?intent=transcription&model=org%2Fmodel%3AQ4' + ); + }); + + it('uses wss for https and resolves relative targets against the page', () => { + const relative: VoiceServerContext = { + baseHref: 'https://example.com/llama/', + resolveUrl: (path) => path + }; + + expect(realtimeUrl(relative, { autoload: false })).toBe( + 'wss://example.com/llama/v1/realtime?intent=transcription&autoload=false' + ); + }); +}); + +describe('RealtimeTranscriber', () => { + it('starts the session, sends the capture rate, and flushes audio queued while connecting', async () => { + const transcriber = new RealtimeTranscriber(context, { language: 'en', sampleRate: 16_000 }); + const connected = transcriber.connect(); + + transcriber.append(Int16Array.from([1, 2])); + expect(socket().sent).toEqual([]); + + socket().emit(sessionCreated()); + await connected; + + expect(transcriber.ready).toBe(true); + expect(transcriber.live).toBe(true); + expect(transcriber.modelSampleRate).toBe(24_000); + expect(socket().sent[0]).toEqual({ + session: { + audio: { + input: { + format: { rate: 16_000, type: 'audio/pcm' }, + transcription: { language: 'en' } + } + }, + type: 'transcription' + }, + type: 'session.update' + }); + expect(socket().sent[1].type).toBe('input_audio_buffer.append'); + expect(Array.from(base64ToBytes(socket().sent[1].audio as string))).toEqual([1, 0, 2, 0]); + + transcriber.append(Int16Array.from([3])); + expect(socket().sent).toHaveLength(3); + }); + + it('streams deltas, ends with a final commit and resolves with the completed transcript', async () => { + const onDelta = vi.fn(); + const onCompleted = vi.fn(); + const transcriber = new RealtimeTranscriber(context, { + onCompleted, + onDelta, + sampleRate: 16_000 + }); + const connected = transcriber.connect(); + + socket().emit(sessionCreated()); + await connected; + transcriber.append(Int16Array.from([1])); + + socket().emit({ delta: 'Hello', type: 'conversation.item.input_audio_transcription.delta' }); + socket().emit({ delta: ' world', type: 'conversation.item.input_audio_transcription.delta' }); + + expect(onDelta).toHaveBeenLastCalledWith(' world', 'Hello world'); + expect(transcriber.text).toBe('Hello world'); + + const finished = transcriber.finish(); + + expect(socket().sent.at(-1)).toEqual({ final: true, type: 'input_audio_buffer.commit' }); + + socket().emit({ + transcript: 'Hello world.', + type: 'conversation.item.input_audio_transcription.completed' + }); + + await expect(finished).resolves.toBe('Hello world.'); + expect(onCompleted).toHaveBeenCalledWith('Hello world.'); + expect(socket().closed).toBe(true); + expect(transcriber.ready).toBe(false); + }); + + it('joins the items completed before the end', async () => { + const transcriber = new RealtimeTranscriber(context, { sampleRate: 16_000 }); + const connected = transcriber.connect(); + + socket().emit(sessionCreated()); + await connected; + transcriber.append(Int16Array.from([1])); + socket().emit({ + transcript: 'First line.\n', + type: 'conversation.item.input_audio_transcription.completed' + }); + socket().emit({ delta: 'Second', type: 'conversation.item.input_audio_transcription.delta' }); + + expect(transcriber.text).toBe('First line.\nSecond'); + + const finished = transcriber.finish(); + + socket().emit({ + transcript: 'Second line.', + type: 'conversation.item.input_audio_transcription.completed' + }); + + await expect(finished).resolves.toBe('First line.\nSecond line.'); + }); + + it('marks a conversation session switched to transcription as not live', async () => { + const transcriber = new RealtimeTranscriber(context, { sampleRate: 16_000 }); + const connected = transcriber.connect(); + + socket().emit(sessionCreated('realtime')); + await connected; + + expect(transcriber.live).toBe(false); + expect((socket().sent[0].session as { type: string }).type).toBe('transcription'); + }); + + it('rejects with RealtimeUnsupportedError when the model cannot stream', async () => { + const transcriber = new RealtimeTranscriber(context, { sampleRate: 16_000 }); + const connected = transcriber.connect(); + + socket().emit({ + error: { + code: null, + message: 'The current model does not support realtime transcription.', + type: 'invalid_request_error' + }, + type: 'error' + }); + socket().drop(1000); + + const error = await connected.catch((e) => e); + + expect(error).toBeInstanceOf(RealtimeUnsupportedError); + expect(error.reason).toBe('model'); + expect(socket().closed).toBe(true); + }); + + it('rejects with a connection RealtimeUnsupportedError when the upgrade is refused', async () => { + const transcriber = new RealtimeTranscriber(context, { sampleRate: 16_000 }); + const connected = transcriber.connect(); + + // e.g. a 401 from a server with an API key: the browser only reports a close + socket().drop(1006); + + const error = await connected.catch((e) => e); + + expect(error).toBeInstanceOf(RealtimeUnsupportedError); + expect(error.reason).toBe('connection'); + }); + + it('passes other errors before the session through as RealtimeError', async () => { + const transcriber = new RealtimeTranscriber(context, { model: 'x', sampleRate: 16_000 }); + const connected = transcriber.connect(); + + socket().emit({ + error: { message: 'model name=x is not running', type: 'server_error' }, + type: 'error' + }); + + const error = await connected.catch((e) => e); + + expect(error).toBeInstanceOf(RealtimeError); + expect(error).not.toBeInstanceOf(RealtimeUnsupportedError); + expect(error.message).toBe('model name=x is not running'); + }); + + it('times out a session that never starts', async () => { + vi.useFakeTimers(); + + const transcriber = new RealtimeTranscriber(context, { + connectTimeoutMs: 1000, + sampleRate: 16_000 + }); + const connected = transcriber.connect().catch((e) => e); + + vi.advanceTimersByTime(1000); + + expect(await connected).toBeInstanceOf(RealtimeError); + expect(socket().closed).toBe(true); + }); + + it('times out a transcript that never completes', async () => { + vi.useFakeTimers(); + + const transcriber = new RealtimeTranscriber(context, { + finishTimeoutMs: 500, + sampleRate: 16_000 + }); + const connected = transcriber.connect(); + + socket().emit(sessionCreated()); + await connected; + transcriber.append(Int16Array.from([1])); + + const finished = transcriber.finish().catch((e) => e); + + vi.advanceTimersByTime(500); + + const error = await finished; + + expect(error).toBeInstanceOf(RealtimeError); + expect(error.message).toMatch(/did not complete/); + }); + + it('rejects finish when the socket closes or errors before the transcript', async () => { + const onError = vi.fn(); + const transcriber = new RealtimeTranscriber(context, { onError, sampleRate: 16_000 }); + const connected = transcriber.connect(); + + socket().emit(sessionCreated()); + await connected; + transcriber.append(Int16Array.from([1])); + + const finished = transcriber.finish(); + + socket().emit({ error: { message: 'server error' }, type: 'error' }); + + await expect(finished).rejects.toThrow('server error'); + expect(onError).toHaveBeenCalledOnce(); + }); + + it('resolves an empty transcript without a commit when no audio was sent', async () => { + const transcriber = new RealtimeTranscriber(context, { sampleRate: 16_000 }); + const connected = transcriber.connect(); + + socket().emit(sessionCreated()); + await connected; + + await expect(transcriber.finish()).resolves.toBe(''); + expect(socket().sent.map((event) => event.type)).toEqual(['session.update']); + expect(socket().closed).toBe(true); + }); + + it('close() rejects a pending connect and stops sending', async () => { + const transcriber = new RealtimeTranscriber(context, { sampleRate: 16_000 }); + const connected = transcriber.connect().catch((e) => e); + const ws = socket(); + + transcriber.close(); + transcriber.append(Int16Array.from([1])); + + expect(await connected).toBeInstanceOf(RealtimeError); + expect(ws.closed).toBe(true); + expect(ws.sent).toEqual([]); + }); +}); diff --git a/ui/apps/chat/tests/unit/voice-session.test.ts b/ui/apps/chat/tests/unit/voice-session.test.ts new file mode 100644 index 000000000000..68f564e9bd80 --- /dev/null +++ b/ui/apps/chat/tests/unit/voice-session.test.ts @@ -0,0 +1,215 @@ +// Guards the voice session: audio goes to the realtime socket while recording and the +// final transcript comes from it; when the model cannot stream, the session falls back +// to one HTTP transcription of the clip without telling the user anything went wrong. + +import { + startVoiceSession, + type VoiceCapture, + type VoiceClip, + type VoicePcmListener, + type VoiceServerContext +} from '#lib/voice/index.ts'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +class FakeWebSocket { + closed = false; + + static instances: FakeWebSocket[] = []; + onclose: ((event: { code: number }) => void) | null = null; + onmessage: ((event: { data: string }) => void) | null = null; + sent: Record[] = []; + + close() { + this.closed = true; + } + + constructor(public url: string) { + FakeWebSocket.instances.push(this); + } + + emit(event: Record) { + this.onmessage?.({ data: JSON.stringify(event) }); + } + + send(data: string) { + this.sent.push(JSON.parse(data)); + } +} + +const clip: VoiceClip = { + blob: new Blob([new Uint8Array(44)], { type: 'audio/wav' }), + durationMs: 1000, + peaks: [0, 1], + sampleRate: 16_000 +}; + +function fakeCapture() { + const pcm = new Set(); + const capture = { + cancel: vi.fn(), + levels: [0, 0], + onLevels: vi.fn(() => () => {}), + onPcm: (listener: VoicePcmListener) => { + pcm.add(listener); + + return () => pcm.delete(listener); + }, + sampleRate: 16_000, + start: vi.fn(async () => {}), + stop: vi.fn(async () => { + // the last chunk reaches the listeners before stop resolves, as in VoiceCapture + for (const listener of pcm) listener(Int16Array.from([9])); + + return clip; + }) + }; + + return { + capture: capture as unknown as VoiceCapture, + emitPcm: (chunk: Int16Array) => pcm.forEach((listener) => listener(chunk)), + raw: capture + }; +} + +function serverContext(fetchImpl: typeof fetch): VoiceServerContext { + return { + baseHref: 'http://localhost/', + fetch: fetchImpl, + resolveUrl: (path) => new URL(path, 'http://localhost/').toString() + }; +} + +const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + +beforeEach(() => { + vi.stubGlobal('WebSocket', FakeWebSocket); +}); + +afterEach(() => { + FakeWebSocket.instances = []; + vi.unstubAllGlobals(); +}); + +describe('startVoiceSession', () => { + it('records only when not asked to transcribe', async () => { + const { capture } = fakeCapture(); + const session = await startVoiceSession({ capture }); + + expect(FakeWebSocket.instances).toHaveLength(0); + await expect(session.stop()).resolves.toEqual({ clip, liveText: null, transcript: null }); + }); + + it('streams to the realtime socket and takes its transcript', async () => { + const fetchMock = vi.fn(); + const onText = vi.fn(); + const onRealtime = vi.fn(); + const { capture, emitPcm } = fakeCapture(); + const session = await startVoiceSession({ + capture, + onRealtime, + onText, + transcribe: { context: serverContext(fetchMock as typeof fetch), model: 'asr' } + }); + const ws = FakeWebSocket.instances[0]; + + expect(ws.url).toBe('ws://localhost/v1/realtime?intent=transcription&model=asr'); + + emitPcm(Int16Array.from([1])); + ws.emit({ session: { type: 'transcription' }, type: 'session.created' }); + await flush(); + + expect(onRealtime).toHaveBeenCalledWith(true); + + ws.emit({ delta: 'Hi', type: 'conversation.item.input_audio_transcription.delta' }); + expect(onText).toHaveBeenLastCalledWith('Hi'); + + const result = await session.stop(); + + expect(result.clip).toBe(clip); + expect(result.liveText).toBe('Hi'); + // the queued chunk, the last chunk from stop(), then the final commit + expect(ws.sent.map((event) => event.type)).toEqual([ + 'session.update', + 'input_audio_buffer.append', + 'input_audio_buffer.append', + 'input_audio_buffer.commit' + ]); + + ws.emit({ transcript: 'Hi.', type: 'conversation.item.input_audio_transcription.completed' }); + + await expect(result.transcript).resolves.toBe('Hi.'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('falls back to HTTP when the model cannot stream', async () => { + const fetchMock = vi.fn(async () => new Response(JSON.stringify({ text: 'From HTTP.' }))); + const onRealtime = vi.fn(); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const { capture } = fakeCapture(); + const session = await startVoiceSession({ + capture, + onRealtime, + transcribe: { context: serverContext(fetchMock as typeof fetch), model: 'omni' } + }); + + FakeWebSocket.instances[0].emit({ + error: { message: 'The current model does not support realtime transcription.' }, + type: 'error' + }); + await flush(); + + expect(onRealtime).toHaveBeenCalledWith(false); + // an expected fallback, not worth a warning + expect(warn).not.toHaveBeenCalled(); + + const result = await session.stop(); + + expect(result.liveText).toBeNull(); + await expect(result.transcript).resolves.toBe('From HTTP.'); + expect(fetchMock).toHaveBeenCalledOnce(); + + const form = (fetchMock.mock.calls[0] as unknown as [string, RequestInit])[1].body as FormData; + + expect(form.get('model')).toBe('omni'); + warn.mockRestore(); + }); + + it('uses HTTP when realtime is still connecting at the end', async () => { + const fetchMock = vi.fn(async () => new Response(JSON.stringify({ text: 'Late.' }))); + const { capture } = fakeCapture(); + const session = await startVoiceSession({ + capture, + transcribe: { context: serverContext(fetchMock as typeof fetch) } + }); + const result = await session.stop(); + + expect(FakeWebSocket.instances[0].closed).toBe(true); + await expect(result.transcript).resolves.toBe('Late.'); + }); + + it('skips the socket when realtime is off', async () => { + const fetchMock = vi.fn(async () => new Response(JSON.stringify({ text: 'ok' }))); + const { capture } = fakeCapture(); + const session = await startVoiceSession({ + capture, + realtime: false, + transcribe: { context: serverContext(fetchMock as typeof fetch) } + }); + + expect(FakeWebSocket.instances).toHaveLength(0); + await expect((await session.stop()).transcript).resolves.toBe('ok'); + }); + + it('cancel drops the capture and the socket', async () => { + const { capture, raw } = fakeCapture(); + const session = await startVoiceSession({ + capture, + transcribe: { context: serverContext(vi.fn() as unknown as typeof fetch) } + }); + + session.cancel(); + + expect(raw.cancel).toHaveBeenCalledOnce(); + expect(FakeWebSocket.instances[0].closed).toBe(true); + }); +}); diff --git a/ui/apps/chat/tests/unit/voice-transcribe.test.ts b/ui/apps/chat/tests/unit/voice-transcribe.test.ts new file mode 100644 index 000000000000..c1853dbee3ad --- /dev/null +++ b/ui/apps/chat/tests/unit/voice-transcribe.test.ts @@ -0,0 +1,116 @@ +// Guards the HTTP transcription client: the multipart request +// /v1/audio/transcriptions expects, auth headers, and the server's error messages. + +import { + transcribeClip, + type VoiceServerContext, + VoiceTranscriptionError +} from '#lib/voice/index.ts'; +import { describe, expect, it, vi } from 'vitest'; + +function contextWith(fetchImpl: typeof fetch): VoiceServerContext { + return { + fetch: fetchImpl, + getHeaders: () => ({ Authorization: 'Bearer secret', 'Content-Type': 'application/json' }), + resolveUrl: (path) => `http://server${path.replace(/^\./, '')}` + }; +} + +describe('transcribeClip', () => { + it('posts the clip as multipart with the model and json format', async () => { + const fetchMock = vi.fn( + async (_url: RequestInfo | URL, _init?: RequestInit) => + new Response(JSON.stringify({ text: ' Hello there. ' }), { status: 200 }) + ); + const blob = new Blob([new Uint8Array([1, 2, 3])], { type: 'audio/wav' }); + const text = await transcribeClip(contextWith(fetchMock as typeof fetch), blob, { + language: 'en', + model: 'org/model:Q8_0' + }); + + expect(text).toBe('Hello there.'); + + const [url, init] = fetchMock.mock.calls[0]; + const form = init!.body as FormData; + const file = form.get('file') as File; + + expect(url).toBe('http://server/v1/audio/transcriptions'); + expect(init!.method).toBe('POST'); + // the browser sets the multipart Content-Type with its boundary + expect(init!.headers).toEqual({ Authorization: 'Bearer secret' }); + expect(file.name).toBe('audio.wav'); + expect(file.type).toBe('audio/wav'); + expect(new Uint8Array(await file.arrayBuffer())).toEqual(new Uint8Array([1, 2, 3])); + expect(form.get('model')).toBe('org/model:Q8_0'); + expect(form.get('language')).toBe('en'); + expect(form.get('response_format')).toBe('json'); + expect(form.has('prompt')).toBe(false); + }); + + it('leaves the model out for a single-model server', async () => { + const fetchMock = vi.fn( + async (_url: RequestInfo | URL, _init?: RequestInit) => + new Response(JSON.stringify({ text: 'ok' })) + ); + + await transcribeClip(contextWith(fetchMock as typeof fetch), new Blob([])); + + expect((fetchMock.mock.calls[0][1]!.body as FormData).has('model')).toBe(false); + }); + + it("surfaces the server's error message and status", async () => { + const fetchMock = vi.fn( + async () => + new Response( + JSON.stringify({ + error: { + code: 501, + message: 'The current model does not support audio input.', + type: 'not_supported_error' + } + }), + { status: 501 } + ) + ); + const error = await transcribeClip(contextWith(fetchMock as typeof fetch), new Blob([])).catch( + (e) => e + ); + + expect(error).toBeInstanceOf(VoiceTranscriptionError); + expect(error.message).toBe('The current model does not support audio input.'); + expect(error.status).toBe(501); + }); + + it('falls back to the status for bodies that say nothing useful', async () => { + const fetchMock = vi.fn( + async () => + new Response('Bad Gateway', { status: 502, statusText: 'Bad Gateway' }) + ); + + await expect( + transcribeClip(contextWith(fetchMock as typeof fetch), new Blob([])) + ).rejects.toThrow('Transcription failed (502 Bad Gateway)'); + }); + + it('rejects a response without text', async () => { + const fetchMock = vi.fn(async () => new Response(JSON.stringify({ choices: [] }))); + + await expect( + transcribeClip(contextWith(fetchMock as typeof fetch), new Blob([])) + ).rejects.toBeInstanceOf(VoiceTranscriptionError); + }); + + it('passes the abort signal on', async () => { + const controller = new AbortController(); + const fetchMock = vi.fn( + async (_url: RequestInfo | URL, _init?: RequestInit) => + new Response(JSON.stringify({ text: '' })) + ); + + await transcribeClip(contextWith(fetchMock as typeof fetch), new Blob([]), { + signal: controller.signal + }); + + expect(fetchMock.mock.calls[0][1]!.signal).toBe(controller.signal); + }); +}); From a7c2576a9dc5b5c8f7838396f2907be9fbbc8555 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Sun, 15 Nov 2026 21:13:01 +0100 Subject: [PATCH 02/19] ui: add voice waveform and transcript components to Llama UI Voice input needs three visuals that the app can feed with its own recording and transcription logic: the live waveform while recording, a playable waveform of a recorded clip, and a clip with its transcript that opens on hover, keyboard focus or a tap. VoiceWaveformLive draws input levels as many bars as fit the width, newest on the right, and holds still with reduced motion. VoiceWaveform plays a clip through its own audio element, fills the played part in the primary colour, seeks by press, drag and arrow keys (a slider) and pauses any other clip that was playing. VoiceTranscriptReveal keeps the transcript in the DOM for screen readers while it is closed, and shows transcribing and error states. The waveform math (resampling peaks, bar capacity, seek and time formatting) is in src/lib/voice with unit tests. The components are exported from the package root and from @ggml-org/llama-ui/voice, which Llama Chat resolves to the sources like the other entry points. Assisted-by: Claude (Opus 5.5 Medium) --- ui/apps/chat/DESIGN.md | 6 + ui/apps/chat/scripts/llama-ui.js | 1 + ui/apps/chat/tsconfig.json | 1 + ui/lib/ui/README.md | 4 + ui/lib/ui/package.json | 4 + .../voice/VoiceTranscriptReveal.svelte | 147 ++++++++++ .../lib/components/voice/VoiceWaveform.svelte | 258 ++++++++++++++++++ .../components/voice/VoiceWaveformLive.svelte | 59 ++++ ui/lib/ui/src/lib/components/voice/index.ts | 9 + ui/lib/ui/src/lib/index.ts | 1 + ui/lib/ui/src/lib/voice/index.ts | 2 + ui/lib/ui/src/lib/voice/types.ts | 5 + ui/lib/ui/src/lib/voice/waveform.spec.ts | 97 +++++++ ui/lib/ui/src/lib/voice/waveform.ts | 106 +++++++ 14 files changed, 700 insertions(+) create mode 100644 ui/lib/ui/src/lib/components/voice/VoiceTranscriptReveal.svelte create mode 100644 ui/lib/ui/src/lib/components/voice/VoiceWaveform.svelte create mode 100644 ui/lib/ui/src/lib/components/voice/VoiceWaveformLive.svelte create mode 100644 ui/lib/ui/src/lib/components/voice/index.ts create mode 100644 ui/lib/ui/src/lib/voice/index.ts create mode 100644 ui/lib/ui/src/lib/voice/types.ts create mode 100644 ui/lib/ui/src/lib/voice/waveform.spec.ts create mode 100644 ui/lib/ui/src/lib/voice/waveform.ts diff --git a/ui/apps/chat/DESIGN.md b/ui/apps/chat/DESIGN.md index 971685e1bc51..79e5215286f3 100644 --- a/ui/apps/chat/DESIGN.md +++ b/ui/apps/chat/DESIGN.md @@ -1151,6 +1151,12 @@ icon buttons in the chat form are fully round. (`--sidebar-reveal-x`), taking no taps while the page is aside. The page keeps room under its last row (`max-md:pb-safe-24`) so the key never covers it. One per page, only for its main action; from `md` up the action stays a button in the header. +- **Voice clips:** a recorded clip is a round primary play key, its waveform and the time in + `caption`: bars in the muted foreground at 40%, the played part in the primary colour, so + the clip stays neutral in both modes. The live waveform while recording is as tall as the + input's line (24px), the newest level on the right, and holds still with reduced motion. A + voice message's transcript opens under the clip on hover, keyboard focus or a tap, and + stays in the DOM for screen readers while closed. - **Copy feedback:** a copy action turns its icon into a `fg-success` check for 1.5s (`COPY_FEEDBACK_RESET_MS`), only when the copy succeeded, then back. The icon pops in with a 150ms fade and scale, skipped on first render and with reduced motion. diff --git a/ui/apps/chat/scripts/llama-ui.js b/ui/apps/chat/scripts/llama-ui.js index c4eb26af77c9..f8963593a33b 100644 --- a/ui/apps/chat/scripts/llama-ui.js +++ b/ui/apps/chat/scripts/llama-ui.js @@ -26,6 +26,7 @@ const LLAMA_UI_EXPORTS = [ ['@ggml-org/llama-ui/enums', `${source}/enums/index.ts`], ['@ggml-org/llama-ui/code', `${source}/utils/index.ts`], ['@ggml-org/llama-ui/utils', `${source}/components/ui/utils.ts`], + ['@ggml-org/llama-ui/voice', `${source}/components/voice/index.ts`], ['@ggml-org/llama-ui/styles/*', `${source}/styles/*`], ['@ggml-org/llama-ui/theme.css', `${source}/styles/theme.css`], ['@ggml-org/llama-ui/tokens.css', `${source}/styles/tokens.css`], diff --git a/ui/apps/chat/tsconfig.json b/ui/apps/chat/tsconfig.json index 4d4c701d3ec8..b91dbef9b853 100644 --- a/ui/apps/chat/tsconfig.json +++ b/ui/apps/chat/tsconfig.json @@ -9,6 +9,7 @@ "@ggml-org/llama-ui/enums": ["../../lib/ui/src/lib/enums/index.ts"], "@ggml-org/llama-ui/code": ["../../lib/ui/src/lib/utils/index.ts"], "@ggml-org/llama-ui/utils": ["../../lib/ui/src/lib/components/ui/utils.ts"], + "@ggml-org/llama-ui/voice": ["../../lib/ui/src/lib/components/voice/index.ts"], "@ggml-org/llama-ui/styles/*": ["../../lib/ui/src/lib/styles/*"], "@ggml-org/llama-ui/theme.css": ["../../lib/ui/src/lib/styles/theme.css"], "@ggml-org/llama-ui/tokens.css": ["../../lib/ui/src/lib/styles/tokens.css"], diff --git a/ui/lib/ui/README.md b/ui/lib/ui/README.md index 2c701d8a6331..2996c478a4b3 100644 --- a/ui/lib/ui/README.md +++ b/ui/lib/ui/README.md @@ -44,6 +44,10 @@ beside the apps built on it in `ui/apps`. `ModelQuantOptions`, `ModelQuantChip`, `ModelQuantRow`, `ModelDraftSidecars`, `ModelDownloadProgressBar`, plus the formatters and `minMemoryTierGb` and `summarizeSizes`, at `@ggml-org/llama-ui/models`. They take model data in and report what the user did out; fetching and downloading stay in the app. +- **Voice UI** (also `@ggml-org/llama-ui/voice`): `VoiceWaveformLive` (the waveform while recording), `VoiceWaveform` (a + recorded clip: play, seek, time) and `VoiceTranscriptReveal` (a clip with its transcript, opened on hover, keyboard + focus or a tap), plus `formatClipTime` and the waveform math. They take levels, peaks, an audio URL and a transcript + in; recording and transcribing stay in the app. Next: llama.app's model page. diff --git a/ui/lib/ui/package.json b/ui/lib/ui/package.json index ff86bd63eade..e8b56071c45d 100644 --- a/ui/lib/ui/package.json +++ b/ui/lib/ui/package.json @@ -54,6 +54,10 @@ "types": "./dist/components/models/index.d.ts", "svelte": "./dist/components/models/index.js" }, + "./voice": { + "types": "./dist/components/voice/index.d.ts", + "svelte": "./dist/components/voice/index.js" + }, "./utils": { "types": "./dist/components/ui/utils.d.ts", "svelte": "./dist/components/ui/utils.js" diff --git a/ui/lib/ui/src/lib/components/voice/VoiceTranscriptReveal.svelte b/ui/lib/ui/src/lib/components/voice/VoiceTranscriptReveal.svelte new file mode 100644 index 000000000000..a5ed97addd42 --- /dev/null +++ b/ui/lib/ui/src/lib/components/voice/VoiceTranscriptReveal.svelte @@ -0,0 +1,147 @@ + + + + + +
+
+
+ {@render children()} +
+ + {#if showToggle} + + {/if} +
+ + {#if hasContent} +
+
+
+ {#if shown === 'transcribing'} +
+ Transcribing... + +
+ {:else if shown === 'error'} +

Transcription unavailable

+ {:else if transcript} +

+ Transcript: {transcript} +

+ {:else} +

No speech detected

+ {/if} +
+
+
+ {/if} +
diff --git a/ui/lib/ui/src/lib/components/voice/VoiceWaveform.svelte b/ui/lib/ui/src/lib/components/voice/VoiceWaveform.svelte new file mode 100644 index 000000000000..5cceabcb7099 --- /dev/null +++ b/ui/lib/ui/src/lib/components/voice/VoiceWaveform.svelte @@ -0,0 +1,258 @@ + + + + + +{#snippet barRow(className: string)} + +{/snippet} + +
+ + +
(dragging = false)} + onpointercancel={handlePointerEnd} + onpointerdown={handlePointerDown} + onpointermove={handlePointerMove} + onpointerup={handlePointerEnd} + role="slider" + tabindex={playable ? 0 : -1} + > + {@render barRow('text-muted-foreground/40')} + +
+ {@render barRow('')} +
+
+ + + + {#if src} + + {/if} +
diff --git a/ui/lib/ui/src/lib/components/voice/VoiceWaveformLive.svelte b/ui/lib/ui/src/lib/components/voice/VoiceWaveformLive.svelte new file mode 100644 index 000000000000..315326519f7a --- /dev/null +++ b/ui/lib/ui/src/lib/components/voice/VoiceWaveformLive.svelte @@ -0,0 +1,59 @@ + + + +
+ {#if label} + {label} + {/if} + + +
diff --git a/ui/lib/ui/src/lib/components/voice/index.ts b/ui/lib/ui/src/lib/components/voice/index.ts new file mode 100644 index 000000000000..a2ed351287a7 --- /dev/null +++ b/ui/lib/ui/src/lib/components/voice/index.ts @@ -0,0 +1,9 @@ +/** + * Voice UI: how a voice clip is shown while it is recorded, once it is recorded, and with its transcript. Every + * component takes its data in (levels, peaks, an audio URL, a transcript); recording, analysing and transcribing stay + * in each app. + */ +export { default as VoiceTranscriptReveal } from "./VoiceTranscriptReveal.svelte"; +export { default as VoiceWaveform } from "./VoiceWaveform.svelte"; +export { default as VoiceWaveformLive } from "./VoiceWaveformLive.svelte"; +export * from "../../voice/index.js"; diff --git a/ui/lib/ui/src/lib/index.ts b/ui/lib/ui/src/lib/index.ts index 8fe7855e9b8a..8ad7f8542396 100644 --- a/ui/lib/ui/src/lib/index.ts +++ b/ui/lib/ui/src/lib/index.ts @@ -39,3 +39,4 @@ export * as ToggleGroup from './components/ui/toggle-group/index.js'; export * as Tooltip from './components/ui/tooltip/index.js'; export * from './components/mockups/index.js'; export * from './components/models/index.js'; +export * from './components/voice/index.js'; diff --git a/ui/lib/ui/src/lib/voice/index.ts b/ui/lib/ui/src/lib/voice/index.ts new file mode 100644 index 000000000000..dd0b40018d30 --- /dev/null +++ b/ui/lib/ui/src/lib/voice/index.ts @@ -0,0 +1,2 @@ +export * from './waveform.js'; +export type * from './types.js'; diff --git a/ui/lib/ui/src/lib/voice/types.ts b/ui/lib/ui/src/lib/voice/types.ts new file mode 100644 index 000000000000..521f93c12876 --- /dev/null +++ b/ui/lib/ui/src/lib/voice/types.ts @@ -0,0 +1,5 @@ +/** Where a voice clip's transcript is: none asked for, on its way, there, or failed. */ +export type VoiceTranscriptStatus = 'idle' | 'transcribing' | 'done' | 'error'; + +/** `sm` for a chip in the chat form, `md` for a message bubble. */ +export type VoiceWaveformSize = 'sm' | 'md'; diff --git a/ui/lib/ui/src/lib/voice/waveform.spec.ts b/ui/lib/ui/src/lib/voice/waveform.spec.ts new file mode 100644 index 000000000000..0674d506a66d --- /dev/null +++ b/ui/lib/ui/src/lib/voice/waveform.spec.ts @@ -0,0 +1,97 @@ +import { + MIN_BAR_HEIGHT, + barCapacity, + barHeightPercent, + formatClipPosition, + formatClipTime, + liveBars, + pointerFraction, + progressFraction, + resamplePeaks, + stepPosition +} from './waveform.js'; +import { describe, expect, it } from 'vite-plus/test'; + +describe('voice clip times', () => { + it('formats m:ss, rounded to the second', () => { + expect(formatClipTime(0)).toBe('0:00'); + expect(formatClipTime(7_000)).toBe('0:07'); + expect(formatClipTime(600)).toBe('0:01'); + expect(formatClipTime(65_400)).toBe('1:05'); + expect(formatClipTime(4_350_000)).toBe('72:30'); + }); + + it('treats negative and non-finite times as zero', () => { + expect(formatClipTime(-5)).toBe('0:00'); + expect(formatClipTime(Number.NaN)).toBe('0:00'); + expect(formatClipTime(Number.POSITIVE_INFINITY)).toBe('0:00'); + }); + + it('says the position for screen readers', () => { + expect(formatClipPosition(12_000, 65_000)).toBe('0:12 of 1:05'); + }); +}); + +describe('waveform bars', () => { + it('counts the bars that fit a width', () => { + expect(barCapacity(100, 3, 2)).toBe(20); + expect(barCapacity(98, 3, 2)).toBe(20); + expect(barCapacity(97, 3, 2)).toBe(19); + expect(barCapacity(0, 3, 2)).toBe(0); + expect(barCapacity(Number.NaN, 3, 2)).toBe(0); + }); + + it('keeps peaks as they are when the count matches, clamped to 0..1', () => { + expect(resamplePeaks([0.2, 1.4, -1, Number.NaN], 4)).toEqual([0.2, 1, 0, 0]); + }); + + it('keeps the loudest peak of each bucket when there are fewer bars', () => { + expect(resamplePeaks([0.1, 0.9, 0.2, 0.3, 0.8, 0.1], 3)).toEqual([0.9, 0.3, 0.8]); + expect(resamplePeaks([0.1, 0.2, 0.7, 0.4, 0.5], 2)).toEqual([0.2, 0.7]); + }); + + it('interpolates when there are more bars', () => { + expect(resamplePeaks([0, 1], 5)).toEqual([0, 0.25, 0.5, 0.75, 1]); + expect(resamplePeaks([0.4], 3)).toEqual([0.4, 0.4, 0.4]); + }); + + it('draws a flat line without peaks', () => { + expect(resamplePeaks([], 3)).toEqual([0, 0, 0]); + expect(resamplePeaks([0.5], 0)).toEqual([]); + }); + + it('fills the live width with the newest levels last', () => { + expect(liveBars([0.1, 0.2, 0.3], 5)).toEqual([null, null, 0.1, 0.2, 0.3]); + expect(liveBars([0.1, 0.2, 0.3, 2], 2)).toEqual([0.3, 1]); + expect(liveBars([0.5], 0)).toEqual([]); + }); + + it('never draws a bar under the resting height', () => { + expect(barHeightPercent(null)).toBe(MIN_BAR_HEIGHT * 100); + expect(barHeightPercent(0)).toBe(MIN_BAR_HEIGHT * 100); + expect(barHeightPercent(0.5)).toBe(50); + expect(barHeightPercent(3)).toBe(100); + }); +}); + +describe('seeking', () => { + it('gives the played fraction', () => { + expect(progressFraction(15_000, 60_000)).toBe(0.25); + expect(progressFraction(90_000, 60_000)).toBe(1); + expect(progressFraction(1_000, 0)).toBe(0); + }); + + it('maps a pointer to a fraction of the track', () => { + expect(pointerFraction(150, 100, 200)).toBe(0.25); + expect(pointerFraction(50, 100, 200)).toBe(0); + expect(pointerFraction(400, 100, 200)).toBe(1); + expect(pointerFraction(150, 100, 0)).toBe(0); + }); + + it('steps the position inside the clip', () => { + expect(stepPosition(10_000, 5_000, 60_000)).toBe(15_000); + expect(stepPosition(3_000, -5_000, 60_000)).toBe(0); + expect(stepPosition(58_000, 5_000, 60_000)).toBe(60_000); + expect(stepPosition(1_000, 5_000, 0)).toBe(0); + }); +}); diff --git a/ui/lib/ui/src/lib/voice/waveform.ts b/ui/lib/ui/src/lib/voice/waveform.ts new file mode 100644 index 000000000000..85f10a3c919d --- /dev/null +++ b/ui/lib/ui/src/lib/voice/waveform.ts @@ -0,0 +1,106 @@ +/** The smallest bar, as a fraction of the waveform's height, so silence still draws a dotted line. */ +export const MIN_BAR_HEIGHT = 0.12; + +function clampUnit(value: number): number { + if (!Number.isFinite(value)) return 0; + + return Math.min(Math.max(value, 0), 1); +} + +/** A clip time as m:ss (`0:07`, `1:05`, `72:30`), rounded to the nearest second. */ +export function formatClipTime(ms: number): string { + const seconds = Number.isFinite(ms) && ms > 0 ? Math.round(ms / 1000) : 0; + + return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, '0')}`; +} + +/** What a screen reader hears for a clip's position, e.g. `0:12 of 1:05`. */ +export function formatClipPosition(elapsedMs: number, totalMs: number): string { + return `${formatClipTime(elapsedMs)} of ${formatClipTime(totalMs)}`; +} + +/** How many bars of `barWidth` with `gap` between them fit in `width` (all in px). */ +export function barCapacity(width: number, barWidth: number, gap: number): number { + if (!(width > 0) || !(barWidth > 0)) return 0; + + return Math.max(Math.floor((width + gap) / (barWidth + gap)), 0); +} + +/** + * Peaks (0..1) resampled to `count` bars: fewer bars keep the loudest peak of each bucket, so short sounds do not + * vanish; more bars interpolate between neighbours. + */ +export function resamplePeaks(peaks: readonly number[], count: number): number[] { + if (count <= 0) return []; + if (peaks.length === 0) return Array.from({ length: count }, () => 0); + if (peaks.length === count) return peaks.map(clampUnit); + + if (count < peaks.length) { + const bars: number[] = []; + + for (let i = 0; i < count; i++) { + const start = Math.floor((i * peaks.length) / count); + const end = Math.max(Math.floor(((i + 1) * peaks.length) / count), start + 1); + let peak = 0; + + for (let j = start; j < end; j++) peak = Math.max(peak, clampUnit(peaks[j])); + + bars.push(peak); + } + + return bars; + } + + if (peaks.length === 1) return Array.from({ length: count }, () => clampUnit(peaks[0])); + + const bars: number[] = []; + + for (let i = 0; i < count; i++) { + const position = (i * (peaks.length - 1)) / (count - 1); + const left = Math.floor(position); + const right = Math.min(left + 1, peaks.length - 1); + const t = position - left; + + bars.push(clampUnit(peaks[left]) * (1 - t) + clampUnit(peaks[right]) * t); + } + + return bars; +} + +/** + * The last `count` live levels, newest last, filling the width: missing older bars are `null` (drawn as the + * resting dot). + */ +export function liveBars(levels: readonly number[], count: number): (number | null)[] { + if (count <= 0) return []; + + const recent = levels.slice(-count).map(clampUnit); + + return [...Array.from({ length: count - recent.length }, () => null), ...recent]; +} + +/** A bar's height in percent of the waveform's, never under `MIN_BAR_HEIGHT`. */ +export function barHeightPercent(level: number | null): number { + return Math.max(clampUnit(level ?? 0), MIN_BAR_HEIGHT) * 100; +} + +/** How far through the clip playback is, 0..1. */ +export function progressFraction(elapsedMs: number, totalMs: number): number { + if (!(totalMs > 0)) return 0; + + return clampUnit(elapsedMs / totalMs); +} + +/** Where a pointer at `clientX` falls along a track starting at `left`, 0..1. */ +export function pointerFraction(clientX: number, left: number, width: number): number { + if (!(width > 0)) return 0; + + return clampUnit((clientX - left) / width); +} + +/** A position moved by `deltaMs`, kept inside the clip. */ +export function stepPosition(elapsedMs: number, deltaMs: number, totalMs: number): number { + if (!(totalMs > 0)) return 0; + + return Math.min(Math.max(elapsedMs + deltaMs, 0), totalMs); +} From 6b712ca9bd56e0737e762bf5865c86ec3ec02058 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Sun, 15 Nov 2026 21:13:01 +0100 Subject: [PATCH 03/19] ui: add stories and tests for the voice components Storybook gets UI/Blocks/Voice: the live waveform fed by a fake microphone meter in a chat form row, the clip waveform in its sizes and states, and the transcript reveal in each state and on touch. The demo audio is a WAV synthesised in the story, so no binary file is added. The play functions check seeking by keys and by press, the reveal on hover, keyboard focus and a tap, and a clip without audio. Pressing play needs a real click before a browser plays audio, so a client test covers play and pause and one clip pausing another. Assisted-by: Claude (Opus 5.5 Medium) --- .../client/voice-waveform.svelte.test.ts | 39 +++++ .../tests/stories/voice/VoiceLiveDemo.svelte | 54 +++++++ .../VoiceTranscriptReveal.stories.svelte | 137 ++++++++++++++++ .../voice/VoiceWaveform.stories.svelte | 150 ++++++++++++++++++ .../voice/VoiceWaveformLive.stories.svelte | 82 ++++++++++ ui/apps/chat/tests/stories/voice/fixtures.ts | 100 ++++++++++++ 6 files changed, 562 insertions(+) create mode 100644 ui/apps/chat/tests/client/voice-waveform.svelte.test.ts create mode 100644 ui/apps/chat/tests/stories/voice/VoiceLiveDemo.svelte create mode 100644 ui/apps/chat/tests/stories/voice/VoiceTranscriptReveal.stories.svelte create mode 100644 ui/apps/chat/tests/stories/voice/VoiceWaveform.stories.svelte create mode 100644 ui/apps/chat/tests/stories/voice/VoiceWaveformLive.stories.svelte create mode 100644 ui/apps/chat/tests/stories/voice/fixtures.ts diff --git a/ui/apps/chat/tests/client/voice-waveform.svelte.test.ts b/ui/apps/chat/tests/client/voice-waveform.svelte.test.ts new file mode 100644 index 000000000000..202907756915 --- /dev/null +++ b/ui/apps/chat/tests/client/voice-waveform.svelte.test.ts @@ -0,0 +1,39 @@ +// Guards VoiceWaveform's playback with real clicks, which a browser needs before it plays audio (the stories' clicks +// are scripted, so their play functions cover the rest): play and pause toggle, and one clip plays at a time. + +import { demoClip } from '../stories/voice/fixtures'; +import { VoiceWaveform } from '@ggml-org/llama-ui/voice'; +import { describe, expect, it } from 'vitest'; +import { page, userEvent } from 'vitest/browser'; +import { render } from 'vitest-browser-svelte'; + +describe('VoiceWaveform playback', () => { + it('toggles between play and pause', async () => { + const clip = demoClip(3, 1); + + render(VoiceWaveform, { durationMs: clip.durationMs, peaks: clip.peaks, src: clip.src }); + + await userEvent.click(page.getByRole('button', { name: 'Play voice message' })); + await expect.element(page.getByRole('button', { name: 'Pause voice message' })).toBeVisible(); + + await userEvent.click(page.getByRole('button', { name: 'Pause voice message' })); + await expect.element(page.getByRole('button', { name: 'Play voice message' })).toBeVisible(); + }); + + it('pauses one clip when another starts', async () => { + const first = demoClip(3, 2); + const second = demoClip(3, 3); + + render(VoiceWaveform, { label: 'First clip', peaks: first.peaks, src: first.src }); + render(VoiceWaveform, { label: 'Second clip', peaks: second.peaks, src: second.src }); + + await userEvent.click(page.getByRole('button', { name: 'Play first clip' })); + await expect.element(page.getByRole('button', { name: 'Pause first clip' })).toBeVisible(); + + await userEvent.click(page.getByRole('button', { name: 'Play second clip' })); + await expect.element(page.getByRole('button', { name: 'Pause second clip' })).toBeVisible(); + await expect.element(page.getByRole('button', { name: 'Play first clip' })).toBeVisible(); + + await userEvent.click(page.getByRole('button', { name: 'Pause second clip' })); + }); +}); diff --git a/ui/apps/chat/tests/stories/voice/VoiceLiveDemo.svelte b/ui/apps/chat/tests/stories/voice/VoiceLiveDemo.svelte new file mode 100644 index 000000000000..c370f377496a --- /dev/null +++ b/ui/apps/chat/tests/stories/voice/VoiceLiveDemo.svelte @@ -0,0 +1,54 @@ + + + +
+ + + + {formatClipTime(elapsedMs)} + + + +
diff --git a/ui/apps/chat/tests/stories/voice/VoiceTranscriptReveal.stories.svelte b/ui/apps/chat/tests/stories/voice/VoiceTranscriptReveal.stories.svelte new file mode 100644 index 000000000000..9ec5d412ced7 --- /dev/null +++ b/ui/apps/chat/tests/stories/voice/VoiceTranscriptReveal.stories.svelte @@ -0,0 +1,137 @@ + + +{#snippet message(status?: VoiceTranscriptStatus, transcript: string | null = TRANSCRIPT)} +
+ + + +
+{/snippet} + + + + + {@render message()} + + + + {@render message('transcribing', null)} + + + + {@render message('error', null)} + + + + {@render message('idle', null)} + + + + {@render message('done', LONG_TRANSCRIPT)} + + + + {@render message('done', '')} + + + + + + { + const root = reveal(canvasElement); + + // closed, but in the DOM for screen readers + await expect(canvas.getByText(TRANSCRIPT)).toBeInTheDocument(); + await expect(root).not.toHaveAttribute('data-revealed'); + await expect(getComputedStyle(region(canvasElement)).opacity).toBe('0'); + + await userEvent.hover(root); + await waitFor(() => expect(root).toHaveAttribute('data-revealed')); + await waitFor(() => expect(getComputedStyle(region(canvasElement)).opacity).toBe('1')); + + await userEvent.unhover(root); + await waitFor(() => expect(root).not.toHaveAttribute('data-revealed')); + + await userEvent.tab(); + await expect(canvas.getByRole('button', { name: 'Play voice message' })).toHaveFocus(); + await waitFor(() => expect(root).toHaveAttribute('data-revealed')); + + await userEvent.tab(); + await expect(canvas.getByRole('slider')).toHaveFocus(); + await expect(root).toHaveAttribute('data-revealed'); + + await userEvent.tab(); + await waitFor(() => expect(root).not.toHaveAttribute('data-revealed')); + }} +> +
+ {@render message()} + + +
+
+ + + { + const root = reveal(canvasElement); + const time = canvas.getByText('0:06'); + + await userEvent.pointer({ keys: '[TouchA]', target: time }); + await waitFor(() => expect(root).toHaveAttribute('data-revealed')); + + const hide = canvas.getByRole('button', { name: 'Hide transcript' }); + + await expect(hide).toHaveAttribute('aria-expanded', 'true'); + await userEvent.pointer({ keys: '[TouchA]', target: hide }); + await waitFor(() => expect(root).not.toHaveAttribute('data-revealed')); + }} +> + + + {@render message()} + + + diff --git a/ui/apps/chat/tests/stories/voice/VoiceWaveform.stories.svelte b/ui/apps/chat/tests/stories/voice/VoiceWaveform.stories.svelte new file mode 100644 index 000000000000..da220ac2d18b --- /dev/null +++ b/ui/apps/chat/tests/stories/voice/VoiceWaveform.stories.svelte @@ -0,0 +1,150 @@ + + + + + +
+ +
+
+ + +
+ +
+
+
+
+ + + + +
+ +
+
+ + +
+ +
+
+ + +
+ +
+
+ + +
+ +
+
+
+
+ + + { + await expect(canvas.getByRole('button', { name: 'Play first clip' })).toBeEnabled(); + await expect(canvas.getByRole('button', { name: 'Play second clip' })).toBeEnabled(); + + const slider = canvas.getByRole('slider', { name: 'Long clip position' }); + + slider.focus(); + await userEvent.keyboard('{End}'); + await expect(slider).toHaveAttribute('aria-valuenow', '135'); + await userEvent.keyboard('{ArrowLeft}'); + await expect(slider).toHaveAttribute('aria-valuenow', '130'); + await expect(slider).toHaveAttribute('aria-valuetext', '2:10 of 2:15'); + await userEvent.keyboard('{Home}'); + await userEvent.keyboard('{ArrowRight}'); + await expect(slider).toHaveAttribute('aria-valuenow', '5'); + + // a press halfway along the bars seeks to the middle + const rect = slider.getBoundingClientRect(); + + await userEvent.pointer({ + coords: { clientX: rect.left + rect.width / 2, clientY: rect.top + rect.height / 2 }, + keys: '[MouseLeft]', + target: slider + }); + await expect(Number(slider.getAttribute('aria-valuenow'))).toBeCloseTo(67.5, -1); + }} +> +
+
+ +
+ +
+ +
+ +
+ +
+
+
+ + + { + await expect(canvas.getByRole('button', { name: 'Play voice message' })).toBeDisabled(); + await expect(canvas.getByRole('slider')).toHaveAttribute('aria-disabled', 'true'); + await expect(canvas.getByText('0:04')).toBeInTheDocument(); + }} +> +
+ +
+
diff --git a/ui/apps/chat/tests/stories/voice/VoiceWaveformLive.stories.svelte b/ui/apps/chat/tests/stories/voice/VoiceWaveformLive.stories.svelte new file mode 100644 index 000000000000..78715741a567 --- /dev/null +++ b/ui/apps/chat/tests/stories/voice/VoiceWaveformLive.stories.svelte @@ -0,0 +1,82 @@ + + + { + const live = canvasElement.querySelector('[data-slot="voice-waveform-live"]')!; + + await expect(live.querySelector('.sr-only')).toHaveTextContent('Recording'); + await expect(live.querySelector('[aria-hidden="true"]')).toBeInTheDocument(); + + const before = barHeights(canvasElement); + + await waitFor(() => expect(barHeights(canvasElement)).not.toBe(before)); + }} +> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
diff --git a/ui/apps/chat/tests/stories/voice/fixtures.ts b/ui/apps/chat/tests/stories/voice/fixtures.ts new file mode 100644 index 000000000000..ca9583245f86 --- /dev/null +++ b/ui/apps/chat/tests/stories/voice/fixtures.ts @@ -0,0 +1,100 @@ +/** + * Demo audio for the voice stories, made in the browser so no binary file sits in the repo: a WAV of a hummed tone + * that rises and falls like syllables, with its peaks. + */ +export interface DemoClip { + src: string; + durationMs: number; + peaks: number[]; +} + +const SAMPLE_RATE = 8000; +const PEAK_COUNT = 64; + +/** Syllable-like loudness at time `t` (s): bursts of about 0.25 s with short gaps between words. */ +function envelope(t: number, seed: number): number { + const syllable = Math.max(Math.sin(t * Math.PI * 4 + seed), 0); + const word = Math.sin(t * Math.PI * 0.9 + seed * 2) > -0.6 ? 1 : 0.08; + const swell = 0.6 + 0.4 * Math.sin(t * 1.3 + seed); + + return syllable * word * swell; +} + +function wavBlob(samples: Float32Array): Blob { + const buffer = new ArrayBuffer(44 + samples.length * 2); + const view = new DataView(buffer); + const write = (offset: number, text: string) => { + for (let i = 0; i < text.length; i++) view.setUint8(offset + i, text.charCodeAt(i)); + }; + + write(0, 'RIFF'); + view.setUint32(4, 36 + samples.length * 2, true); + write(8, 'WAVE'); + write(12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, SAMPLE_RATE, true); + view.setUint32(28, SAMPLE_RATE * 2, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + write(36, 'data'); + view.setUint32(40, samples.length * 2, true); + + for (let i = 0; i < samples.length; i++) { + view.setInt16(44 + i * 2, Math.max(-1, Math.min(1, samples[i])) * 0x7fff, true); + } + + return new Blob([buffer], { type: 'audio/wav' }); +} + +/** The loudest sample of each of `count` buckets, scaled so the loudest bucket is 1. */ +export function peaksOf(samples: Float32Array, count = PEAK_COUNT): number[] { + const size = Math.max(Math.floor(samples.length / count), 1); + const peaks = Array.from({ length: count }, (_, bucket) => { + let peak = 0; + + for (let i = bucket * size; i < Math.min((bucket + 1) * size, samples.length); i++) { + peak = Math.max(peak, Math.abs(samples[i])); + } + + return peak; + }); + const loudest = Math.max(...peaks, 1e-6); + + return peaks.map((peak) => peak / loudest); +} + +const cache = new Map(); + +/** A clip of `seconds`, the same for the same arguments. */ +export function demoClip(seconds: number, seed = 1): DemoClip { + const key = `${seconds}:${seed}`; + const cached = cache.get(key); + + if (cached) return cached; + + const samples = new Float32Array(Math.round(seconds * SAMPLE_RATE)); + + for (let i = 0; i < samples.length; i++) { + const t = i / SAMPLE_RATE; + const pitch = 180 + 40 * Math.sin(t * 2 + seed); + + samples[i] = 0.5 * envelope(t, seed) * Math.sin(2 * Math.PI * pitch * t); + } + + const clip = { + durationMs: seconds * 1000, + peaks: peaksOf(samples), + src: URL.createObjectURL(wavBlob(samples)) + }; + + cache.set(key, clip); + + return clip; +} + +/** One live input level (0..1) at time `t` (s), as a microphone meter would report it while someone speaks. */ +export function demoLevel(t: number, seed = 1): number { + return Math.min(envelope(t, seed) * (0.75 + 0.25 * Math.random()), 1); +} From a62b902dcfad0d008a14d7fe560ee7c1fcfbf2e8 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Sun, 15 Nov 2026 21:25:15 +0100 Subject: [PATCH 04/19] ui: store voice messages with their transcript and send it to models that cannot hear A voice message recorded in the chat form is an audio attachment like an uploaded file, plus what the app needs to draw and read it. The audio attachment gets four optional fields: voice (recorded in the app), transcript, peaks and durationMs. Stored rows, export and import carry them as they are, so no migration is needed and older messages keep working. The transcript is for reading. A model with audio input still gets the audio alone; a model without it (the user switched models later) gets the transcript as a labelled text part in place of the audio. When the model's modalities are not known yet, or there is no transcript, the audio goes as before. A transcript often comes back after the message was sent. The new voiceTranscriptsStore follows each pending transcript by the clip's name, hands it to a send that happens after it arrived, and otherwise writes it into the stored attachment of the message it went out with, in the open chat or, if the user left it, in the database. Copying a message now includes its voice transcript. Assisted-by: Claude (Opus 5.5 Medium) --- .../chat/src/lib/enums/attachment.enums.ts | 3 +- ui/apps/chat/src/lib/services/chat.service.ts | 53 ++++++- .../chat/src/lib/services/database.service.ts | 10 ++ .../src/lib/stores/agentic/index.svelte.ts | 6 +- .../chat/src/lib/stores/chat/index.svelte.ts | 4 +- ui/apps/chat/src/lib/stores/index.ts | 3 + .../lib/stores/voice-transcripts.svelte.ts | 142 ++++++++++++++++++ ui/apps/chat/src/lib/types/chat.d.ts | 8 + ui/apps/chat/src/lib/types/database.d.ts | 7 + ui/apps/chat/src/lib/types/index.ts | 1 + ui/apps/chat/src/lib/utils/clipboard.ts | 10 ++ .../src/lib/utils/convert-files-to-extra.ts | 11 +- .../tests/unit/voice-message-send.test.ts | 119 +++++++++++++++ 13 files changed, 364 insertions(+), 13 deletions(-) create mode 100644 ui/apps/chat/src/lib/stores/voice-transcripts.svelte.ts create mode 100644 ui/apps/chat/tests/unit/voice-message-send.test.ts diff --git a/ui/apps/chat/src/lib/enums/attachment.enums.ts b/ui/apps/chat/src/lib/enums/attachment.enums.ts index 5f096b974486..035fdac9cdd0 100644 --- a/ui/apps/chat/src/lib/enums/attachment.enums.ts +++ b/ui/apps/chat/src/lib/enums/attachment.enums.ts @@ -50,5 +50,6 @@ export enum AttachmentLabel { FILE = 'File', MCP_PROMPT = 'MCP Prompt', MCP_RESOURCE = 'MCP Resource', - PDF_FILE = 'PDF File' + PDF_FILE = 'PDF File', + VOICE_TRANSCRIPT = 'Voice message transcript' } diff --git a/ui/apps/chat/src/lib/services/chat.service.ts b/ui/apps/chat/src/lib/services/chat.service.ts index 848bd06efb39..38c7eba144b1 100644 --- a/ui/apps/chat/src/lib/services/chat.service.ts +++ b/ui/apps/chat/src/lib/services/chat.service.ts @@ -50,6 +50,12 @@ import { getAuthHeaders, getJsonHeaders } from '#lib/utils/api-headers.ts'; import { formatAttachmentText } from '#lib/utils/formatters.ts'; import { streamIdentity } from '#lib/utils/stream-identity.ts'; +/** How stored messages are shaped for the model a request goes to. */ +export interface ApiMessageConversionOptions { + /** False when the model has no audio input: voice messages go as their transcript. */ + audioInput?: boolean; +} + interface ResumableStreamState { bytesReceived: number; updatedAt: number; @@ -133,13 +139,24 @@ export class ChatService { } } + /** + * How messages are shaped for `model`: whether it takes audio, known only once its + * modalities are. Unknown modalities keep the audio, as before voice messages. + */ + static conversionOptionsFor(model?: string | null): ApiMessageConversionOptions { + const modalities = modelsStore.props.getModelModalities(model ?? ''); + + return modalities ? { audioInput: modalities.audio } : {}; + } + /** * Converts a database message with attachments to API chat message format. * Processes various attachment types (images, text files, PDFs) and formats them * as content parts suitable for the chat completion API. */ static async convertDbMessageToApiChatMessageData( - message: DatabaseMessage & { extra?: DatabaseMessageExtra[] } + message: DatabaseMessage & { extra?: DatabaseMessageExtra[] }, + options: ApiMessageConversionOptions = {} ): Promise { // Handle tool result messages (role: 'tool') if (message.role === MessageRole.TOOL && message.toolCallId) { @@ -231,6 +248,20 @@ export class ChatService { ); for (const audio of audioFiles) { + // a model that cannot hear reads what was said instead; one that can gets the audio alone + if (options.audioInput === false && audio.transcript) { + contentParts.push({ + text: formatAttachmentText( + AttachmentLabel.VOICE_TRANSCRIPT, + audio.name, + audio.transcript + ), + type: ContentPartType.TEXT + }); + + continue; + } + contentParts.push({ input_audio: { data: audio.base64Data, @@ -816,14 +847,18 @@ export class ChatService { * messages. Shared by sendMessage, preEncode and the agentic flow. */ static async normalizeMessagesForApi( - messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[] + messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[], + model?: string | null ): Promise { + const options = ChatService.conversionOptionsFor(model); + return ( await Promise.all( messages.map((msg) => { if ('id' in msg && 'convId' in msg && 'timestamp' in msg) { return ChatService.convertDbMessageToApiChatMessageData( - msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] } + msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }, + options ); } @@ -855,8 +890,10 @@ export class ChatService { signal?: AbortSignal, tools?: OpenAIToolDefinition[] ): Promise { - const normalizedMessages: ApiChatMessageData[] = - await ChatService.normalizeMessagesForApi(messages); + const normalizedMessages: ApiChatMessageData[] = await ChatService.normalizeMessagesForApi( + messages, + model + ); const requestBody: Record = { messages: normalizedMessages.map((msg: ApiChatMessageData) => { const mapped: Record = { @@ -1106,8 +1143,10 @@ export class ChatService { xtc_probability, xtc_threshold } = options; - const normalizedMessages: ApiChatMessageData[] = - await ChatService.normalizeMessagesForApi(messages); + const normalizedMessages: ApiChatMessageData[] = await ChatService.normalizeMessagesForApi( + messages, + options.model + ); // Filter out image attachments if the model doesn't support vision if (options.model && !modelsStore.props.modelSupportsVision(options.model)) { diff --git a/ui/apps/chat/src/lib/services/database.service.ts b/ui/apps/chat/src/lib/services/database.service.ts index de8ed4701e12..e77b97a964ca 100644 --- a/ui/apps/chat/src/lib/services/database.service.ts +++ b/ui/apps/chat/src/lib/services/database.service.ts @@ -572,6 +572,16 @@ export class DatabaseService { return result; } + /** + * Gets the messages written at or after `timestamp`, in any conversation. + * + * @param timestamp - Epoch ms + * @returns Messages, oldest first + */ + static async getMessagesSince(timestamp: number): Promise { + return await db[IDXDB_TABLES.messages].where('timestamp').aboveOrEqual(timestamp).toArray(); + } + /** * Imports multiple conversations and their messages. * Skips conversations that already exist. diff --git a/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts b/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts index b607a1373c56..2eb93ad3aa98 100644 --- a/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts +++ b/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts @@ -347,8 +347,10 @@ class AgenticStore { console.log(`[AgenticStore] Starting agentic flow with ${tools.length} tools`); - const normalizedMessages: ApiChatMessageData[] = - await ChatService.normalizeMessagesForApi(messages); + const normalizedMessages: ApiChatMessageData[] = await ChatService.normalizeMessagesForApi( + messages, + options.model + ); this.updateSession(conversationId, { currentTurn: 0, diff --git a/ui/apps/chat/src/lib/stores/chat/index.svelte.ts b/ui/apps/chat/src/lib/stores/chat/index.svelte.ts index ef8a1110035b..20c95be39867 100644 --- a/ui/apps/chat/src/lib/stores/chat/index.svelte.ts +++ b/ui/apps/chat/src/lib/stores/chat/index.svelte.ts @@ -1480,7 +1480,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { const allMessages = await conversationsStore.getConversationMessages(convId); const branchPath = [...filterByLeafNodeId(allMessages, leafId, false)]; const request = await agenticStore.requestFor( - await ChatService.normalizeMessagesForApi(sliceAtLastCompaction(branchPath)), + await ChatService.normalizeMessagesForApi(sliceAtLastCompaction(branchPath), model), this.toolPolicy() ); @@ -1646,7 +1646,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { // message shaping, cut before the preserved steps: the server // reuses the prefix it cached and only the instruction is new. const request = await agenticStore.requestFor( - await ChatService.normalizeMessagesForApi(source.slice(0, cut)), + await ChatService.normalizeMessagesForApi(source.slice(0, cut), effectiveModel), this.toolPolicy() ); const configValue = settingsStore.config; diff --git a/ui/apps/chat/src/lib/stores/index.ts b/ui/apps/chat/src/lib/stores/index.ts index e98a48d06c45..345b5630afed 100644 --- a/ui/apps/chat/src/lib/stores/index.ts +++ b/ui/apps/chat/src/lib/stores/index.ts @@ -61,6 +61,9 @@ export { permissionsStore } from './permissions.svelte'; // TOOLS export { toolsStore } from './tools.svelte'; +// VOICE (transcripts of voice messages on their way) +export { voiceTranscriptsStore } from './voice-transcripts.svelte'; + // ENVIRONMENT / META export { versionStore } from './version.svelte'; diff --git a/ui/apps/chat/src/lib/stores/voice-transcripts.svelte.ts b/ui/apps/chat/src/lib/stores/voice-transcripts.svelte.ts new file mode 100644 index 000000000000..d0395a4f123b --- /dev/null +++ b/ui/apps/chat/src/lib/stores/voice-transcripts.svelte.ts @@ -0,0 +1,142 @@ +/** + * voiceTranscriptsStore - Transcripts of voice messages that are on their way + * + * A voice message is attached as soon as the recording stops, while its transcript + * can take a few seconds more. This keeps each pending transcript by the clip's name + * (unique per recording), so the chip in the form and the sent message show its + * state, a send picks it up once it is there, and a message sent before it arrived + * gets it written into its stored attachment afterwards. Kept in memory only. + */ + +import { AttachmentType } from '#lib/enums/index.ts'; +import { DatabaseService } from '#lib/services/database.service.ts'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { conversationsStore } from '#lib/stores/conversations/index.svelte.ts'; +import type { + DatabaseMessage, + DatabaseMessageExtra, + DatabaseMessageExtraAudioFile +} from '#lib/types/index.ts'; +import { SvelteMap } from 'svelte/reactivity'; + +export type VoiceTranscriptState = + | { status: 'transcribing' } + | { status: 'done'; transcript: string } + | { status: 'error'; message: string }; + +/** How long a transcript that came back keeps looking for the message it belongs to. */ +const APPLY_RETRY_DELAYS_MS = [500, 1500, 4000]; + +function isPendingClip(extra: DatabaseMessageExtra, name: string): boolean { + return ( + extra.type === AttachmentType.AUDIO && + extra.name === name && + extra.voice === true && + extra.transcript === undefined + ); +} + +/** The extras with the clip's transcript filled in, or null when none of them is that clip. */ +export function withTranscript( + extras: DatabaseMessageExtra[] | undefined, + name: string, + transcript: string +): DatabaseMessageExtra[] | null { + if (!extras?.some((extra) => isPendingClip(extra, name))) return null; + + return extras.map((extra) => + isPendingClip(extra, name) + ? ({ ...(extra as DatabaseMessageExtraAudioFile), transcript } as DatabaseMessageExtra) + : extra + ); +} + +class VoiceTranscriptsStore { + private entries = new SvelteMap(); + private recordedAt = new Map(); + + /** Drops a clip removed from the form before it was sent. */ + forget(name: string): void { + this.entries.delete(name); + this.recordedAt.delete(name); + } + + /** The state of a clip's transcript, undefined when none was asked for in this session. */ + get(name: string): VoiceTranscriptState | undefined { + return this.entries.get(name); + } + + /** Follows a clip's transcript, and writes it into the message it went out with. */ + track(name: string, transcript: Promise): void { + this.entries.set(name, { status: 'transcribing' }); + this.recordedAt.set(name, Date.now()); + + transcript.then( + (text) => { + if (!this.entries.has(name)) return; + + this.entries.set(name, { status: 'done', transcript: text }); + void this.applyToMessage(name, text); + }, + (error: unknown) => { + if (!this.entries.has(name)) return; + + console.warn('Voice message transcription failed:', error); + this.entries.set(name, { + message: error instanceof Error ? error.message : String(error), + status: 'error' + }); + } + ); + } + + /** The transcript once it is there. */ + transcriptOf(name: string): string | undefined { + const entry = this.entries.get(name); + + return entry?.status === 'done' ? entry.transcript : undefined; + } + + private async applyToMessage(name: string, transcript: string): Promise { + for (const delay of [0, ...APPLY_RETRY_DELAYS_MS]) { + if (delay) await new Promise((resolve) => setTimeout(resolve, delay)); + + if (await this.writeTranscript(name, transcript)) return; + } + // not sent (still in the form, or dropped): a send reads it from here + } + + private async writeTranscript(name: string, transcript: string): Promise { + const active = (conversationsStore.activeMessages as DatabaseMessage[]).find((message) => + message.extra?.some((extra) => isPendingClip(extra, name)) + ); + + if (active) { + // plain objects: IndexedDB cannot clone the state proxies + const extra = withTranscript($state.snapshot(active.extra), name, transcript)!; + + await DatabaseService.updateMessage(active.id, { extra }); + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(active.id), { + extra + }); + + return true; + } + + // the chat was left before the transcript came back: the message is newer than the clip + const since = this.recordedAt.get(name) ?? Date.now(); + const stored = (await DatabaseService.getMessagesSince(since)).find( + (message: DatabaseMessage) => message.extra?.some((extra) => isPendingClip(extra, name)) + ); + + if (!stored) return false; + + await DatabaseService.updateMessage(stored.id, { + extra: withTranscript(stored.extra, name, transcript)! + }); + + return true; + } +} + +export const voiceTranscriptsStore = new VoiceTranscriptsStore(); diff --git a/ui/apps/chat/src/lib/types/chat.d.ts b/ui/apps/chat/src/lib/types/chat.d.ts index eca0dbec87f8..a72a8dfff984 100644 --- a/ui/apps/chat/src/lib/types/chat.d.ts +++ b/ui/apps/chat/src/lib/types/chat.d.ts @@ -48,6 +48,14 @@ export interface ChatUploadedFile { }; isLoading?: boolean; loadError?: string; + /** A voice message recorded in the chat form. */ + voice?: ChatUploadedVoice; +} + +/** What a voice message recorded in the chat form carries besides its WAV file. */ +export interface ChatUploadedVoice { + durationMs: number; + peaks: number[]; } export interface ChatAttachmentDisplayItem { diff --git a/ui/apps/chat/src/lib/types/database.d.ts b/ui/apps/chat/src/lib/types/database.d.ts index 50f54dc0b556..e47d60fffb7c 100644 --- a/ui/apps/chat/src/lib/types/database.d.ts +++ b/ui/apps/chat/src/lib/types/database.d.ts @@ -38,6 +38,13 @@ export interface DatabaseMessageExtraAudioFile { size?: number; base64Data: string; mimeType: string; + /** Recorded in the app as a voice message, as opposed to an uploaded audio file. */ + voice?: true; + /** What was said, for reading; a model without audio input gets it in place of the audio. */ + transcript?: string; + /** Normalised 0..1 waveform of the clip; computed from the audio when left out. */ + peaks?: number[]; + durationMs?: number; } export interface DatabaseMessageExtraVideoFile { diff --git a/ui/apps/chat/src/lib/types/index.ts b/ui/apps/chat/src/lib/types/index.ts index 68cd6e92f1e7..bfed7569c17f 100644 --- a/ui/apps/chat/src/lib/types/index.ts +++ b/ui/apps/chat/src/lib/types/index.ts @@ -57,6 +57,7 @@ export type { export type { AttachmentMenuItem, ChatUploadedFile, + ChatUploadedVoice, ChatAttachmentDisplayItem, ChatMessageSiblingInfo, ChatMessageActions, diff --git a/ui/apps/chat/src/lib/utils/clipboard.ts b/ui/apps/chat/src/lib/utils/clipboard.ts index a5c75e873317..c2b05985e0e0 100644 --- a/ui/apps/chat/src/lib/utils/clipboard.ts +++ b/ui/apps/chat/src/lib/utils/clipboard.ts @@ -110,6 +110,16 @@ export function formatMessageForClipboard( extras?: DatabaseMessageExtra[], asPlainText: boolean = false ): string { + // what was said in a voice message reads as the message's text + const transcripts = + extras?.flatMap((extra) => + extra.type === AttachmentType.AUDIO && extra.transcript ? [extra.transcript] : [] + ) ?? []; + + if (transcripts.length > 0) { + content = [...transcripts, content].filter((part) => part.trim()).join('\n\n'); + } + // Filter text-like attachments (TEXT, LEGACY_CONTEXT, MCP_PROMPT, and MCP_RESOURCE types) const textAttachments = extras?.filter( diff --git a/ui/apps/chat/src/lib/utils/convert-files-to-extra.ts b/ui/apps/chat/src/lib/utils/convert-files-to-extra.ts index 43d29f43c119..4e50b089c5bc 100644 --- a/ui/apps/chat/src/lib/utils/convert-files-to-extra.ts +++ b/ui/apps/chat/src/lib/utils/convert-files-to-extra.ts @@ -6,6 +6,7 @@ import { SETTINGS_KEYS } from '#lib/constants/index.ts'; import { AttachmentType, FileTypeCategory, SpecialFileType } from '#lib/enums/index.ts'; import { modelsStore } from '#lib/stores/models/index.svelte.ts'; import { settingsStore } from '#lib/stores/settings/index.svelte.ts'; +import { voiceTranscriptsStore } from '#lib/stores/voice-transcripts.svelte.ts'; import type { ChatUploadedFile, DatabaseMessageExtra, @@ -83,13 +84,21 @@ export async function parseFilesToMessageExtras( // Process audio files (MP3 and WAV) try { const base64Data = await readFileAsBase64(file.file); + // a voice message keeps its waveform, and its transcript once it is there + const transcript = file.voice ? voiceTranscriptsStore.transcriptOf(file.name) : undefined; extras.push({ base64Data: base64Data, mimeType: file.type, name: file.name, size: file.size, - type: AttachmentType.AUDIO + type: AttachmentType.AUDIO, + ...(file.voice && { + durationMs: file.voice.durationMs, + peaks: file.voice.peaks, + voice: true as const, + ...(transcript !== undefined && { transcript }) + }) }); } catch (error) { console.error(`Failed to process audio file ${file.name}:`, error); diff --git a/ui/apps/chat/tests/unit/voice-message-send.test.ts b/ui/apps/chat/tests/unit/voice-message-send.test.ts new file mode 100644 index 000000000000..36858e504202 --- /dev/null +++ b/ui/apps/chat/tests/unit/voice-message-send.test.ts @@ -0,0 +1,119 @@ +// Guards how a voice message goes to a model: the audio alone to a model with audio input, +// its transcript as text to one without, and the copied message carrying the transcript. + +import { AttachmentType, ContentPartType, MessageRole, MessageType } from '#lib/enums/index.ts'; +import { ChatService } from '#lib/services/chat.service.ts'; +import { modelsStore } from '#lib/stores/models/index.svelte.ts'; +import { withTranscript } from '#lib/stores/voice-transcripts.svelte.ts'; +import type { DatabaseMessage, DatabaseMessageExtra } from '#lib/types/index.ts'; +import { formatMessageForClipboard } from '#lib/utils/clipboard.ts'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +const voiceClip: DatabaseMessageExtra = { + base64Data: 'UklGRg==', + durationMs: 2400, + mimeType: 'audio/wav', + name: 'voice-1.wav', + peaks: [0.2, 1, 0.4], + transcript: 'What is the weather like on Mars?', + type: AttachmentType.AUDIO, + voice: true +}; + +function userMessage(extra: DatabaseMessageExtra[], content = ''): DatabaseMessage { + return { + children: [], + content, + convId: 'c1', + extra, + id: 'm1', + parent: null, + role: MessageRole.USER, + timestamp: 0, + type: MessageType.TEXT + } as DatabaseMessage; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('sending a voice message', () => { + it('sends the audio alone to a model with audio input', async () => { + const result = await ChatService.convertDbMessageToApiChatMessageData( + userMessage([voiceClip]), + { audioInput: true } + ); + const parts = result.content as { type: string; text?: string }[]; + + expect(parts).toHaveLength(1); + expect(parts[0].type).toBe(ContentPartType.INPUT_AUDIO); + expect(JSON.stringify(parts)).not.toContain('Mars'); + }); + + it('sends the transcript as text to a model without audio input', async () => { + const result = await ChatService.convertDbMessageToApiChatMessageData( + userMessage([voiceClip], 'Answer briefly.'), + { audioInput: false } + ); + const parts = result.content as { type: string; text?: string }[]; + + expect(parts.map((part) => part.type)).toEqual([ContentPartType.TEXT, ContentPartType.TEXT]); + expect(parts[0].text).toContain('Voice message transcript'); + expect(parts[0].text).toContain('What is the weather like on Mars?'); + expect(parts[1].text).toBe('Answer briefly.'); + }); + + it('keeps the audio when there is no transcript to send instead', async () => { + const { transcript: _transcript, ...withoutTranscript } = voiceClip as { + transcript?: string; + } & DatabaseMessageExtra; + const result = await ChatService.convertDbMessageToApiChatMessageData( + userMessage([withoutTranscript as DatabaseMessageExtra]), + { audioInput: false } + ); + + expect((result.content as { type: string }[])[0].type).toBe(ContentPartType.INPUT_AUDIO); + }); + + it('keeps the audio while the model modalities are unknown', () => { + vi.spyOn(modelsStore.props, 'getModelModalities').mockReturnValue(null); + + expect(ChatService.conversionOptionsFor('some-model')).toEqual({}); + }); + + it('reads the audio input from the model modalities', () => { + vi.spyOn(modelsStore.props, 'getModelModalities').mockReturnValue({ + audio: false, + video: false, + vision: true + }); + + expect(ChatService.conversionOptionsFor('text-model')).toEqual({ audioInput: false }); + }); +}); + +describe('a transcript that arrives after the send', () => { + it('fills in the clip it belongs to only', () => { + const pending = { ...voiceClip, transcript: undefined } as DatabaseMessageExtra; + const other = { ...pending, name: 'voice-2.wav' } as DatabaseMessageExtra; + const next = withTranscript([pending, other], 'voice-1.wav', 'Hello'); + + expect(next?.[0]).toMatchObject({ name: 'voice-1.wav', transcript: 'Hello' }); + expect(next?.[1]).toBe(other); + }); + + it('leaves a message without that clip alone', () => { + expect(withTranscript([voiceClip], 'voice-1.wav', 'Hello')).toBeNull(); + expect(withTranscript(undefined, 'voice-1.wav', 'Hello')).toBeNull(); + }); +}); + +describe('copying a voice message', () => { + it('copies the transcript as the text', () => { + expect(formatMessageForClipboard('', [voiceClip])).toBe('What is the weather like on Mars?'); + expect(formatMessageForClipboard('Briefly.', [voiceClip], true)).toBe( + 'What is the weather like on Mars?\n\nBriefly.' + ); + }); +}); From a93442fc990e0d5d813dded6368dcdf0fe713eb7 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Sun, 15 Nov 2026 21:27:50 +0100 Subject: [PATCH 05/19] ui: add Voice settings and resolve what the microphone does per model The user should not have to think about which kind of voice input is in use, so the app decides it from the conversation's model and two settings. resolveVoiceMode is a small pure function: with Voice input on Automatic and a chat model that has audio input, a recording is a voice message, transcribed by that same model so a router swaps nothing. Otherwise it is dictation, with the chat model when it can hear, else, in router mode, with the chosen transcription model. When no model can transcribe, the microphone is unavailable and says why. Settings get a Voice section of their own rather than more rows in General, which is already long: these four belong together and one of them lists the server's models. - Voice input: Automatic (default) or Always dictate. - Transcription model: Same as chat model (default) or, in router mode, any model with audio input; on a single-model server it only says whether that model can transcribe. It notes that dictating with another model loads it beside the chat model. - Transcribe voice messages: on by default. - Transcription language: empty for automatic. The keys live in the settings registry like the others, so defaults, storage, reset and export need nothing new. The model picker is drawn by SettingsChatVoiceTab, the way Extensions and Tools draw their sections. Assisted-by: Claude (Opus 5.5 Medium) --- .../settings/SettingsChat/SettingsChat.svelte | 7 + .../SettingsChat/SettingsChatVoiceTab.svelte | 109 +++++++++++++++ .../src/lib/components/app/settings/index.ts | 9 ++ ui/apps/chat/src/lib/constants/index.ts | 1 + .../lib/constants/settings-keys.constants.ts | 5 + .../src/lib/constants/settings.constants.ts | 58 +++++++- .../chat/src/lib/constants/voice.constants.ts | 17 +++ ui/apps/chat/src/lib/enums/index.ts | 2 + ui/apps/chat/src/lib/enums/voice.enums.ts | 17 +++ ui/apps/chat/src/lib/utils/index.ts | 8 ++ ui/apps/chat/src/lib/utils/voice-mode.ts | 71 ++++++++++ .../settings/SettingsSections.stories.svelte | 2 + ui/apps/chat/tests/unit/voice-mode.test.ts | 124 ++++++++++++++++++ 13 files changed, 425 insertions(+), 5 deletions(-) create mode 100644 ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChatVoiceTab.svelte create mode 100644 ui/apps/chat/src/lib/constants/voice.constants.ts create mode 100644 ui/apps/chat/src/lib/enums/voice.enums.ts create mode 100644 ui/apps/chat/src/lib/utils/voice-mode.ts create mode 100644 ui/apps/chat/tests/unit/voice-mode.test.ts diff --git a/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte b/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte index 1ced25e50f2f..4c1e3281ef9b 100644 --- a/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte +++ b/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte @@ -7,6 +7,7 @@ SettingsChatImportExportTab, SettingsChatMobileHeader, SettingsChatToolsTab, + SettingsChatVoiceTab, SettingsFooter, SettingsResetButton } from '#lib/components/app/settings/index.ts'; @@ -181,6 +182,12 @@ {:else if currentSection.slug === SETTINGS_SECTION_SLUGS.IMPORT_EXPORT} + {:else if currentSection.slug === SETTINGS_SECTION_SLUGS.VOICE && currentSection.fields} + {:else if currentSection.fields}
+ + +
+ + +
+ + + {#if isRouter} + onConfigChange(KEY, value)} + type="single" + value={current} + > + {selectedLabel} + + + {#each options as option (option.value)} + {option.label} + {/each} + + + +

+ {SETTING_CONFIG_INFO[KEY]} Only models with audio input are listed. +

+ +

+ A transcription model other than the chat model is loaded on the server when you dictate, + alongside the chat model. A server that keeps one model at a time (--models-max 1) unloads the chat model to do it. +

+ {:else} +

+ {#if singleModel} + This server runs one model, {singleModel}. + {singleModelHasAudio + ? 'It has audio input, so it transcribes your voice.' + : 'It has no audio input, so voice input is not available on this server.'} + {:else} + This server runs one model, which transcribes your voice when it has audio input. + {/if} +

+ {/if} +
+ + +
diff --git a/ui/apps/chat/src/lib/components/app/settings/index.ts b/ui/apps/chat/src/lib/components/app/settings/index.ts index 2d2aca5af17a..2d44c88921ba 100644 --- a/ui/apps/chat/src/lib/components/app/settings/index.ts +++ b/ui/apps/chat/src/lib/components/app/settings/index.ts @@ -52,6 +52,15 @@ export { default as SettingsResetButton } from './SettingsResetButton.svelte'; */ export { default as SettingsChatImportExportTab } from './SettingsChat/SettingsChatImportExportTab.svelte'; +/** + * **SettingsChatVoiceTab** - Voice section of the settings + * + * The registry's voice fields, with the transcription model picker between them: + * in router mode it lists the server's models with audio input, on a single-model + * server it says whether that model can transcribe. + */ +export { default as SettingsChatVoiceTab } from './SettingsChat/SettingsChatVoiceTab.svelte'; + /** * Section wrapper for import/export sections. Displays a title, description, * icon button, and optional summary of recent actions. diff --git a/ui/apps/chat/src/lib/constants/index.ts b/ui/apps/chat/src/lib/constants/index.ts index 46feb4a6dd4a..d7a8c982f403 100644 --- a/ui/apps/chat/src/lib/constants/index.ts +++ b/ui/apps/chat/src/lib/constants/index.ts @@ -74,3 +74,4 @@ export * from './read-media'; export * from './get-datetime'; export * from './browser-info'; export * from './search-chats'; +export * from './voice.constants'; diff --git a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts index c94f97ff7d72..12c941e6faec 100644 --- a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts +++ b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts @@ -85,6 +85,11 @@ export const SETTINGS_KEYS = { TYP_P: 'typ_p', // General USE_HUGGING_FACE_HUB: 'useHuggingFaceHub', + // Voice + VOICE_INPUT_MODE: 'voiceInputMode', + VOICE_TRANSCRIBE_MESSAGES: 'voiceTranscribeMessages', + VOICE_TRANSCRIPTION_LANGUAGE: 'voiceTranscriptionLanguage', + VOICE_TRANSCRIPTION_MODEL: 'voiceTranscriptionModel', XTC_PROBABILITY: 'xtc_probability', XTC_THRESHOLD: 'xtc_threshold' } as const; diff --git a/ui/apps/chat/src/lib/constants/settings.constants.ts b/ui/apps/chat/src/lib/constants/settings.constants.ts index 669ccd6ecbb4..053311375b7c 100644 --- a/ui/apps/chat/src/lib/constants/settings.constants.ts +++ b/ui/apps/chat/src/lib/constants/settings.constants.ts @@ -6,19 +6,22 @@ import { SETTINGS_KEYS } from './settings-keys.constants'; import { TITLE_GENERATION } from './title-generation.constants'; import { FILE_GLOB_SEARCH_PICKERS } from './working-directory.constants'; import { + AudioLines, Code, Database, File, Funnel, GlassWater, ListRestart, + Mic, Monitor, Moon, PencilRuler, SlidersVertical, - Sun + Sun, + TextCursorInput } from '@lucide/svelte'; -import { SurfaceTheme, SyncableParameterType } from '#lib/enums/index.ts'; +import { SurfaceTheme, SyncableParameterType, VoiceInputMode } from '#lib/enums/index.ts'; import { SettingsFieldType } from '#lib/enums/settings.enums.ts'; import { ColorMode } from '#lib/enums/ui.enums.ts'; import type { @@ -37,7 +40,8 @@ export const SETTINGS_SECTIONS = { GENERAL: { slug: 'general', title: 'General' }, IMPORT_EXPORT: { slug: 'import-export', title: 'Import/Export' }, SAMPLING_PENALTIES: { slug: 'sampling-penalties', title: 'Sampling & Penalties' }, - TOOLS: { slug: 'tools', title: 'Tools' } + TOOLS: { slug: 'tools', title: 'Tools' }, + VOICE: { slug: 'voice', title: 'Voice' } } as const; export const SETTINGS_SECTION_SLUGS = { @@ -47,7 +51,8 @@ export const SETTINGS_SECTION_SLUGS = { GENERAL: SETTINGS_SECTIONS.GENERAL.slug, IMPORT_EXPORT: SETTINGS_SECTIONS.IMPORT_EXPORT.slug, SAMPLING_PENALTIES: SETTINGS_SECTIONS.SAMPLING_PENALTIES.slug, - TOOLS: SETTINGS_SECTIONS.TOOLS.slug + TOOLS: SETTINGS_SECTIONS.TOOLS.slug, + VOICE: SETTINGS_SECTIONS.VOICE.slug } as const; export const SETTINGS_SECTION_TITLES = { @@ -57,7 +62,8 @@ export const SETTINGS_SECTION_TITLES = { GENERAL: SETTINGS_SECTIONS.GENERAL.title, IMPORT_EXPORT: SETTINGS_SECTIONS.IMPORT_EXPORT.title, SAMPLING_PENALTIES: SETTINGS_SECTIONS.SAMPLING_PENALTIES.title, - TOOLS: SETTINGS_SECTIONS.TOOLS.title + TOOLS: SETTINGS_SECTIONS.TOOLS.title, + VOICE: SETTINGS_SECTIONS.VOICE.title } as const; export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [ @@ -393,6 +399,48 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [ slug: SETTINGS_SECTION_SLUGS.DISPLAY, title: SETTINGS_SECTION_TITLES.DISPLAY }, + // Voice: the transcription model picker lists the server's models (SettingsChatVoiceTab) + { + icon: Mic, + settings: [ + { + defaultValue: VoiceInputMode.AUTO, + help: 'Automatic sends your voice to a model that understands audio, as a voice message with its transcript, and turns it into text in the message input for any other model. Always dictate turns it into text for every model, so only the text reaches the chat.', + key: SETTINGS_KEYS.VOICE_INPUT_MODE, + label: 'Voice input', + options: [ + { icon: AudioLines, label: 'Automatic', value: VoiceInputMode.AUTO }, + { icon: TextCursorInput, label: 'Always dictate', value: VoiceInputMode.DICTATION } + ], + type: SettingsFieldType.SELECT + }, + { + defaultValue: '', + help: 'The model that turns speech into text when the chat model has no audio input. Empty uses the chat model.', + key: SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL, + label: 'Transcription model', + standaloneField: false, + type: SettingsFieldType.SELECT + }, + { + defaultValue: true, + help: 'Write down what was said in each voice message, to read under its waveform. The chat model writes it, and a model without audio input gets it in place of the audio.', + key: SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES, + label: 'Transcribe voice messages', + type: SettingsFieldType.CHECKBOX + }, + { + defaultValue: '', + help: 'The language you speak, as a two-letter code such as en or de, passed to the transcription. Leave it empty to let the model tell.', + key: SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE, + label: 'Transcription language', + placeholder: 'Automatic', + type: SettingsFieldType.INPUT + } + ], + slug: SETTINGS_SECTION_SLUGS.VOICE, + title: SETTINGS_SECTION_TITLES.VOICE + }, // MCP Servers (non-UI config object) { icon: PencilRuler, diff --git a/ui/apps/chat/src/lib/constants/voice.constants.ts b/ui/apps/chat/src/lib/constants/voice.constants.ts new file mode 100644 index 000000000000..9f8d9d69eeab --- /dev/null +++ b/ui/apps/chat/src/lib/constants/voice.constants.ts @@ -0,0 +1,17 @@ +import { STORAGE_APP_NAME } from './storage.constants'; + +/** What the chat form's microphone says it will do, or why it can't, on its button. */ +export const VOICE_MODE_MESSAGE = { + dictate: 'Dictate', + separateModelHint: + 'It may be loaded on the server alongside the chat model, which a server that keeps one model at a time unloads.', + unavailable: + 'No model with audio input is available. In router mode, pick a transcription model in Settings > Voice.', + voiceMessage: 'Send a voice message' +} as const; + +/** Where the chat form remembers that it showed the hint about a separate transcription model. */ +export const VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.voiceSeparateModelHintSeen`; + +/** Name prefix of a voice message's WAV file; the rest makes it unique per recording. */ +export const VOICE_CLIP_NAME_PREFIX = 'voice-'; diff --git a/ui/apps/chat/src/lib/enums/index.ts b/ui/apps/chat/src/lib/enums/index.ts index 29fd94373ee0..a4eed39d8190 100644 --- a/ui/apps/chat/src/lib/enums/index.ts +++ b/ui/apps/chat/src/lib/enums/index.ts @@ -121,3 +121,5 @@ export { } from './tools.enums'; export { SplashOrientation } from './splash.enums'; + +export { VoiceInputMode, VoiceMode } from './voice.enums'; diff --git a/ui/apps/chat/src/lib/enums/voice.enums.ts b/ui/apps/chat/src/lib/enums/voice.enums.ts new file mode 100644 index 000000000000..4fb2c92bd7b0 --- /dev/null +++ b/ui/apps/chat/src/lib/enums/voice.enums.ts @@ -0,0 +1,17 @@ +/** The Voice input setting: what the microphone does with a model that understands audio. */ +export enum VoiceInputMode { + /** Send the voice to a model with audio input, dictate otherwise. */ + AUTO = 'auto', + /** Always turn speech into text; only the text reaches the chat. */ + DICTATION = 'dictation' +} + +/** What a recording becomes for the conversation's model, as the chat form resolves it. */ +export enum VoiceMode { + /** Speech becomes text in the input; no audio is attached. */ + DICTATION = 'dictation', + /** No model can take audio, so the microphone has nothing to do. */ + UNAVAILABLE = 'unavailable', + /** The clip is attached and sent as audio, with a transcript to read. */ + VOICE_MESSAGE = 'voice-message' +} diff --git a/ui/apps/chat/src/lib/utils/index.ts b/ui/apps/chat/src/lib/utils/index.ts index a0544551393e..0737b08f58d4 100644 --- a/ui/apps/chat/src/lib/utils/index.ts +++ b/ui/apps/chat/src/lib/utils/index.ts @@ -384,3 +384,11 @@ export { groupModelFamilies, modelFamilyKey, type ModelFamilyGroup } from './mod // Svelte actions export { nearViewport } from './near-viewport'; + +// Voice: what a recording becomes for the conversation's model +export { + resolveVoiceMode, + voiceModeLabel, + type VoiceModeInput, + type VoiceModeResolution +} from './voice-mode'; diff --git a/ui/apps/chat/src/lib/utils/voice-mode.ts b/ui/apps/chat/src/lib/utils/voice-mode.ts new file mode 100644 index 000000000000..953615a48442 --- /dev/null +++ b/ui/apps/chat/src/lib/utils/voice-mode.ts @@ -0,0 +1,71 @@ +import { VOICE_MODE_MESSAGE } from '#lib/constants/voice.constants.ts'; +import { VoiceInputMode, VoiceMode } from '#lib/enums/index.ts'; + +export interface VoiceModeInput { + /** The Voice input setting. */ + setting: VoiceInputMode | string; + /** The conversation's effective model, null when none is picked. */ + chatModel: string | null; + chatModelHasAudio: boolean; + isRouter: boolean; + /** The Transcription model setting; empty means the chat model. */ + transcriptionModel: string; + transcriptionModelHasAudio: boolean; +} + +export type VoiceModeResolution = + | { + mode: VoiceMode.VOICE_MESSAGE; + /** The chat model: it writes the transcript too, so a router swaps no model. */ + transcriptionModel: string | null; + separateModel: false; + } + | { + mode: VoiceMode.DICTATION; + transcriptionModel: string | null; + /** A router model other than the chat model, which transcribing may load. */ + separateModel: boolean; + } + | { mode: VoiceMode.UNAVAILABLE; reason: string }; + +/** + * What a recording becomes for the conversation's model. Automatic sends the voice to a + * model with audio input and dictates otherwise; dictation transcribes with the chat model + * when it can hear, else (router mode) with the chosen transcription model. + */ +export function resolveVoiceMode(input: VoiceModeInput): VoiceModeResolution { + const { chatModel, chatModelHasAudio, isRouter, setting } = input; + + if (setting !== VoiceInputMode.DICTATION && chatModelHasAudio) { + return { mode: VoiceMode.VOICE_MESSAGE, separateModel: false, transcriptionModel: chatModel }; + } + + if (chatModelHasAudio) { + return { mode: VoiceMode.DICTATION, separateModel: false, transcriptionModel: chatModel }; + } + + // a single-model server has no other model to transcribe with + const chosen = isRouter ? input.transcriptionModel.trim() : ''; + + if (chosen && input.transcriptionModelHasAudio) { + return { + mode: VoiceMode.DICTATION, + separateModel: chosen !== chatModel, + transcriptionModel: chosen + }; + } + + return { mode: VoiceMode.UNAVAILABLE, reason: VOICE_MODE_MESSAGE.unavailable }; +} + +/** What the microphone button says it will do, or why it can't. */ +export function voiceModeLabel(resolution: VoiceModeResolution): string { + switch (resolution.mode) { + case VoiceMode.VOICE_MESSAGE: + return VOICE_MODE_MESSAGE.voiceMessage; + case VoiceMode.DICTATION: + return VOICE_MODE_MESSAGE.dictate; + default: + return resolution.reason; + } +} diff --git a/ui/apps/chat/tests/stories/settings/SettingsSections.stories.svelte b/ui/apps/chat/tests/stories/settings/SettingsSections.stories.svelte index 24b2c5c13b9b..de2d6b8f9063 100644 --- a/ui/apps/chat/tests/stories/settings/SettingsSections.stories.svelte +++ b/ui/apps/chat/tests/stories/settings/SettingsSections.stories.svelte @@ -17,6 +17,8 @@ + + diff --git a/ui/apps/chat/tests/unit/voice-mode.test.ts b/ui/apps/chat/tests/unit/voice-mode.test.ts new file mode 100644 index 000000000000..316b7179b4e5 --- /dev/null +++ b/ui/apps/chat/tests/unit/voice-mode.test.ts @@ -0,0 +1,124 @@ +// Guards what the microphone does for a conversation's model: a voice message for a model +// that hears, dictation otherwise, and why it can't when no model can transcribe. + +import { + SETTING_CONFIG_DEFAULT, + SETTINGS_CHAT_SECTIONS, + SETTINGS_KEYS, + SETTINGS_SECTION_SLUGS, + VOICE_MODE_MESSAGE +} from '#lib/constants/index.ts'; +import { SettingsFieldType, VoiceInputMode, VoiceMode } from '#lib/enums/index.ts'; +import { resolveVoiceMode, type VoiceModeInput, voiceModeLabel } from '#lib/utils/voice-mode.ts'; +import { describe, expect, it } from 'vitest'; + +const base: VoiceModeInput = { + chatModel: 'gemma-3n', + chatModelHasAudio: true, + isRouter: true, + setting: VoiceInputMode.AUTO, + transcriptionModel: '', + transcriptionModelHasAudio: false +}; + +describe('resolveVoiceMode', () => { + it('sends a voice message to a chat model with audio, transcribed by that model', () => { + expect(resolveVoiceMode(base)).toEqual({ + mode: VoiceMode.VOICE_MESSAGE, + separateModel: false, + transcriptionModel: 'gemma-3n' + }); + }); + + it('dictates with the chat model when the user always dictates', () => { + expect(resolveVoiceMode({ ...base, setting: VoiceInputMode.DICTATION })).toEqual({ + mode: VoiceMode.DICTATION, + separateModel: false, + transcriptionModel: 'gemma-3n' + }); + }); + + it('prefers the chat model over a chosen transcription model when it can hear', () => { + const result = resolveVoiceMode({ + ...base, + setting: VoiceInputMode.DICTATION, + transcriptionModel: 'voxtral', + transcriptionModelHasAudio: true + }); + + expect(result).toMatchObject({ separateModel: false, transcriptionModel: 'gemma-3n' }); + }); + + it('dictates with the chosen model for a chat model without audio, in router mode', () => { + expect( + resolveVoiceMode({ + ...base, + chatModel: 'qwen3', + chatModelHasAudio: false, + transcriptionModel: 'voxtral', + transcriptionModelHasAudio: true + }) + ).toEqual({ mode: VoiceMode.DICTATION, separateModel: true, transcriptionModel: 'voxtral' }); + }); + + it('is unavailable when no model can transcribe', () => { + const noAudio = { ...base, chatModel: 'qwen3', chatModelHasAudio: false }; + + expect(resolveVoiceMode(noAudio)).toEqual({ + mode: VoiceMode.UNAVAILABLE, + reason: VOICE_MODE_MESSAGE.unavailable + }); + // a chosen model without audio input does not count + expect( + resolveVoiceMode({ + ...noAudio, + transcriptionModel: 'llama', + transcriptionModelHasAudio: false + }).mode + ).toBe(VoiceMode.UNAVAILABLE); + }); + + it('ignores the transcription model on a single-model server', () => { + expect( + resolveVoiceMode({ + ...base, + chatModelHasAudio: false, + isRouter: false, + transcriptionModel: 'voxtral', + transcriptionModelHasAudio: true + }).mode + ).toBe(VoiceMode.UNAVAILABLE); + }); + + it('labels the microphone with what it will do', () => { + expect(voiceModeLabel(resolveVoiceMode(base))).toBe('Send a voice message'); + expect(voiceModeLabel(resolveVoiceMode({ ...base, setting: VoiceInputMode.DICTATION }))).toBe( + 'Dictate' + ); + expect(voiceModeLabel(resolveVoiceMode({ ...base, chatModelHasAudio: false }))).toBe( + VOICE_MODE_MESSAGE.unavailable + ); + }); +}); + +describe('voice settings', () => { + const section = SETTINGS_CHAT_SECTIONS.find((s) => s.slug === SETTINGS_SECTION_SLUGS.VOICE); + + it('default to automatic voice input, transcribed by the chat model', () => { + expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_INPUT_MODE]).toBe(VoiceInputMode.AUTO); + expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES]).toBe(true); + expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL]).toBe(''); + expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE]).toBe(''); + }); + + it('have a section of their own, the model picker drawn by it', () => { + const keys = section?.fields?.map((field) => field.key); + + expect(keys).toEqual([ + SETTINGS_KEYS.VOICE_INPUT_MODE, + SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES, + SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE + ]); + expect(section?.fields?.[0].type).toBe(SettingsFieldType.SELECT); + }); +}); From 8bb7b9dcfee687733ef2b951d5f3165c72d504ab Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Sun, 15 Nov 2026 21:41:41 +0100 Subject: [PATCH 06/19] ui: record voice messages and dictation from the chat form The microphone now does what the conversation's model allows, without the user choosing a mode: its label says "Send a voice message" or "Dictate", and when no model can take audio it is disabled and says why (shown in router mode only when some model there has audio input, so the reason can be acted on; a single-model server without audio keeps the send button as before). The microphone errors keep their warning state and toast, now also for the voice engine's own capture errors. Recording goes through the voice engine's VoiceRecorder instead of the MediaRecorder flow. While it records, the actions row becomes a recording row: cancel, the live waveform, the elapsed time and stop. Escape or cancel drops the recording. The form keeps its two rows while recording, so the message stays visible on desktop and the row fits the phone layouts and the expanded sheet. - Dictation writes the live transcript into the message at the caret as it streams. DictationSpan replaces only the dictated stretch, moves it when the user types before it, keeps what is typed after it, and hands text the user edits over to them. When nothing streams, the row shows Transcribing... after stop and the text goes in once it is back. Cancel takes the dictated text out again. No audio is attached. - A voice message streams its live text, if any, under the waveform, not into the message. On stop the clip is attached as a voice file with its peaks and length, and its transcript is followed by the voice transcripts store (unless Transcribe voice messages is off). Realtime transcription is skipped for servers reached through a tunnel or with an API key, which a browser websocket cannot carry. The first dictation with a separate router model shows a one-time note that it loads beside the chat model. The form takes an optional voiceRecorder, so tests and stories can run it without a microphone. Assisted-by: Claude (Opus 5.5 Medium) --- .../app/chat/ChatForm/ChatForm.svelte | 160 ++++--- .../ChatFormActionRecord.svelte | 32 +- .../ChatFormActions/ChatFormActions.svelte | 17 +- .../ChatFormActionsRecording.svelte | 92 +++++ .../chat/src/lib/components/app/chat/index.ts | 14 +- .../src/lib/constants/settings.constants.ts | 2 +- .../chat/src/lib/constants/voice.constants.ts | 5 +- .../lib/hooks/use-chat-form-voice.svelte.ts | 391 ++++++++++++++++++ ui/apps/chat/src/lib/utils/dictation-span.ts | 90 ++++ .../ChatFormActionRecordWrapper.svelte | 12 +- .../microphone-unavailable.svelte.test.ts | 12 +- .../form/ChatFormActionRecord.stories.svelte | 50 ++- .../tests/stories/voice/VoiceLiveDemo.svelte | 2 +- .../chat/tests/unit/dictation-span.test.ts | 90 ++++ 14 files changed, 860 insertions(+), 109 deletions(-) create mode 100644 ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionsRecording.svelte create mode 100644 ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts create mode 100644 ui/apps/chat/src/lib/utils/dictation-span.ts create mode 100644 ui/apps/chat/tests/unit/dictation-span.test.ts diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte index 0c44d125fb9c..d37dfd002b36 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -5,6 +5,7 @@ ActionIcon, ChatAttachmentsList, ChatFormActions, + ChatFormActionsRecording, ChatFormCurrentWorkingDirectory, ChatFormInput, ChatFormInputFileInputInvisible, @@ -30,6 +31,10 @@ } from '#lib/enums/index.ts'; import { useChatFormExpansion } from '#lib/hooks/use-chat-form-expansion.svelte.ts'; import { useChatFormPickers } from '#lib/hooks/use-chat-form-pickers.svelte.ts'; + import { + useChatFormVoice, + type VoiceRecorderLike + } from '#lib/hooks/use-chat-form-voice.svelte.ts'; import { chatStore, conversationsStore, @@ -39,7 +44,8 @@ serverStore, settingsStore, toolsStore, - uiStore + uiStore, + voiceTranscriptsStore } from '#lib/stores/index.ts'; import type { FileMentionEntry, @@ -48,14 +54,7 @@ MCPResourceInfo, PromptMessage } from '#lib/types/index.ts'; - import { - AudioRecorder, - convertToWav, - createAudioFile, - describeMicrophoneError, - isAudioRecordingSupported, - microphoneUnavailableReason - } from '#lib/utils/browser-only.ts'; + import { microphoneUnavailableReason } from '#lib/utils/browser-only.ts'; import { buildMentionInsertion, findCommandToken, @@ -65,6 +64,7 @@ parseClipboardContent, uuid } from '#lib/utils/index.ts'; + import { VoiceRecorder } from '#lib/voice/index.ts'; import { onMount, untrack } from 'svelte'; import type { ClassValue } from 'svelte/elements'; import { toast } from 'svelte-sonner'; @@ -96,6 +96,9 @@ onUploadedFileRemove?: (fileId: string) => void; onUploadedFilesChange?: (files: ChatUploadedFile[]) => void; onValueChange?: (value: string) => void; + + /** Records voice input; tests and stories pass a fake, the app a VoiceRecorder. */ + voiceRecorder?: VoiceRecorderLike; } let { @@ -117,21 +120,20 @@ showModelSelector = true, singleRow = false, uploadedFiles = $bindable([]), - value = $bindable('') + value = $bindable(''), + voiceRecorder }: Props = $props(); // Component References - let audioRecorder: AudioRecorder | undefined; let chatFormActionsRef: ChatFormActions | undefined = $state(undefined); let fileInputRef: ChatFormInputFileInputInvisible | undefined = $state(undefined); let pickersRef: { handleKeydown: (event: KeyboardEvent) => boolean } | undefined = $state(undefined); let inputRef: ChatFormInput | undefined = $state(undefined); - // Audio Recording State - let isRecording = $state(false); - let recordingSupported = $state(false); + // Voice input + const recorder: VoiceRecorderLike = untrack(() => voiceRecorder) ?? new VoiceRecorder(); /** Why this page can't record at all (no HTTPS, no recorder), which no tap fixes. */ let micUnavailableReason = $state(null); /** Why the last start failed (denied, no microphone, busy); the next tap tries again. */ @@ -189,6 +191,25 @@ } }); + const voice = useChatFormVoice({ + addVoiceFile: (file) => { + uploadedFiles = [...uploadedFiles, file]; + onUploadedFilesChange?.(uploadedFiles); + }, + getCaretOffset: () => inputRef?.getCaretOffset() ?? value.length, + getRecorder: () => recorder, + getValue: () => value, + onMicError: (message) => (micError = message), + // on a phone placing the caret would open the keyboard + setCaretOffset: (offset) => { + if (!deviceStore.isMobile) inputRef?.setCaretOffset(offset); + }, + setValue: (next) => { + value = next; + onValueChange?.(next); + } + }); + async function handleWorkingDirectoryChange(newDir: string | null) { // Committing a directory consumes the `/cwd` token; the chip's // clear-X path has no token to consume. @@ -263,6 +284,7 @@ singleRow && !deviceStore.isMobile && !isExpanded && + !voice.isActive && !hasAttachments && !mcpStore.resources.hasAttachments && !value.includes('\n') @@ -327,9 +349,7 @@ ); onMount(() => { - recordingSupported = isAudioRecordingSupported(); micUnavailableReason = microphoneUnavailableReason(); - audioRecorder = new AudioRecorder(); }); /** Expands for the message as it stands, e.g. one put back after a send failed. */ @@ -375,6 +395,10 @@ onAttachmentRemove?.(index); } } else { + const removed = uploadedFiles.find((file) => file.id === fileId); + + if (removed?.voice) voiceTranscriptsStore.forget(removed.name); + onUploadedFileRemove?.(fileId); } } @@ -586,7 +610,7 @@ inputRef?.replaceText(built.newValue, built.caretOffset); } - async function handleMicClick() { + function handleMicClick() { // a phone shows no tooltip on a tap, so the reason comes as a toast if (micUnavailableReason) { toast.error(micUnavailableReason); @@ -594,36 +618,20 @@ return; } - if (!audioRecorder || !recordingSupported) { - console.warn('Audio recording not supported'); - - return; - } + void voice.start(); + } - if (isRecording) { - isRecording = false; - try { - const audioBlob = await audioRecorder.stopRecording(); - const wavBlob = await convertToWav(audioBlob); - const audioFile = createAudioFile(wavBlob); + // Escape drops a recording, wherever the focus is + function handleWindowKeydown(event: KeyboardEvent) { + if (event.key !== KeyboardKey.ESCAPE || !voice.isActive || event.defaultPrevented) return; - onFilesAdd?.([audioFile]); - } catch (error) { - console.error('Failed to stop recording:', error); - } - } else { - try { - await audioRecorder.startRecording(); - isRecording = true; - micError = null; - } catch (error) { - micError = describeMicrophoneError(error); - toast.error(micError); - } - } + event.preventDefault(); + voice.cancel(); } + +
{ + voice.handleUserInput(); expansion.handleInput(); pickers.handleInput(); onValueChange?.(value); @@ -776,29 +785,50 @@
- - onSystemPromptClick?.({ files: uploadedFiles, message: value })} - onWorkingDirectoryClick={isWorkingDirectoryInAddMenu - ? pickers.handleWorkingDirectoryOpen - : undefined} - {showAddButton} - {showModelSelector} - {uploadedFiles} - workingDirectory={cwd} - /> + {#if voice.isActive} + void voice.stop()} + state={voice.state} + /> + {/if} + + +
+ + onSystemPromptClick?.({ files: uploadedFiles, message: value })} + onWorkingDirectoryClick={isWorkingDirectoryInAddMenu + ? pickers.handleWorkingDirectoryOpen + : undefined} + {showAddButton} + {showModelSelector} + {uploadedFiles} + voice={{ + action: voice.label, + hint: voice.hint, + show: voice.showButton, + unavailableReason: voice.unavailableReason + }} + workingDirectory={cwd} + /> +
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte index 8664af6b2ea7..2abe0f5b56ec 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte @@ -2,34 +2,44 @@ import { Button } from '@ggml-org/llama-ui/ui/button'; import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip'; import { Mic, MicOff, Square } from '@lucide/svelte'; - import { ICON_CLASS_DEFAULT } from '#lib/constants/index.ts'; + import { ICON_CLASS_DEFAULT, VOICE_MODE_MESSAGE } from '#lib/constants/index.ts'; interface Props { + /** What a press does: send a voice message or dictate. */ + action?: string; class?: string; disabled?: boolean; - hasAudioModality?: boolean; + /** Said after the action in the tooltip, e.g. that another model may be loaded. */ + hint?: string | null; isLoading?: boolean; isRecording?: boolean; /** Why the microphone can't record (no HTTPS, access denied...), shown on the button. */ micWarning?: string | null; + /** Why no model can take the recording; disables the button. */ + unavailableReason?: string | null; onMicClick?: () => void; } let { + action = VOICE_MODE_MESSAGE.voiceMessage, class: className = '', disabled = false, - hasAudioModality = false, + hint = null, isLoading = false, isRecording = false, micWarning = null, - onMicClick + onMicClick, + unavailableReason = null }: Props = $props(); - // a model without audio input disables the button; a microphone that can't record keeps it + // no model to take the recording disables the button; a microphone that can't record keeps it // tappable, so a tap can say why - const showWarning = $derived(hasAudioModality && !isRecording && !!micWarning); + const showWarning = $derived(!unavailableReason && !isRecording && !!micWarning); const tooltip = $derived( - !hasAudioModality ? 'Current model does not support audio' : showWarning ? micWarning : null + isRecording + ? null + : (unavailableReason ?? + (showWarning ? micWarning : [action, hint].filter(Boolean).join('. '))) ); @@ -38,11 +48,11 @@ {:else if shouldShowRecordButton} {:else} + + +
+
+ + + {#if state === 'transcribing'} +
+ + {VOICE_MODE_MESSAGE.transcribing} +
+ {:else} + + + + {formatClipTime(elapsedMs)} + + {/if} + + +
+ + {#if liveCaption} +

+ {liveCaption} +

+ {/if} +
diff --git a/ui/apps/chat/src/lib/components/app/chat/index.ts b/ui/apps/chat/src/lib/components/app/chat/index.ts index fce3962c4fe4..32d26aca5a60 100644 --- a/ui/apps/chat/src/lib/components/app/chat/index.ts +++ b/ui/apps/chat/src/lib/components/app/chat/index.ts @@ -170,12 +170,20 @@ export { default as ChatForm } from './ChatForm/ChatForm.svelte'; export { default as ChatFormActionsAdd } from './ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte'; /** - * Audio recording button with real-time recording indicator. Records audio - * and converts to WAV format for upload. Only visible when the active model - * supports audio modality and setting for automatic audio input is enabled. Shows recording duration while active. + * Microphone button: starts a voice message or a dictation (its label says which), + * or, disabled, says why no model can take a recording; while recording it is the + * stop button. Shown on an empty message when voice input is available and the + * setting for the microphone on empty input is on. */ export { default as ChatFormActionRecord } from './ChatForm/ChatFormActions/ChatFormActionRecord.svelte'; +/** + * The actions row while the chat form records: cancel, the live waveform and the + * elapsed time, and stop, with a voice message's streaming text under the waveform + * and a waiting state while dictated text is transcribed after recording. + */ +export { default as ChatFormActionsRecording } from './ChatForm/ChatFormActions/ChatFormActionsRecording.svelte'; + /** * Container for chat form action buttons. Arranges file attachment, audio record, * and submit/stop buttons in a horizontal layout. Handles conditional visibility diff --git a/ui/apps/chat/src/lib/constants/settings.constants.ts b/ui/apps/chat/src/lib/constants/settings.constants.ts index 053311375b7c..cdd1689e4520 100644 --- a/ui/apps/chat/src/lib/constants/settings.constants.ts +++ b/ui/apps/chat/src/lib/constants/settings.constants.ts @@ -140,7 +140,7 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [ }, { defaultValue: true, - help: 'Automatically show microphone button instead of send button when textarea is empty for models with audio modality support.', + help: 'Show the microphone in place of the send button while the message is empty, whenever voice input is available (see Voice).', key: SETTINGS_KEYS.AUTO_MIC_ON_EMPTY, label: 'Show microphone on empty input', type: SettingsFieldType.CHECKBOX diff --git a/ui/apps/chat/src/lib/constants/voice.constants.ts b/ui/apps/chat/src/lib/constants/voice.constants.ts index 9f8d9d69eeab..a74d3e3d02d0 100644 --- a/ui/apps/chat/src/lib/constants/voice.constants.ts +++ b/ui/apps/chat/src/lib/constants/voice.constants.ts @@ -3,8 +3,11 @@ import { STORAGE_APP_NAME } from './storage.constants'; /** What the chat form's microphone says it will do, or why it can't, on its button. */ export const VOICE_MODE_MESSAGE = { dictate: 'Dictate', + dictating: 'Dictating', + recording: 'Recording a voice message', separateModelHint: - 'It may be loaded on the server alongside the chat model, which a server that keeps one model at a time unloads.', + 'The transcription model is loaded on the server alongside the chat model, which a server that keeps one model at a time unloads.', + transcribing: 'Transcribing...', unavailable: 'No model with audio input is available. In router mode, pick a transcription model in Settings > Voice.', voiceMessage: 'Send a voice message' diff --git a/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts b/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts new file mode 100644 index 000000000000..7a4bbc0b2080 --- /dev/null +++ b/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts @@ -0,0 +1,391 @@ +/** + * Voice input for the chat form. + * + * Resolves what the microphone does for the conversation's model (a voice message, + * dictation, or nothing, see resolveVoiceMode) and runs a recording: live levels and + * elapsed time, dictated text written into the message at the caret as it streams, and + * on stop either a voice message attachment, whose transcript follows, or the text. + */ + +import { + SETTINGS_KEYS, + VOICE_CLIP_NAME_PREFIX, + VOICE_MODE_MESSAGE, + VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY +} from '#lib/constants/index.ts'; +import { MimeTypeAudio, VoiceInputMode, VoiceMode } from '#lib/enums/index.ts'; +import { + modelsStore, + serverStore, + settingsStore, + voiceTranscriptsStore +} from '#lib/stores/index.ts'; +import type { ChatUploadedFile } from '#lib/types/index.ts'; +import { getAuthHeaders } from '#lib/utils/api-headers.ts'; +import { describeMicrophoneError } from '#lib/utils/audio-recording.ts'; +import { withBase } from '#lib/utils/base-path.ts'; +import { DictationSpan } from '#lib/utils/dictation-span.ts'; +import { uuid } from '#lib/utils/index.ts'; +import { resolveVoiceMode, voiceModeLabel } from '#lib/utils/voice-mode.ts'; +import { + VoiceCaptureError, + type VoiceRecorderStatus, + type VoiceServerContext, + type VoiceSessionOptions, + type VoiceSessionResult +} from '#lib/voice/index.ts'; +import { onDestroy } from 'svelte'; +import { toast } from 'svelte-sonner'; + +/** What the form needs of a VoiceRecorder; tests and stories pass a fake. */ +export interface VoiceRecorderLike { + readonly levels: number[]; + readonly live: boolean; + readonly liveText: string; + readonly status: VoiceRecorderStatus; + start(options?: VoiceSessionOptions): Promise; + stop(): Promise; + cancel(): void; +} + +/** Where a recording is: off, waiting for the microphone, recording, or waiting for dictated text. */ +export type ChatFormVoiceState = 'idle' | 'starting' | 'recording' | 'transcribing'; + +interface UseChatFormVoiceOptions { + getRecorder: () => VoiceRecorderLike | undefined; + getValue: () => string; + /** Writes the message as dictation changes it; not a user edit. */ + setValue: (value: string) => void; + getCaretOffset: () => number; + /** Puts the caret after the dictated text once it is final. */ + setCaretOffset: (offset: number) => void; + addVoiceFile: (file: ChatUploadedFile) => void; + /** Why the last start failed, for the microphone button; null once one worked. */ + onMicError: (message: string | null) => void; +} + +const ELAPSED_TICK_MS = 200; + +function readHintSeen(): boolean { + try { + return localStorage.getItem(VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY) === 'true'; + } catch { + return false; + } +} + +function writeHintSeen(): void { + try { + localStorage.setItem(VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY, 'true'); + } catch { + // private mode: the hint shows again next time + } +} + +function readAsDataUrl(blob: Blob): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + + reader.onload = () => resolve(reader.result as string); + reader.onerror = () => reject(reader.error); + reader.readAsDataURL(blob); + }); +} + +/** The microphone's server: its routes, its auth, and whether a websocket reaches it. */ +function serverContext(): { context: VoiceServerContext; realtime: boolean } { + const headers = getAuthHeaders(); + // a browser websocket can not send the API key + const hasAuth = Object.keys(headers).some((key) => key.toLowerCase() === 'authorization'); + + return { + context: { + getHeaders: () => getAuthHeaders(), + resolveUrl: (path) => withBase(path.replace(/^\./, '')) + }, + realtime: !hasAuth + }; +} + +export function useChatFormVoice(options: UseChatFormVoiceOptions) { + const config = $derived(settingsStore.config); + const isRouter = $derived(serverStore.isRouterMode); + const chatModel = $derived(modelsStore.activeModelId); + const transcriptionSetting = $derived( + String(config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL] ?? '') + ); + const hasAudio = (model: string | null) => { + void modelsStore.props.cacheVersion; + + return model ? modelsStore.props.modelSupportsAudio(model) : false; + }; + const resolution = $derived( + resolveVoiceMode({ + chatModel, + chatModelHasAudio: hasAudio(chatModel), + isRouter, + setting: String(config[SETTINGS_KEYS.VOICE_INPUT_MODE] ?? VoiceInputMode.AUTO), + transcriptionModel: transcriptionSetting, + transcriptionModelHasAudio: hasAudio(transcriptionSetting || null) + }) + ); + // a router with a model that hears keeps the button, so it can say where to pick it + const anyAudioModel = $derived.by(() => { + void modelsStore.props.cacheVersion; + + return isRouter && modelsStore.models.some((option) => hasAudio(option.model)); + }); + + let hintSeen = $state(readHintSeen()); + let state = $state('idle'); + let elapsedMs = $state(0); + // the mode of the recording under way, fixed when it starts + let recordingMode = $state(VoiceMode.UNAVAILABLE); + let ticker: ReturnType | undefined; + let span: DictationSpan | null = null; + let lastValue = ''; + // bumped by cancel, so a transcript still on its way is dropped + let generation = 0; + + const recorder = $derived(options.getRecorder()); + const unavailableReason = $derived( + resolution.mode === VoiceMode.UNAVAILABLE ? resolution.reason : null + ); + const hint = $derived( + resolution.mode === VoiceMode.DICTATION && resolution.separateModel && !hintSeen + ? VOICE_MODE_MESSAGE.separateModelHint + : null + ); + + function startTicker() { + const startedAt = Date.now(); + + elapsedMs = 0; + ticker = setInterval(() => (elapsedMs = Date.now() - startedAt), ELAPSED_TICK_MS); + } + + function stopTicker() { + clearInterval(ticker); + ticker = undefined; + } + + function writeDictation(text: string) { + if (!span) return; + + const next = span.apply(options.getValue(), text); + + lastValue = next; + options.setValue(next); + } + + async function start() { + if (!recorder || state !== 'idle' || resolution.mode === VoiceMode.UNAVAILABLE) return; + + const mode = resolution.mode; + const model = resolution.transcriptionModel; + const { context, realtime } = serverContext(); + const language = String(config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE] ?? '').trim(); + const transcribe = + mode === VoiceMode.DICTATION || config[SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES] !== false; + + if (resolution.mode === VoiceMode.DICTATION && resolution.separateModel && !hintSeen) { + hintSeen = true; + writeHintSeen(); + } + + recordingMode = mode; + state = 'starting'; + span = null; + + if (mode === VoiceMode.DICTATION) { + lastValue = options.getValue(); + span = new DictationSpan(lastValue, options.getCaretOffset()); + } + + const run = ++generation; + + try { + await recorder.start({ + onText: mode === VoiceMode.DICTATION ? (text) => writeDictation(text) : undefined, + realtime, + transcribe: transcribe + ? { + context, + language: language || undefined, + model: isRouter && model ? model : undefined + } + : undefined + }); + } catch (error) { + if (run !== generation) return; + + state = 'idle'; + span = null; + + const message = + error instanceof VoiceCaptureError ? error.message : describeMicrophoneError(error); + + options.onMicError(message); + toast.error(message); + + return; + } + + if (run !== generation) return; + + options.onMicError(null); + state = 'recording'; + startTicker(); + } + + async function stop() { + if (!recorder || state !== 'recording') return; + + const run = generation; + const mode = recordingMode; + + stopTicker(); + + let result: VoiceSessionResult; + + try { + result = await recorder.stop(); + } catch (error) { + console.error('Failed to stop the recording:', error); + state = 'idle'; + + return; + } + + if (run !== generation) return; + + if (mode === VoiceMode.VOICE_MESSAGE) { + state = 'idle'; + await attachVoiceMessage(result); + + return; + } + + await finishDictation(result, run); + } + + async function attachVoiceMessage({ clip, transcript }: VoiceSessionResult) { + const name = `${VOICE_CLIP_NAME_PREFIX}${Date.now()}-${uuid().slice(0, 8)}.wav`; + const file = new File([clip.blob], name, { type: MimeTypeAudio.WAV }); + + if (transcript) voiceTranscriptsStore.track(name, transcript); + + options.addVoiceFile({ + file, + id: uuid(), + name, + preview: await readAsDataUrl(clip.blob), + size: file.size, + type: MimeTypeAudio.WAV, + voice: { durationMs: clip.durationMs, peaks: clip.peaks } + }); + } + + async function finishDictation({ liveText, transcript }: VoiceSessionResult, run: number) { + const dictated = span; + + if (!transcript || !dictated) { + state = 'idle'; + + return; + } + + // with live text the message already shows it; otherwise wait for the text + if (!liveText) state = 'transcribing'; + else state = 'idle'; + + try { + const text = await transcript; + + if (run !== generation) return; + + if (text.trim() || !liveText) writeDictation(text); + } catch (error) { + if (run !== generation) return; + + toast.error(error instanceof Error ? error.message : 'Transcription failed'); + } + + state = 'idle'; + span = null; + + if (!dictated.isEmpty) options.setCaretOffset(dictated.end); + } + + /** Drops the recording and any text it dictated. */ + function cancel() { + generation++; + stopTicker(); + recorder?.cancel(); + + if (span) { + const next = span.remove(options.getValue()); + + options.setValue(next); + span = null; + } + + state = 'idle'; + } + + /** Follows an edit the user typed while dictation writes into the message. */ + function handleUserInput() { + const next = options.getValue(); + + span?.edited(lastValue, next); + lastValue = next; + } + + onDestroy(() => { + if (state !== 'idle') cancel(); + }); + + return { + cancel, + get elapsedMs() { + return elapsedMs; + }, + handleUserInput, + get hint() { + return hint; + }, + /** A recording (or its dictated text) is under way: the form shows the recording bar. */ + get isActive() { + return state !== 'idle'; + }, + /** What the microphone button says it will do. */ + get label() { + return voiceModeLabel(resolution); + }, + get levels() { + return recorder?.levels ?? []; + }, + /** Text streaming while a voice message is recorded, shown under the waveform. */ + get liveCaption() { + return recordingMode === VoiceMode.VOICE_MESSAGE ? (recorder?.liveText ?? '') : ''; + }, + get mode() { + return state === 'idle' ? resolution.mode : recordingMode; + }, + get resolution() { + return resolution; + }, + /** Whether the form shows a microphone button at all. */ + get showButton() { + return resolution.mode !== VoiceMode.UNAVAILABLE || anyAudioModel; + }, + start, + get state() { + return state; + }, + stop, + get unavailableReason() { + return unavailableReason; + } + }; +} diff --git a/ui/apps/chat/src/lib/utils/dictation-span.ts b/ui/apps/chat/src/lib/utils/dictation-span.ts new file mode 100644 index 000000000000..b8cb088d7dee --- /dev/null +++ b/ui/apps/chat/src/lib/utils/dictation-span.ts @@ -0,0 +1,90 @@ +/** + * The stretch of the message that dictation writes: it starts at the caret, and each + * new version of the dictated text replaces only that stretch, so what the user typed + * before and after it stays. An edit before the stretch moves it; an edit inside it + * hands that text over to the user, and dictation goes on after it. + */ +export class DictationSpan { + // characters of the dictated text already handed over to the user + private handedOver = 0; + private length = 0; + // characters of the dictated text the stretch shows + private shown = 0; + private start: number; + + /** Where the dictated text ends, e.g. to put the caret after it. */ + get end(): number { + return this.start + this.length; + } + + /** Whether the stretch shows any text. */ + get isEmpty(): boolean { + return this.length === 0; + } + + /** + * Writes the dictated text so far into `value` in place of the stretch, with a space + * on either side where it would otherwise run into the neighbouring text. + */ + apply(value: string, text: string): string { + const body = text.slice(this.handedOver).trim(); + const before = value.slice(0, this.start); + const after = value.slice(this.start + this.length); + const lead = body && before && !/\s$/.test(before) ? ' ' : ''; + const trail = body && after && !/^\s/.test(after) ? ' ' : ''; + const inserted = lead + body + trail; + + this.length = inserted.length; + this.shown = text.length; + + return before + inserted + after; + } + + constructor(value: string, caret: number) { + this.start = Math.max(0, Math.min(caret, value.length)); + } + + /** Follows an edit the user made, `previous` to `next`. */ + edited(previous: string, next: string): void { + const { from, insert, to } = changeBetween(previous, next); + const end = this.end; + const delta = insert.length - (to - from); + + if (to <= this.start) { + this.start += delta; + } else if (from >= end) { + return; + } else { + // typed into the dictated text: it is theirs now, what comes next goes after it + this.handedOver = this.shown; + this.start = to <= end ? end + delta : from + insert.length; + this.length = 0; + } + } + + /** Takes the stretch out of `value`, as a cancelled dictation leaves the message. */ + remove(value: string): string { + const next = value.slice(0, this.start) + value.slice(this.start + this.length); + + this.length = 0; + + return next; + } +} + +/** The single change that turns `current` into `next`: their differing middle. */ +function changeBetween(current: string, next: string) { + let from = 0; + + while (from < current.length && from < next.length && current[from] === next[from]) from++; + + let currentTo = current.length; + let nextTo = next.length; + + while (currentTo > from && nextTo > from && current[currentTo - 1] === next[nextTo - 1]) { + currentTo--; + nextTo--; + } + + return { from, insert: next.slice(from, nextTo), to: currentTo }; +} diff --git a/ui/apps/chat/tests/client/components/ChatFormActionRecordWrapper.svelte b/ui/apps/chat/tests/client/components/ChatFormActionRecordWrapper.svelte index cdc4a46ed3f6..d836cd6dd057 100644 --- a/ui/apps/chat/tests/client/components/ChatFormActionRecordWrapper.svelte +++ b/ui/apps/chat/tests/client/components/ChatFormActionRecordWrapper.svelte @@ -3,12 +3,16 @@ import ChatFormActionRecord from '#lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte'; let { - hasAudioModality = true, micWarning = null, - onMicClick - }: { hasAudioModality?: boolean; micWarning?: string | null; onMicClick?: () => void } = $props(); + onMicClick, + unavailableReason = null + }: { + micWarning?: string | null; + onMicClick?: () => void; + unavailableReason?: string | null; + } = $props(); - + diff --git a/ui/apps/chat/tests/client/microphone-unavailable.svelte.test.ts b/ui/apps/chat/tests/client/microphone-unavailable.svelte.test.ts index 361506341c61..47ea061c4279 100644 --- a/ui/apps/chat/tests/client/microphone-unavailable.svelte.test.ts +++ b/ui/apps/chat/tests/client/microphone-unavailable.svelte.test.ts @@ -4,7 +4,7 @@ // while staying tappable, so a tap on a phone, which shows no tooltip, can explain it. import ChatFormActionRecordWrapper from './components/ChatFormActionRecordWrapper.svelte'; -import { MICROPHONE_UNAVAILABLE_MESSAGE } from '#lib/constants/index.ts'; +import { MICROPHONE_UNAVAILABLE_MESSAGE, VOICE_MODE_MESSAGE } from '#lib/constants/index.ts'; import { describeMicrophoneError, microphoneUnavailableReason } from '#lib/utils/browser-only.ts'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { render } from 'vitest-browser-svelte'; @@ -71,18 +71,18 @@ describe('the record button', () => { const screen = render(ChatFormActionRecordWrapper, {}); await expect - .element(screen.getByRole('button', { name: 'Start recording' }).last()) + .element(screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last()) .toBeEnabled(); }); - it('stays disabled for a model without audio, whatever the microphone', async () => { + it('stays disabled when no model can take audio, whatever the microphone', async () => { const screen = render(ChatFormActionRecordWrapper, { - hasAudioModality: false, - micWarning: MICROPHONE_UNAVAILABLE_MESSAGE.insecure + micWarning: MICROPHONE_UNAVAILABLE_MESSAGE.insecure, + unavailableReason: VOICE_MODE_MESSAGE.unavailable }); await expect - .element(screen.getByRole('button', { name: 'Start recording' }).last()) + .element(screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last()) .toBeDisabled(); }); }); diff --git a/ui/apps/chat/tests/stories/chat/form/ChatFormActionRecord.stories.svelte b/ui/apps/chat/tests/stories/chat/form/ChatFormActionRecord.stories.svelte index 476e9397c469..66143acfa1f9 100644 --- a/ui/apps/chat/tests/stories/chat/form/ChatFormActionRecord.stories.svelte +++ b/ui/apps/chat/tests/stories/chat/form/ChatFormActionRecord.stories.svelte @@ -1,8 +1,10 @@ - - - + + + - - + + + + + + - + - + - - + + { + const button = canvas.getAllByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }); + + await expect(button.at(-1)).toBeDisabled(); // the tooltip trigger wraps the disabled button, hover the trigger - await userEvent.hover(canvas.getAllByRole('button', { name: 'Start recording' })[0]); + await userEvent.hover(button[0]); + await expect( + (await screen.findAllByText(VOICE_MODE_MESSAGE.unavailable)).length + ).toBeGreaterThan(0); + }} +/> + + { + await userEvent.hover(canvas.getAllByRole('button', { name: VOICE_MODE_MESSAGE.dictate })[0]); + await expect((await screen.findAllByText(/alongside the chat model/)).length).toBeGreaterThan( + 0 + ); }} /> diff --git a/ui/apps/chat/tests/stories/voice/VoiceLiveDemo.svelte b/ui/apps/chat/tests/stories/voice/VoiceLiveDemo.svelte index c370f377496a..d85786b6dc58 100644 --- a/ui/apps/chat/tests/stories/voice/VoiceLiveDemo.svelte +++ b/ui/apps/chat/tests/stories/voice/VoiceLiveDemo.svelte @@ -50,5 +50,5 @@ {formatClipTime(elapsedMs)} - + diff --git a/ui/apps/chat/tests/unit/dictation-span.test.ts b/ui/apps/chat/tests/unit/dictation-span.test.ts new file mode 100644 index 000000000000..13f081be479d --- /dev/null +++ b/ui/apps/chat/tests/unit/dictation-span.test.ts @@ -0,0 +1,90 @@ +// Guards how dictation writes into the message: at the caret, replacing only its own text +// as the transcript grows, and keeping what the user typed around it. + +import { DictationSpan } from '#lib/utils/dictation-span.ts'; +import { describe, expect, it } from 'vitest'; + +describe('DictationSpan', () => { + it('writes into an empty message and replaces its own text as it grows', () => { + const span = new DictationSpan('', 0); + + let value = span.apply('', 'Hello'); + + expect(value).toBe('Hello'); + + value = span.apply(value, 'Hello world'); + expect(value).toBe('Hello world'); + expect(span.end).toBe(value.length); + }); + + it('inserts at the caret with spaces, keeping the text before and after', () => { + const start = 'Translate into German'; + const span = new DictationSpan(start, 'Translate'.length); + + let value = span.apply(start, 'good'); + + expect(value).toBe('Translate good into German'); + + value = span.apply(value, 'good morning'); + expect(value).toBe('Translate good morning into German'); + }); + + it('adds a space where the dictated text would run into a word', () => { + const span = new DictationSpan('Say', 3); + + expect(span.apply('Say', 'hi')).toBe('Say hi'); + + const middle = new DictationSpan('ab', 1); + + expect(middle.apply('ab', 'x')).toBe('a x b'); + }); + + it('follows text typed before it while it streams', () => { + const span = new DictationSpan('', 0); + + let value = span.apply('', 'one'); + + const typed = `Note: ${value}`; + + span.edited(value, typed); + value = span.apply(typed, 'one two'); + + expect(value).toBe('Note: one two'); + }); + + it('keeps text typed after it', () => { + const span = new DictationSpan('', 0); + + let value = span.apply('', 'one'); + + const typed = `${value}!`; + + span.edited(value, typed); + value = span.apply(typed, 'one two'); + + expect(value).toBe('one two !'); + }); + + it('hands over text the user edits and dictates on after it', () => { + const span = new DictationSpan('', 0); + + let value = span.apply('', 'helo'); + + const fixed = 'hello'; + + span.edited(value, fixed); + value = span.apply(fixed, 'helo there'); + + expect(value).toBe('hello there'); + }); + + it('takes its text back out when the dictation is cancelled', () => { + const start = 'Before after'; + const span = new DictationSpan(start, 6); + const value = span.apply(start, 'middle'); + + expect(value).toBe('Before middle after'); + expect(span.remove(value)).toBe(start); + expect(span.isEmpty).toBe(true); + }); +}); From e0412fe79155f29a6dc2f23f49926d7a8237fccd Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Sun, 15 Nov 2026 21:41:53 +0100 Subject: [PATCH 07/19] ui: draw audio attachments as playable waveforms with their transcript A voice message should read as a clip, not as a file chip. Audio attachments now render with Llama UI's VoiceWaveform inside VoiceTranscriptReveal, through a direct branch in the core attachment item (voice is core, so it does not go through the extension renderers): the removable sm chip in the chat form, and the md clip in a sent message, on the user bubble's surface. - The audio plays from an object URL made from the stored base64 (or the uploaded file's data URL), revoked when the clip leaves the screen. - A clip stored without peaks, an older voice message or an uploaded audio file, gets them computed from its audio once per session, in memory only; uploaded files keep their name under the waveform. - The transcript opens under the clip on hover, keyboard focus or a tap, and shows Transcribing... while the voice transcripts store waits for it, Transcription unavailable when it failed, and nothing when none was asked for. - The full-screen preview keeps its player and shows the transcript. Tests run the chat form with a fake recorder: the microphone per mode, the recording row, dictation around typed text, the Transcribing... state, cancel, and a voice message chip whose transcript arrives later and goes with the send. Stories show the form recording each kind, a recorded chip, and sent voice messages with the transcript, pending, and computed peaks. Assisted-by: Claude (Opus 5.5 Medium) --- .../ChatAttachmentsListItem.svelte | 14 +- .../ChatAttachmentsListItemVoice.svelte | 116 +++++++++ ...tAttachmentsPreviewCurrentItemAudio.svelte | 29 ++- .../chat/src/lib/components/app/chat/index.ts | 8 + .../chat/src/lib/utils/voice-attachment.ts | 108 +++++++++ .../client/chat-form-voice.svelte.test.ts | 225 ++++++++++++++++++ .../ChatFormVoiceTestWrapper.svelte | 17 ++ .../chat/form/ChatFormVoice.stories.svelte | 117 +++++++++ .../chat/form/ChatFormVoiceStoryHost.svelte | 33 +++ .../messages/ChatMessageVoice.stories.svelte | 131 ++++++++++ .../voice/fake-voice-recorder.svelte.ts | 150 ++++++++++++ ui/apps/chat/tests/stories/voice/fixtures.ts | 46 +++- 12 files changed, 981 insertions(+), 13 deletions(-) create mode 100644 ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemVoice.svelte create mode 100644 ui/apps/chat/src/lib/utils/voice-attachment.ts create mode 100644 ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts create mode 100644 ui/apps/chat/tests/client/components/ChatFormVoiceTestWrapper.svelte create mode 100644 ui/apps/chat/tests/stories/chat/form/ChatFormVoice.stories.svelte create mode 100644 ui/apps/chat/tests/stories/chat/form/ChatFormVoiceStoryHost.svelte create mode 100644 ui/apps/chat/tests/stories/chat/messages/ChatMessageVoice.stories.svelte create mode 100644 ui/apps/chat/tests/stories/voice/fake-voice-recorder.svelte.ts diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte index 5bb7a47d71ea..eb1a8163c43a 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte @@ -3,7 +3,8 @@ ChatAttachmentsListItemMcpPrompt, ChatAttachmentsListItemMcpResource, ChatAttachmentsListItemThumbnailFile, - ChatAttachmentsListItemThumbnailImage + ChatAttachmentsListItemThumbnailImage, + ChatAttachmentsListItemVoice } from '#lib/components/app/index.ts'; import { AttachmentType } from '#lib/enums/index.ts'; import type { @@ -12,7 +13,7 @@ DatabaseMessageExtraMcpResource, MCPResourceAttachment } from '#lib/types/index.ts'; - import { isMcpPrompt, isMcpResource, isPdfFile } from '#lib/utils/index.ts'; + import { isAudioFile, isMcpPrompt, isMcpResource, isPdfFile } from '#lib/utils/index.ts'; interface Props { class?: string; @@ -93,6 +94,15 @@ class="flex-shrink-0 {className} {scrollClasses}" onclick={() => onMcpResourcePreview?.(mcpResource)} /> +{:else if isAudioFile(item.attachment, item.uploadedFile)} + + {:else if item.isImage && item.preview} + + +
+ + + + + {#if !source.voice} +

{source.name}

+ {/if} + + {#if !readonly && onRemove} +
+ onRemove?.(item.id)} + stopPropagationOnClick + tooltip="Remove" + /> +
+ {/if} +
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemAudio.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemAudio.svelte index 53653599f8a7..cfc3d738fb44 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemAudio.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemAudio.svelte @@ -1,12 +1,24 @@
@@ -21,6 +33,19 @@

Audio preview not available

{/if} -

{currentItem?.name || 'Audio'}

+

+ {source?.voice ? 'Voice message' : currentItem?.name || 'Audio'} +

+ + {#if transcript} +

+ Transcript: {transcript} +

+ {:else if transcribing} +

Transcribing...

+ {/if}
diff --git a/ui/apps/chat/src/lib/components/app/chat/index.ts b/ui/apps/chat/src/lib/components/app/chat/index.ts index 32d26aca5a60..f7626c73a4f5 100644 --- a/ui/apps/chat/src/lib/components/app/chat/index.ts +++ b/ui/apps/chat/src/lib/components/app/chat/index.ts @@ -79,6 +79,14 @@ export { default as ChatAttachmentsListItemMcpResource } from './ChatAttachments */ export { default as ChatAttachmentsListItemThumbnailFile } from './ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemThumbnailFile.svelte'; +/** + * An audio attachment drawn as a clip: a playable waveform (peaks computed from + * the audio when it was stored without them) and, for a voice message, its + * transcript under it on hover, focus or a tap, with the transcribing and failed + * states. `sm` in the chat form, removable; `md` in a sent message. + */ +export { default as ChatAttachmentsListItemVoice } from './ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemVoice.svelte'; + /** * Thumbnail for image attachments with lazy loading and error fallback. * Displays image preview with configurable dimensions. Falls back to placeholder diff --git a/ui/apps/chat/src/lib/utils/voice-attachment.ts b/ui/apps/chat/src/lib/utils/voice-attachment.ts new file mode 100644 index 000000000000..79380ee7f7fb --- /dev/null +++ b/ui/apps/chat/src/lib/utils/voice-attachment.ts @@ -0,0 +1,108 @@ +import { AttachmentType } from '#lib/enums/index.ts'; +import type { ChatAttachmentDisplayItem, DatabaseMessageExtraAudioFile } from '#lib/types/index.ts'; +import { computePeaksFromAudio } from '#lib/voice/index.ts'; + +// peaks of clips stored without them, kept for the session only +const peaksCache = new Map>(); + +/** What a list item needs to draw an audio attachment as a clip. */ +export interface VoiceAttachmentSource { + /** Base64 or a URL of the audio, or the file itself. */ + audio: string | Blob | null; + durationMs?: number; + mimeType: string; + name: string; + peaks?: number[]; + /** Recorded in the app, as opposed to an uploaded file. */ + voice: boolean; + transcript?: string; +} + +/** The clip behind an audio item of the form or of a message. */ +export function voiceAttachmentSource(item: ChatAttachmentDisplayItem): VoiceAttachmentSource { + const uploaded = item.uploadedFile; + + if (uploaded) { + return { + audio: uploaded.preview ?? (uploaded.file.size > 0 ? uploaded.file : null), + durationMs: uploaded.voice?.durationMs, + mimeType: uploaded.type, + name: uploaded.name, + peaks: uploaded.voice?.peaks, + voice: !!uploaded.voice + }; + } + + const attachment = + item.attachment?.type === AttachmentType.AUDIO + ? (item.attachment as DatabaseMessageExtraAudioFile) + : null; + + return { + audio: attachment?.base64Data || null, + durationMs: attachment?.durationMs, + mimeType: attachment?.mimeType ?? '', + name: item.name, + peaks: attachment?.peaks, + transcript: attachment?.transcript, + voice: attachment?.voice === true + }; +} + +/** A URL an