diff --git a/ui/apps/chat/.storybook/main.ts b/ui/apps/chat/.storybook/main.ts index 3e8c1a39d8dc..e2b8dad14377 100644 --- a/ui/apps/chat/.storybook/main.ts +++ b/ui/apps/chat/.storybook/main.ts @@ -16,7 +16,7 @@ const config: StorybookConfig = { '../tests/stories/**/*.stories.@(js|ts|svelte)', { directory: '../../../lib/ui/tests/stories', - files: '{foundations,components,app,models,mockups}/**/*.stories.svelte' + files: '{foundations,components,app,models,mockups,voice}/**/*.stories.svelte' } ], addons: [ diff --git a/ui/apps/chat/DESIGN.md b/ui/apps/chat/DESIGN.md index 971685e1bc51..79e5215286f3 100644 --- a/ui/apps/chat/DESIGN.md +++ b/ui/apps/chat/DESIGN.md @@ -1151,6 +1151,12 @@ icon buttons in the chat form are fully round. (`--sidebar-reveal-x`), taking no taps while the page is aside. The page keeps room under its last row (`max-md:pb-safe-24`) so the key never covers it. One per page, only for its main action; from `md` up the action stays a button in the header. +- **Voice clips:** a recorded clip is a round primary play key, its waveform and the time in + `caption`: bars in the muted foreground at 40%, the played part in the primary colour, so + the clip stays neutral in both modes. The live waveform while recording is as tall as the + input's line (24px), the newest level on the right, and holds still with reduced motion. A + voice message's transcript opens under the clip on hover, keyboard focus or a tap, and + stays in the DOM for screen readers while closed. - **Copy feedback:** a copy action turns its icon into a `fg-success` check for 1.5s (`COPY_FEEDBACK_RESET_MS`), only when the copy succeeded, then back. The icon pops in with a 150ms fade and scale, skipped on first render and with reduced motion. diff --git a/ui/apps/chat/scripts/llama-ui.js b/ui/apps/chat/scripts/llama-ui.js index c4eb26af77c9..f8963593a33b 100644 --- a/ui/apps/chat/scripts/llama-ui.js +++ b/ui/apps/chat/scripts/llama-ui.js @@ -26,6 +26,7 @@ const LLAMA_UI_EXPORTS = [ ['@ggml-org/llama-ui/enums', `${source}/enums/index.ts`], ['@ggml-org/llama-ui/code', `${source}/utils/index.ts`], ['@ggml-org/llama-ui/utils', `${source}/components/ui/utils.ts`], + ['@ggml-org/llama-ui/voice', `${source}/components/voice/index.ts`], ['@ggml-org/llama-ui/styles/*', `${source}/styles/*`], ['@ggml-org/llama-ui/theme.css', `${source}/styles/theme.css`], ['@ggml-org/llama-ui/tokens.css', `${source}/styles/tokens.css`], diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte index 5bb7a47d71ea..eb1a8163c43a 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte @@ -3,7 +3,8 @@ ChatAttachmentsListItemMcpPrompt, ChatAttachmentsListItemMcpResource, ChatAttachmentsListItemThumbnailFile, - ChatAttachmentsListItemThumbnailImage + ChatAttachmentsListItemThumbnailImage, + ChatAttachmentsListItemVoice } from '#lib/components/app/index.ts'; import { AttachmentType } from '#lib/enums/index.ts'; import type { @@ -12,7 +13,7 @@ DatabaseMessageExtraMcpResource, MCPResourceAttachment } from '#lib/types/index.ts'; - import { isMcpPrompt, isMcpResource, isPdfFile } from '#lib/utils/index.ts'; + import { isAudioFile, isMcpPrompt, isMcpResource, isPdfFile } from '#lib/utils/index.ts'; interface Props { class?: string; @@ -93,6 +94,15 @@ class="flex-shrink-0 {className} {scrollClasses}" onclick={() => onMcpResourcePreview?.(mcpResource)} /> +{:else if isAudioFile(item.attachment, item.uploadedFile)} + + {:else if item.isImage && item.preview} + + +
+ {#key collapsed} + + + + {/key} + + {#if !source.voice} +

{source.name}

+ {/if} + + {#if !readonly && onRemove} +
+ onRemove?.(item.id)} + stopPropagationOnClick + tooltip="Remove" + /> +
+ {/if} +
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemAudio.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemAudio.svelte index 53653599f8a7..cfc3d738fb44 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemAudio.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemAudio.svelte @@ -1,12 +1,24 @@
@@ -21,6 +33,19 @@

Audio preview not available

{/if} -

{currentItem?.name || 'Audio'}

+

+ {source?.voice ? 'Voice message' : currentItem?.name || 'Audio'} +

+ + {#if transcript} +

+ Transcript: {transcript} +

+ {:else if transcribing} +

Transcribing...

+ {/if}
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte index 0c44d125fb9c..5575783653ba 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -5,6 +5,7 @@ ActionIcon, ChatAttachmentsList, ChatFormActions, + ChatFormActionsRecording, ChatFormCurrentWorkingDirectory, ChatFormInput, ChatFormInputFileInputInvisible, @@ -30,6 +31,10 @@ } from '#lib/enums/index.ts'; import { useChatFormExpansion } from '#lib/hooks/use-chat-form-expansion.svelte.ts'; import { useChatFormPickers } from '#lib/hooks/use-chat-form-pickers.svelte.ts'; + import { + useChatFormVoice, + type VoiceRecorderLike + } from '#lib/hooks/use-chat-form-voice.svelte.ts'; import { chatStore, conversationsStore, @@ -39,7 +44,8 @@ serverStore, settingsStore, toolsStore, - uiStore + uiStore, + voiceTranscriptsStore } from '#lib/stores/index.ts'; import type { FileMentionEntry, @@ -48,14 +54,7 @@ MCPResourceInfo, PromptMessage } from '#lib/types/index.ts'; - import { - AudioRecorder, - convertToWav, - createAudioFile, - describeMicrophoneError, - isAudioRecordingSupported, - microphoneUnavailableReason - } from '#lib/utils/browser-only.ts'; + import { microphoneUnavailableReason } from '#lib/utils/browser-only.ts'; import { buildMentionInsertion, findCommandToken, @@ -65,7 +64,8 @@ parseClipboardContent, uuid } from '#lib/utils/index.ts'; - import { onMount, untrack } from 'svelte'; + import { VoiceRecorder, warmUpVoiceCapture } from '#lib/voice/index.ts'; + import { onMount, tick, untrack } from 'svelte'; import type { ClassValue } from 'svelte/elements'; import { toast } from 'svelte-sonner'; @@ -96,6 +96,9 @@ onUploadedFileRemove?: (fileId: string) => void; onUploadedFilesChange?: (files: ChatUploadedFile[]) => void; onValueChange?: (value: string) => void; + + /** Records voice input; tests and stories pass a fake, the app a VoiceRecorder. */ + voiceRecorder?: VoiceRecorderLike; } let { @@ -117,21 +120,20 @@ showModelSelector = true, singleRow = false, uploadedFiles = $bindable([]), - value = $bindable('') + value = $bindable(''), + voiceRecorder }: Props = $props(); // Component References - let audioRecorder: AudioRecorder | undefined; let chatFormActionsRef: ChatFormActions | undefined = $state(undefined); let fileInputRef: ChatFormInputFileInputInvisible | undefined = $state(undefined); let pickersRef: { handleKeydown: (event: KeyboardEvent) => boolean } | undefined = $state(undefined); let inputRef: ChatFormInput | undefined = $state(undefined); - // Audio Recording State - let isRecording = $state(false); - let recordingSupported = $state(false); + // Voice input + const recorder: VoiceRecorderLike = untrack(() => voiceRecorder) ?? new VoiceRecorder(); /** Why this page can't record at all (no HTTPS, no recorder), which no tap fixes. */ let micUnavailableReason = $state(null); /** Why the last start failed (denied, no microphone, busy); the next tap tries again. */ @@ -189,6 +191,25 @@ } }); + const voice = useChatFormVoice({ + addVoiceFile: (file) => { + uploadedFiles = [...uploadedFiles, file]; + onUploadedFilesChange?.(uploadedFiles); + }, + getCaretOffset: () => inputRef?.getCaretOffset() ?? value.length, + getRecorder: () => recorder, + getValue: () => value, + onMicError: (message) => (micError = message), + // on a phone placing the caret would open the keyboard + setCaretOffset: (offset) => { + if (!deviceStore.isMobile) inputRef?.setCaretOffset(offset); + }, + setValue: (next) => { + value = next; + onValueChange?.(next); + } + }); + async function handleWorkingDirectoryChange(newDir: string | null) { // Committing a directory consumes the `/cwd` token; the chip's // clear-X path has no token to consume. @@ -234,6 +255,11 @@ (attachments && attachments.length > 0) || (uploadedFiles && uploadedFiles.length > 0) ); let canSubmit = $derived(value.trim().length > 0 || hasAttachments); + // while a voice message records, what is being said shows in the empty message box; once + // it stops, the text goes with the clip, as its transcript + let voicePlaceholder = $derived( + voice.isActive && voice.liveCaption && !value.trim() ? voice.liveCaption : null + ); const EXPANDED_SHEET_FILL = 'max-md:bg-(--chat-form-sheet-fill)'; @@ -263,6 +289,7 @@ singleRow && !deviceStore.isMobile && !isExpanded && + !voice.isActive && !hasAttachments && !mcpStore.resources.hasAttachments && !value.includes('\n') @@ -327,9 +354,7 @@ ); onMount(() => { - recordingSupported = isAudioRecordingSupported(); micUnavailableReason = microphoneUnavailableReason(); - audioRecorder = new AudioRecorder(); }); /** Expands for the message as it stands, e.g. one put back after a send failed. */ @@ -375,6 +400,10 @@ onAttachmentRemove?.(index); } } else { + const removed = uploadedFiles.find((file) => file.id === fileId); + + if (removed?.voice) voiceTranscriptsStore.forget(removed.name); + onUploadedFileRemove?.(fileId); } } @@ -586,7 +615,7 @@ inputRef?.replaceText(built.newValue, built.caretOffset); } - async function handleMicClick() { + function handleMicClick() { // a phone shows no tooltip on a tap, so the reason comes as a toast if (micUnavailableReason) { toast.error(micUnavailableReason); @@ -594,36 +623,41 @@ return; } - if (!audioRecorder || !recordingSupported) { - console.warn('Audio recording not supported'); + void voice.start(); + } - return; + // the audio set-up takes a moment, so it is made while the message is being written; + // it asks for no microphone, so the browser shows no recording indicator + function handleFocusIn(event: FocusEvent) { + if (voice.showButton && !voice.isActive) warmUpVoiceCapture(); + + // the voice message's live text shown as the placeholder turns into text to edit + if (voicePlaceholder && (event.target as Element | null)?.closest('.cm-editor')) { + voice.adoptLiveText(); } + } - if (isRecording) { - isRecording = false; - try { - const audioBlob = await audioRecorder.stopRecording(); - const wavBlob = await convertToWav(audioBlob); - const audioFile = createAudioFile(wavBlob); + /** Stops the recording and sends the message once its text or clip is in the form. */ + async function handleStopAndSend() { + await voice.stop(); + await tick(); - onFilesAdd?.([audioFile]); - } catch (error) { - console.error('Failed to stop recording:', error); - } - } else { - try { - await audioRecorder.startRecording(); - isRecording = true; - micError = null; - } catch (error) { - micError = describeMicrophoneError(error); - toast.error(micError); - } - } + if (!canSubmit || disabled || hasLoadingAttachments || isLoading) return; + + onSubmit?.(); + } + + // Escape drops a recording, wherever the focus is + function handleWindowKeydown(event: KeyboardEvent) { + if (event.key !== KeyboardKey.ESCAPE || !voice.isActive || event.defaultPrevented) return; + + event.preventDefault(); + voice.cancel(); } + +
{ event.preventDefault(); @@ -755,13 +790,14 @@ {disabled} expanded={isExpanded} onInput={() => { + voice.handleUserInput(); expansion.handleInput(); pickers.handleInput(); onValueChange?.(value); }} onKeydown={handleKeydown} onPaste={handlePaste} - {placeholder} + placeholder={voicePlaceholder ?? placeholder} /> {#if mcpStore.resources.hasAttachments} @@ -776,29 +812,51 @@
- - onSystemPromptClick?.({ files: uploadedFiles, message: value })} - onWorkingDirectoryClick={isWorkingDirectoryInAddMenu - ? pickers.handleWorkingDirectoryOpen - : undefined} - {showAddButton} - {showModelSelector} - {uploadedFiles} - workingDirectory={cwd} - /> + {#if voice.isActive} + void handleStopAndSend()} + onStop={() => void voice.stop()} + state={voice.state} + /> + {/if} + + +
+ + onSystemPromptClick?.({ files: uploadedFiles, message: value })} + onWorkingDirectoryClick={isWorkingDirectoryInAddMenu + ? pickers.handleWorkingDirectoryOpen + : undefined} + {showAddButton} + {showModelSelector} + {uploadedFiles} + voice={{ + action: voice.label, + hint: voice.hint, + show: voice.showButton, + unavailableReason: voice.unavailableReason + }} + workingDirectory={cwd} + /> +
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte index 8664af6b2ea7..7c3dcfe49860 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte @@ -2,34 +2,73 @@ import { Button } from '@ggml-org/llama-ui/ui/button'; import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip'; import { Mic, MicOff, Square } from '@lucide/svelte'; - import { ICON_CLASS_DEFAULT } from '#lib/constants/index.ts'; + import { ICON_CLASS_DEFAULT, VOICE_MODE_MESSAGE } from '#lib/constants/index.ts'; interface Props { + /** What a press does: send a voice message or dictate. */ + action?: string; class?: string; disabled?: boolean; - hasAudioModality?: boolean; + /** Said after the action in the tooltip, e.g. that another model may be loaded. */ + hint?: string | null; isLoading?: boolean; isRecording?: boolean; /** Why the microphone can't record (no HTTPS, access denied...), shown on the button. */ micWarning?: string | null; + /** Why no model can take the recording; disables the button. */ + unavailableReason?: string | null; + /** A quieter button, beside the main one (stop or send). */ + secondary?: boolean; onMicClick?: () => void; } let { + action = VOICE_MODE_MESSAGE.voiceMessage, class: className = '', disabled = false, - hasAudioModality = false, + hint = null, isLoading = false, isRecording = false, micWarning = null, - onMicClick + onMicClick, + secondary = false, + unavailableReason = null }: Props = $props(); - // a model without audio input disables the button; a microphone that can't record keeps it + // no model to take the recording disables the button; a microphone that can't record keeps it // tappable, so a tap can say why - const showWarning = $derived(hasAudioModality && !isRecording && !!micWarning); + const showWarning = $derived(!unavailableReason && !isRecording && !!micWarning); + // a mouse starts recording on press, not on release, so the first words are not lost; + // a touch or a key goes through the click, which is when the browser allows it + const PRESS_CLICK_MS = 1000; + let pressedAt = -Infinity; + + function handlePointerDown(event: PointerEvent) { + if (isRecording || disabled || isLoading || unavailableReason) return; + + if (event.pointerType !== 'mouse' || event.button !== 0) return; + + pressedAt = performance.now(); + onMicClick?.(); + } + + function handleClick() { + // the click that ends the press that already started it; the recording bar may have + // taken the button's place meanwhile, so this one can also never come + if (performance.now() - pressedAt < PRESS_CLICK_MS) { + pressedAt = -Infinity; + + return; + } + + onMicClick?.(); + } + const tooltip = $derived( - !hasAudioModality ? 'Current model does not support audio' : showWarning ? micWarning : null + isRecording + ? null + : (unavailableReason ?? + (showWarning ? micWarning : [action, hint].filter(Boolean).join('. '))) ); @@ -37,28 +76,31 @@ {/if} - {#if isLoading && !canSend} + {#if shouldShowRecordBeside} + + {/if} + + {#if showStop} + + {#if state === 'transcribing'} +
+ + {VOICE_MODE_MESSAGE.transcribing} +
+ {:else} + + + + {#if state === 'starting'} + + {VOICE_MODE_MESSAGE.starting} + + {:else} + + {formatClipTime(elapsedMs)} + + {/if} + {/if} + + + + {#if onSend} + + {/if} + + + {#if liveCaption} +

+ +

+ {/if} + diff --git a/ui/apps/chat/src/lib/components/app/chat/index.ts b/ui/apps/chat/src/lib/components/app/chat/index.ts index fce3962c4fe4..f7626c73a4f5 100644 --- a/ui/apps/chat/src/lib/components/app/chat/index.ts +++ b/ui/apps/chat/src/lib/components/app/chat/index.ts @@ -79,6 +79,14 @@ export { default as ChatAttachmentsListItemMcpResource } from './ChatAttachments */ export { default as ChatAttachmentsListItemThumbnailFile } from './ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemThumbnailFile.svelte'; +/** + * An audio attachment drawn as a clip: a playable waveform (peaks computed from + * the audio when it was stored without them) and, for a voice message, its + * transcript under it on hover, focus or a tap, with the transcribing and failed + * states. `sm` in the chat form, removable; `md` in a sent message. + */ +export { default as ChatAttachmentsListItemVoice } from './ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemVoice.svelte'; + /** * Thumbnail for image attachments with lazy loading and error fallback. * Displays image preview with configurable dimensions. Falls back to placeholder @@ -170,12 +178,20 @@ export { default as ChatForm } from './ChatForm/ChatForm.svelte'; export { default as ChatFormActionsAdd } from './ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte'; /** - * Audio recording button with real-time recording indicator. Records audio - * and converts to WAV format for upload. Only visible when the active model - * supports audio modality and setting for automatic audio input is enabled. Shows recording duration while active. + * Microphone button: starts a voice message or a dictation (its label says which), + * or, disabled, says why no model can take a recording; while recording it is the + * stop button. Shown on an empty message when voice input is available and the + * setting for the microphone on empty input is on. */ export { default as ChatFormActionRecord } from './ChatForm/ChatFormActions/ChatFormActionRecord.svelte'; +/** + * The actions row while the chat form records: cancel, the live waveform and the + * elapsed time, and stop, with a voice message's streaming text under the waveform + * and a waiting state while dictated text is transcribed after recording. + */ +export { default as ChatFormActionsRecording } from './ChatForm/ChatFormActions/ChatFormActionsRecording.svelte'; + /** * Container for chat form action buttons. Arranges file attachment, audio record, * and submit/stop buttons in a horizontal layout. Handles conditional visibility diff --git a/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte b/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte index 1ced25e50f2f..4c1e3281ef9b 100644 --- a/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte +++ b/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte @@ -7,6 +7,7 @@ SettingsChatImportExportTab, SettingsChatMobileHeader, SettingsChatToolsTab, + SettingsChatVoiceTab, SettingsFooter, SettingsResetButton } from '#lib/components/app/settings/index.ts'; @@ -181,6 +182,12 @@ {:else if currentSection.slug === SETTINGS_SECTION_SLUGS.IMPORT_EXPORT} + {:else if currentSection.slug === SETTINGS_SECTION_SLUGS.VOICE && currentSection.fields} + {:else if currentSection.fields}
+ + +
+ + +
+ + + onConfigChange(MIC_KEY, value)} + type="single" + value={micCurrent} + > + {micLabel} + + + {#each micOptions as option (option.value)} + {option.label} + {/each} + + + +

{SETTING_CONFIG_INFO[MIC_KEY]}

+ + {#if micNamesHidden} +

+ The browser names your microphones once you have allowed one, e.g. after your first + recording. +

+ {/if} +
+ +
+ + + {#if isRouter} + onConfigChange(KEY, value)} + type="single" + value={current} + > + {selectedLabel} + + + {#each options as option (option.value)} + {option.label} + {/each} + + + +

+ {SETTING_CONFIG_INFO[KEY]} Only models with audio input are listed. +

+ +

+ A model you pick here is loaded on the server when you dictate, alongside the chat model, if + it isn't loaded already. A server that keeps one model at a time (--models-max 1) unloads the chat model to do it. +

+ {:else} +

+ {#if singleModel} + This server runs one model, {singleModel}. + {singleModelHasAudio + ? 'It has audio input, so it transcribes your voice.' + : 'It has no audio input, so voice input is not available on this server.'} + {:else} + This server runs one model, which transcribes your voice when it has audio input. + {/if} +

+ {/if} +
+ + +
diff --git a/ui/apps/chat/src/lib/components/app/settings/index.ts b/ui/apps/chat/src/lib/components/app/settings/index.ts index 2d2aca5af17a..2d44c88921ba 100644 --- a/ui/apps/chat/src/lib/components/app/settings/index.ts +++ b/ui/apps/chat/src/lib/components/app/settings/index.ts @@ -52,6 +52,15 @@ export { default as SettingsResetButton } from './SettingsResetButton.svelte'; */ export { default as SettingsChatImportExportTab } from './SettingsChat/SettingsChatImportExportTab.svelte'; +/** + * **SettingsChatVoiceTab** - Voice section of the settings + * + * The registry's voice fields, with the transcription model picker between them: + * in router mode it lists the server's models with audio input, on a single-model + * server it says whether that model can transcribe. + */ +export { default as SettingsChatVoiceTab } from './SettingsChat/SettingsChatVoiceTab.svelte'; + /** * Section wrapper for import/export sections. Displays a title, description, * icon button, and optional summary of recent actions. diff --git a/ui/apps/chat/src/lib/constants/index.ts b/ui/apps/chat/src/lib/constants/index.ts index 46feb4a6dd4a..d7a8c982f403 100644 --- a/ui/apps/chat/src/lib/constants/index.ts +++ b/ui/apps/chat/src/lib/constants/index.ts @@ -74,3 +74,4 @@ export * from './read-media'; export * from './get-datetime'; export * from './browser-info'; export * from './search-chats'; +export * from './voice.constants'; diff --git a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts index c94f97ff7d72..ef274927ec4a 100644 --- a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts +++ b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts @@ -85,6 +85,13 @@ export const SETTINGS_KEYS = { TYP_P: 'typ_p', // General USE_HUGGING_FACE_HUB: 'useHuggingFaceHub', + // Voice + VOICE_COLLAPSE_TRANSCRIPTS: 'voiceCollapseTranscripts', + VOICE_INPUT_MODE: 'voiceInputMode', + VOICE_MICROPHONE: 'voiceMicrophone', + VOICE_TRANSCRIBE_MESSAGES: 'voiceTranscribeMessages', + VOICE_TRANSCRIPTION_LANGUAGE: 'voiceTranscriptionLanguage', + VOICE_TRANSCRIPTION_MODEL: 'voiceTranscriptionModel', XTC_PROBABILITY: 'xtc_probability', XTC_THRESHOLD: 'xtc_threshold' } as const; diff --git a/ui/apps/chat/src/lib/constants/settings.constants.ts b/ui/apps/chat/src/lib/constants/settings.constants.ts index 669ccd6ecbb4..050a72580619 100644 --- a/ui/apps/chat/src/lib/constants/settings.constants.ts +++ b/ui/apps/chat/src/lib/constants/settings.constants.ts @@ -4,21 +4,25 @@ import { COMPACTION } from './compaction.constants'; import { DEFAULT_MCP_CONFIG } from './mcp.constants'; import { SETTINGS_KEYS } from './settings-keys.constants'; import { TITLE_GENERATION } from './title-generation.constants'; +import { VOICE_TRANSCRIPTION_MODEL_AUTO } from './voice.constants'; import { FILE_GLOB_SEARCH_PICKERS } from './working-directory.constants'; import { + AudioLines, Code, Database, File, Funnel, GlassWater, ListRestart, + Mic, Monitor, Moon, PencilRuler, SlidersVertical, - Sun + Sun, + TextCursorInput } from '@lucide/svelte'; -import { SurfaceTheme, SyncableParameterType } from '#lib/enums/index.ts'; +import { SurfaceTheme, SyncableParameterType, VoiceInputMode } from '#lib/enums/index.ts'; import { SettingsFieldType } from '#lib/enums/settings.enums.ts'; import { ColorMode } from '#lib/enums/ui.enums.ts'; import type { @@ -28,6 +32,7 @@ import type { SettingsSection, SettingsSectionEntry } from '#lib/types/index.ts'; +import { VOICE_MICROPHONE_AUTO } from '#lib/voice/microphone.ts'; /** Settings sections — slug is the routing identity, title is the display label. */ export const SETTINGS_SECTIONS = { @@ -37,7 +42,8 @@ export const SETTINGS_SECTIONS = { GENERAL: { slug: 'general', title: 'General' }, IMPORT_EXPORT: { slug: 'import-export', title: 'Import/Export' }, SAMPLING_PENALTIES: { slug: 'sampling-penalties', title: 'Sampling & Penalties' }, - TOOLS: { slug: 'tools', title: 'Tools' } + TOOLS: { slug: 'tools', title: 'Tools' }, + VOICE: { slug: 'voice', title: 'Voice' } } as const; export const SETTINGS_SECTION_SLUGS = { @@ -47,7 +53,8 @@ export const SETTINGS_SECTION_SLUGS = { GENERAL: SETTINGS_SECTIONS.GENERAL.slug, IMPORT_EXPORT: SETTINGS_SECTIONS.IMPORT_EXPORT.slug, SAMPLING_PENALTIES: SETTINGS_SECTIONS.SAMPLING_PENALTIES.slug, - TOOLS: SETTINGS_SECTIONS.TOOLS.slug + TOOLS: SETTINGS_SECTIONS.TOOLS.slug, + VOICE: SETTINGS_SECTIONS.VOICE.slug } as const; export const SETTINGS_SECTION_TITLES = { @@ -57,7 +64,8 @@ export const SETTINGS_SECTION_TITLES = { GENERAL: SETTINGS_SECTIONS.GENERAL.title, IMPORT_EXPORT: SETTINGS_SECTIONS.IMPORT_EXPORT.title, SAMPLING_PENALTIES: SETTINGS_SECTIONS.SAMPLING_PENALTIES.title, - TOOLS: SETTINGS_SECTIONS.TOOLS.title + TOOLS: SETTINGS_SECTIONS.TOOLS.title, + VOICE: SETTINGS_SECTIONS.VOICE.title } as const; export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [ @@ -134,7 +142,7 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [ }, { defaultValue: true, - help: 'Automatically show microphone button instead of send button when textarea is empty for models with audio modality support.', + help: 'Show the microphone in place of the send button while the message is empty, whenever voice input is available (see Voice).', key: SETTINGS_KEYS.AUTO_MIC_ON_EMPTY, label: 'Show microphone on empty input', type: SettingsFieldType.CHECKBOX @@ -393,6 +401,63 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [ slug: SETTINGS_SECTION_SLUGS.DISPLAY, title: SETTINGS_SECTION_TITLES.DISPLAY }, + // Voice: the transcription model picker lists the server's models (SettingsChatVoiceTab) + { + icon: Mic, + settings: [ + { + defaultValue: VoiceInputMode.AUTO, + help: 'Automatic sends your voice to a model that understands audio, as a voice message with its transcript, and turns it into text in the message input for any other model. Always dictate turns it into text for every model, so only the text reaches the chat.', + key: SETTINGS_KEYS.VOICE_INPUT_MODE, + label: 'Voice input', + options: [ + { icon: AudioLines, label: 'Automatic', value: VoiceInputMode.AUTO }, + { icon: TextCursorInput, label: 'Always dictate', value: VoiceInputMode.DICTATION } + ], + type: SettingsFieldType.SELECT + }, + { + defaultValue: VOICE_TRANSCRIPTION_MODEL_AUTO, + help: 'The model that turns speech into text when the chat model has no audio input. Automatic uses a model with audio input that is already loaded, so dictating loads nothing.', + key: SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL, + label: 'Transcription model', + standaloneField: false, + type: SettingsFieldType.SELECT + }, + { + defaultValue: VOICE_MICROPHONE_AUTO, + help: "The microphone you record from. Automatic takes the system's default one, unless that is a Bluetooth headset's: recording from it switches the headset to its call mode, which plays music in low quality, so the built-in microphone is used instead.", + key: SETTINGS_KEYS.VOICE_MICROPHONE, + label: 'Microphone', + standaloneField: false, + type: SettingsFieldType.SELECT + }, + { + defaultValue: true, + help: 'Write down what was said in each voice message, to read under its waveform. The chat model writes it, and a model without audio input gets it in place of the audio.', + key: SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES, + label: 'Transcribe voice messages', + type: SettingsFieldType.CHECKBOX + }, + { + defaultValue: false, + help: 'Show the transcript of a voice message folded under its waveform, to open on hover or with a tap, instead of open.', + key: SETTINGS_KEYS.VOICE_COLLAPSE_TRANSCRIPTS, + label: 'Collapse transcripts', + type: SettingsFieldType.CHECKBOX + }, + { + defaultValue: '', + help: 'The language you speak, as a two-letter code such as en or de, passed to the transcription. Leave it empty to let the model tell.', + key: SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE, + label: 'Transcription language', + placeholder: 'Automatic', + type: SettingsFieldType.INPUT + } + ], + slug: SETTINGS_SECTION_SLUGS.VOICE, + title: SETTINGS_SECTION_TITLES.VOICE + }, // MCP Servers (non-UI config object) { icon: PencilRuler, diff --git a/ui/apps/chat/src/lib/constants/voice.constants.ts b/ui/apps/chat/src/lib/constants/voice.constants.ts new file mode 100644 index 000000000000..c46b3fcdb009 --- /dev/null +++ b/ui/apps/chat/src/lib/constants/voice.constants.ts @@ -0,0 +1,27 @@ +import { STORAGE_APP_NAME } from './storage.constants'; + +/** What the chat form's microphone says it will do, or why it can't, on its button. */ +export const VOICE_MODE_MESSAGE = { + dictate: 'Dictate', + dictating: 'Dictating', + recording: 'Recording a voice message', + separateModelHint: + 'The transcription model is loaded on the server alongside the chat model, which a server that keeps one model at a time unloads.', + starting: 'Starting...', + transcribing: 'Transcribing...', + unavailable: + 'No model with audio input is available. In router mode, load one or pick a transcription model in Settings > Voice.', + voiceMessage: 'Send a voice message' +} as const; + +/** Where the chat form remembers that it showed the hint about a separate transcription model. */ +export const VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.voiceSeparateModelHintSeen`; + +/** Name prefix of a voice message's WAV file; the rest makes it unique per recording. */ +export const VOICE_CLIP_NAME_PREFIX = 'voice-'; + +/** + * The Transcription model setting's Automatic choice: the chat model when it hears, else a + * model with audio input that is already loaded, so dictating loads nothing. + */ +export const VOICE_TRANSCRIPTION_MODEL_AUTO = ''; diff --git a/ui/apps/chat/src/lib/enums/attachment.enums.ts b/ui/apps/chat/src/lib/enums/attachment.enums.ts index 5f096b974486..035fdac9cdd0 100644 --- a/ui/apps/chat/src/lib/enums/attachment.enums.ts +++ b/ui/apps/chat/src/lib/enums/attachment.enums.ts @@ -50,5 +50,6 @@ export enum AttachmentLabel { FILE = 'File', MCP_PROMPT = 'MCP Prompt', MCP_RESOURCE = 'MCP Resource', - PDF_FILE = 'PDF File' + PDF_FILE = 'PDF File', + VOICE_TRANSCRIPT = 'Voice message transcript' } diff --git a/ui/apps/chat/src/lib/enums/index.ts b/ui/apps/chat/src/lib/enums/index.ts index 29fd94373ee0..a4eed39d8190 100644 --- a/ui/apps/chat/src/lib/enums/index.ts +++ b/ui/apps/chat/src/lib/enums/index.ts @@ -121,3 +121,5 @@ export { } from './tools.enums'; export { SplashOrientation } from './splash.enums'; + +export { VoiceInputMode, VoiceMode } from './voice.enums'; diff --git a/ui/apps/chat/src/lib/enums/voice.enums.ts b/ui/apps/chat/src/lib/enums/voice.enums.ts new file mode 100644 index 000000000000..4fb2c92bd7b0 --- /dev/null +++ b/ui/apps/chat/src/lib/enums/voice.enums.ts @@ -0,0 +1,17 @@ +/** The Voice input setting: what the microphone does with a model that understands audio. */ +export enum VoiceInputMode { + /** Send the voice to a model with audio input, dictate otherwise. */ + AUTO = 'auto', + /** Always turn speech into text; only the text reaches the chat. */ + DICTATION = 'dictation' +} + +/** What a recording becomes for the conversation's model, as the chat form resolves it. */ +export enum VoiceMode { + /** Speech becomes text in the input; no audio is attached. */ + DICTATION = 'dictation', + /** No model can take audio, so the microphone has nothing to do. */ + UNAVAILABLE = 'unavailable', + /** The clip is attached and sent as audio, with a transcript to read. */ + VOICE_MESSAGE = 'voice-message' +} diff --git a/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts b/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts new file mode 100644 index 000000000000..9ef8eefa92cd --- /dev/null +++ b/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts @@ -0,0 +1,460 @@ +/** + * Voice input for the chat form. + * + * Resolves what the microphone does for the conversation's model (a voice message, + * dictation, or nothing, see resolveVoiceMode) and runs a recording: live levels and + * elapsed time, dictated text written into the message at the caret as it streams, and + * on stop either a voice message attachment, whose transcript follows, or the text. + * A voice message's live text shows as the empty message box's placeholder; focusing + * the box makes it editable text there until the recording stops, when it goes with + * the clip (edited, as its transcript). + */ + +import { + SETTINGS_KEYS, + VOICE_CLIP_NAME_PREFIX, + VOICE_MODE_MESSAGE, + VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY +} from '#lib/constants/index.ts'; +import { MimeTypeAudio, VoiceInputMode, VoiceMode } from '#lib/enums/index.ts'; +import { + modelsStore, + serverStore, + settingsStore, + voiceTranscriptsStore +} from '#lib/stores/index.ts'; +import type { ChatUploadedFile } from '#lib/types/index.ts'; +import { getAuthHeaders } from '#lib/utils/api-headers.ts'; +import { describeMicrophoneError } from '#lib/utils/audio-recording.ts'; +import { withBase } from '#lib/utils/base-path.ts'; +import { DictationSpan } from '#lib/utils/dictation-span.ts'; +import { uuid } from '#lib/utils/index.ts'; +import { resolveVoiceMode, voiceModeLabel } from '#lib/utils/voice-mode.ts'; +import { + resolveMicrophone, + VOICE_MICROPHONE_AUTO, + VoiceCaptureError, + type VoiceRecorderStatus, + type VoiceServerContext, + type VoiceSessionOptions, + type VoiceSessionResult +} from '#lib/voice/index.ts'; +import { onDestroy } from 'svelte'; +import { toast } from 'svelte-sonner'; + +/** What the form needs of a VoiceRecorder; tests and stories pass a fake. */ +export interface VoiceRecorderLike { + readonly levels: number[]; + readonly live: boolean; + readonly liveText: string; + readonly status: VoiceRecorderStatus; + start(options?: VoiceSessionOptions): Promise; + stop(): Promise; + cancel(): void; +} + +/** Where a recording is: off, waiting for the microphone, recording, or waiting for dictated text. */ +export type ChatFormVoiceState = 'idle' | 'starting' | 'recording' | 'transcribing'; + +interface UseChatFormVoiceOptions { + getRecorder: () => VoiceRecorderLike | undefined; + getValue: () => string; + /** Writes the message as dictation changes it; not a user edit. */ + setValue: (value: string) => void; + getCaretOffset: () => number; + /** Puts the caret after the dictated text once it is final. */ + setCaretOffset: (offset: number) => void; + addVoiceFile: (file: ChatUploadedFile) => void; + /** Why the last start failed, for the microphone button; null once one worked. */ + onMicError: (message: string | null) => void; +} + +const ELAPSED_TICK_MS = 200; + +function readHintSeen(): boolean { + try { + return localStorage.getItem(VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY) === 'true'; + } catch { + return false; + } +} + +function writeHintSeen(): void { + try { + localStorage.setItem(VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY, 'true'); + } catch { + // private mode: the hint shows again next time + } +} + +function readAsDataUrl(blob: Blob): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + + reader.onload = () => resolve(reader.result as string); + reader.onerror = () => reject(reader.error); + reader.readAsDataURL(blob); + }); +} + +/** The microphone's server: its routes, its auth, and whether a websocket reaches it. */ +function serverContext(): { context: VoiceServerContext; realtime: boolean } { + const headers = getAuthHeaders(); + // a browser websocket can not send the API key + const hasAuth = Object.keys(headers).some((key) => key.toLowerCase() === 'authorization'); + + return { + context: { + getHeaders: () => getAuthHeaders(), + resolveUrl: (path) => withBase(path.replace(/^\./, '')) + }, + realtime: !hasAuth + }; +} + +export function useChatFormVoice(options: UseChatFormVoiceOptions) { + const config = $derived(settingsStore.config); + const isRouter = $derived(serverStore.isRouterMode); + const chatModel = $derived(modelsStore.activeModelId); + const transcriptionSetting = $derived( + String(config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL] ?? '') + ); + const hasAudio = (model: string | null) => { + void modelsStore.props.cacheVersion; + + return model ? modelsStore.props.modelSupportsAudio(model) : false; + }; + // what Automatic transcribes with when the chat model can't hear: one already loaded + const loadedAudioModel = $derived.by(() => { + void modelsStore.props.cacheVersion; + + if (!isRouter) return null; + + return ( + modelsStore.models.find( + (option) => hasAudio(option.model) && modelsStore.isModelLoaded(option.model) + )?.model ?? null + ); + }); + const resolution = $derived( + resolveVoiceMode({ + chatModel, + chatModelHasAudio: hasAudio(chatModel), + isRouter, + loadedAudioModel: loadedAudioModel, + setting: String(config[SETTINGS_KEYS.VOICE_INPUT_MODE] ?? VoiceInputMode.AUTO), + transcriptionModel: transcriptionSetting, + transcriptionModelHasAudio: hasAudio(transcriptionSetting || null) + }) + ); + // a router with a model that hears keeps the button, so it can say where to pick it + const anyAudioModel = $derived.by(() => { + void modelsStore.props.cacheVersion; + + return isRouter && modelsStore.models.some((option) => hasAudio(option.model)); + }); + + let hintSeen = $state(readHintSeen()); + let state = $state('idle'); + let elapsedMs = $state(0); + // the mode of the recording under way, fixed when it starts + let recordingMode = $state(VoiceMode.UNAVAILABLE); + let ticker: ReturnType | undefined; + // state: a voice message's text leaves the placeholder once it is in the message + let span = $state(null); + let lastValue = ''; + // bumped by cancel, so a transcript still on its way is dropped + let generation = 0; + + const recorder = $derived(options.getRecorder()); + const unavailableReason = $derived( + resolution.mode === VoiceMode.UNAVAILABLE ? resolution.reason : null + ); + const hint = $derived( + resolution.mode === VoiceMode.DICTATION && resolution.separateModel && !hintSeen + ? VOICE_MODE_MESSAGE.separateModelHint + : null + ); + + function startTicker() { + const startedAt = Date.now(); + + elapsedMs = 0; + ticker = setInterval(() => (elapsedMs = Date.now() - startedAt), ELAPSED_TICK_MS); + } + + function stopTicker() { + clearInterval(ticker); + ticker = undefined; + } + + function writeDictation(text: string) { + if (!span) return; + + const next = span.apply(options.getValue(), text); + + lastValue = next; + options.setValue(next); + } + + /** + * A voice message's live text moves from the placeholder into the message, as text the + * user can edit, and what is said next follows it there. + */ + function adoptLiveText() { + if (recordingMode !== VoiceMode.VOICE_MESSAGE || span) return; + + if (state !== 'starting' && state !== 'recording') return; + + lastValue = options.getValue(); + span = new DictationSpan(lastValue, lastValue.length); + span.keepsEdits = true; + writeDictation(recorder?.liveText ?? ''); + options.setCaretOffset(span.end); + } + + async function start() { + if (!recorder || state !== 'idle' || resolution.mode === VoiceMode.UNAVAILABLE) return; + + const mode = resolution.mode; + const model = resolution.transcriptionModel; + const { context, realtime } = serverContext(); + const language = String(config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE] ?? '').trim(); + const microphone = String(config[SETTINGS_KEYS.VOICE_MICROPHONE] ?? VOICE_MICROPHONE_AUTO); + const transcribe = + mode === VoiceMode.DICTATION || config[SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES] !== false; + + if (resolution.mode === VoiceMode.DICTATION && resolution.separateModel && !hintSeen) { + hintSeen = true; + writeHintSeen(); + } + + recordingMode = mode; + state = 'starting'; + span = null; + + if (mode === VoiceMode.DICTATION) { + lastValue = options.getValue(); + span = new DictationSpan(lastValue, options.getCaretOffset()); + } + + const run = ++generation; + + try { + const deviceId = await resolveMicrophone(microphone); + + await recorder.start({ + capture: deviceId ? { deviceId } : undefined, + // a voice message's text reaches the message only once the user made it editable + onText: (text) => writeDictation(text), + realtime, + transcribe: transcribe + ? { + context, + language: language || undefined, + model: isRouter && model ? model : undefined + } + : undefined + }); + } catch (error) { + if (run !== generation) return; + + state = 'idle'; + span = null; + + const message = + error instanceof VoiceCaptureError ? error.message : describeMicrophoneError(error); + + options.onMicError(message); + toast.error(message); + + return; + } + + if (run !== generation) return; + + options.onMicError(null); + state = 'recording'; + startTicker(); + } + + async function stop() { + if (!recorder || state !== 'recording') return; + + const run = generation; + const mode = recordingMode; + + stopTicker(); + + let result: VoiceSessionResult; + + try { + result = await recorder.stop(); + } catch (error) { + console.error('Failed to stop the recording:', error); + state = 'idle'; + + return; + } + + if (run !== generation) return; + + if (mode === VoiceMode.VOICE_MESSAGE) { + const edited = takeLiveText(); + + state = 'idle'; + await attachVoiceMessage(result, edited); + + return; + } + + await finishDictation(result, run); + } + + /** + * Takes a voice message's text back out of the message once it stops: the clip carries + * it. Returns the text when the user edited it, which then is the clip's transcript. + */ + function takeLiveText(): string | null { + const taken = span; + + span = null; + + if (!taken || taken.isEmpty) return null; + + const previous = options.getValue(); + const text = taken.text(previous).trim(); + + lastValue = taken.remove(previous); + options.setValue(lastValue); + + return taken.userEdited ? text : null; + } + + async function attachVoiceMessage( + { clip, transcript }: VoiceSessionResult, + edited: string | null = null + ) { + const name = `${VOICE_CLIP_NAME_PREFIX}${Date.now()}-${uuid().slice(0, 8)}.wav`; + const file = new File([clip.blob], name, { type: MimeTypeAudio.WAV }); + + if (transcript) voiceTranscriptsStore.track(name, transcript); + + if (edited !== null) voiceTranscriptsStore.setTranscript(name, edited); + + options.addVoiceFile({ + file, + id: uuid(), + name, + preview: await readAsDataUrl(clip.blob), + size: file.size, + type: MimeTypeAudio.WAV, + voice: { durationMs: clip.durationMs, peaks: clip.peaks } + }); + } + + async function finishDictation({ liveText, transcript }: VoiceSessionResult, run: number) { + const dictated = span; + + if (!transcript || !dictated) { + state = 'idle'; + + return; + } + + // with live text the message already shows it; otherwise wait for the text + if (!liveText) state = 'transcribing'; + else state = 'idle'; + + try { + const text = await transcript; + + if (run !== generation) return; + + if (text.trim() || !liveText) writeDictation(text); + } catch (error) { + if (run !== generation) return; + + toast.error(error instanceof Error ? error.message : 'Transcription failed'); + } + + state = 'idle'; + span = null; + + if (!dictated.isEmpty) options.setCaretOffset(dictated.end); + } + + /** Drops the recording and any text it dictated. */ + function cancel() { + generation++; + stopTicker(); + recorder?.cancel(); + + if (span) { + const next = span.remove(options.getValue()); + + options.setValue(next); + span = null; + } + + state = 'idle'; + } + + /** Follows an edit the user typed while dictation writes into the message. */ + function handleUserInput() { + const next = options.getValue(); + + span?.edited(lastValue, next); + lastValue = next; + } + + onDestroy(() => { + if (state !== 'idle') cancel(); + }); + + return { + adoptLiveText, + cancel, + get elapsedMs() { + return elapsedMs; + }, + handleUserInput, + get hint() { + return hint; + }, + /** A recording (or its dictated text) is under way: the form shows the recording bar. */ + get isActive() { + return state !== 'idle'; + }, + /** What the microphone button says it will do. */ + get label() { + return voiceModeLabel(resolution); + }, + get levels() { + return recorder?.levels ?? []; + }, + /** Text streaming while a voice message is recorded, shown under the waveform. */ + get liveCaption() { + // once made editable, the text is in the message instead + return recordingMode === VoiceMode.VOICE_MESSAGE && !span ? (recorder?.liveText ?? '') : ''; + }, + get mode() { + return state === 'idle' ? resolution.mode : recordingMode; + }, + get resolution() { + return resolution; + }, + /** Whether the form shows a microphone button at all. */ + get showButton() { + return resolution.mode !== VoiceMode.UNAVAILABLE || anyAudioModel; + }, + start, + get state() { + return state; + }, + stop, + get unavailableReason() { + return unavailableReason; + } + }; +} diff --git a/ui/apps/chat/src/lib/services/chat.service.ts b/ui/apps/chat/src/lib/services/chat.service.ts index 848bd06efb39..38c7eba144b1 100644 --- a/ui/apps/chat/src/lib/services/chat.service.ts +++ b/ui/apps/chat/src/lib/services/chat.service.ts @@ -50,6 +50,12 @@ import { getAuthHeaders, getJsonHeaders } from '#lib/utils/api-headers.ts'; import { formatAttachmentText } from '#lib/utils/formatters.ts'; import { streamIdentity } from '#lib/utils/stream-identity.ts'; +/** How stored messages are shaped for the model a request goes to. */ +export interface ApiMessageConversionOptions { + /** False when the model has no audio input: voice messages go as their transcript. */ + audioInput?: boolean; +} + interface ResumableStreamState { bytesReceived: number; updatedAt: number; @@ -133,13 +139,24 @@ export class ChatService { } } + /** + * How messages are shaped for `model`: whether it takes audio, known only once its + * modalities are. Unknown modalities keep the audio, as before voice messages. + */ + static conversionOptionsFor(model?: string | null): ApiMessageConversionOptions { + const modalities = modelsStore.props.getModelModalities(model ?? ''); + + return modalities ? { audioInput: modalities.audio } : {}; + } + /** * Converts a database message with attachments to API chat message format. * Processes various attachment types (images, text files, PDFs) and formats them * as content parts suitable for the chat completion API. */ static async convertDbMessageToApiChatMessageData( - message: DatabaseMessage & { extra?: DatabaseMessageExtra[] } + message: DatabaseMessage & { extra?: DatabaseMessageExtra[] }, + options: ApiMessageConversionOptions = {} ): Promise { // Handle tool result messages (role: 'tool') if (message.role === MessageRole.TOOL && message.toolCallId) { @@ -231,6 +248,20 @@ export class ChatService { ); for (const audio of audioFiles) { + // a model that cannot hear reads what was said instead; one that can gets the audio alone + if (options.audioInput === false && audio.transcript) { + contentParts.push({ + text: formatAttachmentText( + AttachmentLabel.VOICE_TRANSCRIPT, + audio.name, + audio.transcript + ), + type: ContentPartType.TEXT + }); + + continue; + } + contentParts.push({ input_audio: { data: audio.base64Data, @@ -816,14 +847,18 @@ export class ChatService { * messages. Shared by sendMessage, preEncode and the agentic flow. */ static async normalizeMessagesForApi( - messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[] + messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[], + model?: string | null ): Promise { + const options = ChatService.conversionOptionsFor(model); + return ( await Promise.all( messages.map((msg) => { if ('id' in msg && 'convId' in msg && 'timestamp' in msg) { return ChatService.convertDbMessageToApiChatMessageData( - msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] } + msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }, + options ); } @@ -855,8 +890,10 @@ export class ChatService { signal?: AbortSignal, tools?: OpenAIToolDefinition[] ): Promise { - const normalizedMessages: ApiChatMessageData[] = - await ChatService.normalizeMessagesForApi(messages); + const normalizedMessages: ApiChatMessageData[] = await ChatService.normalizeMessagesForApi( + messages, + model + ); const requestBody: Record = { messages: normalizedMessages.map((msg: ApiChatMessageData) => { const mapped: Record = { @@ -1106,8 +1143,10 @@ export class ChatService { xtc_probability, xtc_threshold } = options; - const normalizedMessages: ApiChatMessageData[] = - await ChatService.normalizeMessagesForApi(messages); + const normalizedMessages: ApiChatMessageData[] = await ChatService.normalizeMessagesForApi( + messages, + options.model + ); // Filter out image attachments if the model doesn't support vision if (options.model && !modelsStore.props.modelSupportsVision(options.model)) { diff --git a/ui/apps/chat/src/lib/services/database.service.ts b/ui/apps/chat/src/lib/services/database.service.ts index de8ed4701e12..e77b97a964ca 100644 --- a/ui/apps/chat/src/lib/services/database.service.ts +++ b/ui/apps/chat/src/lib/services/database.service.ts @@ -572,6 +572,16 @@ export class DatabaseService { return result; } + /** + * Gets the messages written at or after `timestamp`, in any conversation. + * + * @param timestamp - Epoch ms + * @returns Messages, oldest first + */ + static async getMessagesSince(timestamp: number): Promise { + return await db[IDXDB_TABLES.messages].where('timestamp').aboveOrEqual(timestamp).toArray(); + } + /** * Imports multiple conversations and their messages. * Skips conversations that already exist. diff --git a/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts b/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts index b607a1373c56..2eb93ad3aa98 100644 --- a/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts +++ b/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts @@ -347,8 +347,10 @@ class AgenticStore { console.log(`[AgenticStore] Starting agentic flow with ${tools.length} tools`); - const normalizedMessages: ApiChatMessageData[] = - await ChatService.normalizeMessagesForApi(messages); + const normalizedMessages: ApiChatMessageData[] = await ChatService.normalizeMessagesForApi( + messages, + options.model + ); this.updateSession(conversationId, { currentTurn: 0, diff --git a/ui/apps/chat/src/lib/stores/chat/index.svelte.ts b/ui/apps/chat/src/lib/stores/chat/index.svelte.ts index ef8a1110035b..20c95be39867 100644 --- a/ui/apps/chat/src/lib/stores/chat/index.svelte.ts +++ b/ui/apps/chat/src/lib/stores/chat/index.svelte.ts @@ -1480,7 +1480,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { const allMessages = await conversationsStore.getConversationMessages(convId); const branchPath = [...filterByLeafNodeId(allMessages, leafId, false)]; const request = await agenticStore.requestFor( - await ChatService.normalizeMessagesForApi(sliceAtLastCompaction(branchPath)), + await ChatService.normalizeMessagesForApi(sliceAtLastCompaction(branchPath), model), this.toolPolicy() ); @@ -1646,7 +1646,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { // message shaping, cut before the preserved steps: the server // reuses the prefix it cached and only the instruction is new. const request = await agenticStore.requestFor( - await ChatService.normalizeMessagesForApi(source.slice(0, cut)), + await ChatService.normalizeMessagesForApi(source.slice(0, cut), effectiveModel), this.toolPolicy() ); const configValue = settingsStore.config; diff --git a/ui/apps/chat/src/lib/stores/index.ts b/ui/apps/chat/src/lib/stores/index.ts index e98a48d06c45..345b5630afed 100644 --- a/ui/apps/chat/src/lib/stores/index.ts +++ b/ui/apps/chat/src/lib/stores/index.ts @@ -61,6 +61,9 @@ export { permissionsStore } from './permissions.svelte'; // TOOLS export { toolsStore } from './tools.svelte'; +// VOICE (transcripts of voice messages on their way) +export { voiceTranscriptsStore } from './voice-transcripts.svelte'; + // ENVIRONMENT / META export { versionStore } from './version.svelte'; diff --git a/ui/apps/chat/src/lib/stores/voice-transcripts.svelte.ts b/ui/apps/chat/src/lib/stores/voice-transcripts.svelte.ts new file mode 100644 index 000000000000..2202cd0315bc --- /dev/null +++ b/ui/apps/chat/src/lib/stores/voice-transcripts.svelte.ts @@ -0,0 +1,152 @@ +/** + * voiceTranscriptsStore - Transcripts of voice messages that are on their way + * + * A voice message is attached as soon as the recording stops, while its transcript + * can take a few seconds more. This keeps each pending transcript by the clip's name + * (unique per recording), so the chip in the form and the sent message show its + * state, a send picks it up once it is there, and a message sent before it arrived + * gets it written into its stored attachment afterwards. Kept in memory only. + */ + +import { AttachmentType } from '#lib/enums/index.ts'; +import { DatabaseService } from '#lib/services/database.service.ts'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { conversationsStore } from '#lib/stores/conversations/index.svelte.ts'; +import type { + DatabaseMessage, + DatabaseMessageExtra, + DatabaseMessageExtraAudioFile +} from '#lib/types/index.ts'; +import { SvelteMap } from 'svelte/reactivity'; + +export type VoiceTranscriptState = + | { status: 'transcribing' } + | { status: 'done'; transcript: string } + | { status: 'error'; message: string }; + +/** How long a transcript that came back keeps looking for the message it belongs to. */ +const APPLY_RETRY_DELAYS_MS = [500, 1500, 4000]; + +function isPendingClip(extra: DatabaseMessageExtra, name: string): boolean { + return ( + extra.type === AttachmentType.AUDIO && + extra.name === name && + extra.voice === true && + extra.transcript === undefined + ); +} + +/** The extras with the clip's transcript filled in, or null when none of them is that clip. */ +export function withTranscript( + extras: DatabaseMessageExtra[] | undefined, + name: string, + transcript: string +): DatabaseMessageExtra[] | null { + if (!extras?.some((extra) => isPendingClip(extra, name))) return null; + + return extras.map((extra) => + isPendingClip(extra, name) + ? ({ ...(extra as DatabaseMessageExtraAudioFile), transcript } as DatabaseMessageExtra) + : extra + ); +} + +class VoiceTranscriptsStore { + // clips whose transcript the user wrote while recording: the model's does not replace it + private edited = new Set(); + private entries = new SvelteMap(); + private recordedAt = new Map(); + + /** Drops a clip removed from the form before it was sent. */ + forget(name: string): void { + this.entries.delete(name); + this.recordedAt.delete(name); + this.edited.delete(name); + } + + /** The state of a clip's transcript, undefined when none was asked for in this session. */ + get(name: string): VoiceTranscriptState | undefined { + return this.entries.get(name); + } + + /** Sets a clip's transcript to the text the user edited while recording; the model's is ignored. */ + setTranscript(name: string, transcript: string): void { + this.edited.add(name); + this.recordedAt.set(name, this.recordedAt.get(name) ?? Date.now()); + this.entries.set(name, { status: 'done', transcript }); + } + + /** Follows a clip's transcript, and writes it into the message it went out with. */ + track(name: string, transcript: Promise): void { + this.entries.set(name, { status: 'transcribing' }); + this.recordedAt.set(name, Date.now()); + + transcript.then( + (text) => { + if (!this.entries.has(name) || this.edited.has(name)) return; + + this.entries.set(name, { status: 'done', transcript: text }); + void this.applyToMessage(name, text); + }, + (error: unknown) => { + if (!this.entries.has(name) || this.edited.has(name)) return; + + console.warn('Voice message transcription failed:', error); + this.entries.set(name, { + message: error instanceof Error ? error.message : String(error), + status: 'error' + }); + } + ); + } + + /** The transcript once it is there. */ + transcriptOf(name: string): string | undefined { + const entry = this.entries.get(name); + + return entry?.status === 'done' ? entry.transcript : undefined; + } + + private async applyToMessage(name: string, transcript: string): Promise { + for (const delay of [0, ...APPLY_RETRY_DELAYS_MS]) { + if (delay) await new Promise((resolve) => setTimeout(resolve, delay)); + + if (await this.writeTranscript(name, transcript)) return; + } + // not sent (still in the form, or dropped): a send reads it from here + } + + private async writeTranscript(name: string, transcript: string): Promise { + const active = (conversationsStore.activeMessages as DatabaseMessage[]).find((message) => + message.extra?.some((extra) => isPendingClip(extra, name)) + ); + + if (active) { + // plain objects: IndexedDB cannot clone the state proxies + const extra = withTranscript($state.snapshot(active.extra), name, transcript)!; + + await DatabaseService.updateMessage(active.id, { extra }); + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(active.id), { + extra + }); + + return true; + } + + // the chat was left before the transcript came back: the message is newer than the clip + const since = this.recordedAt.get(name) ?? Date.now(); + const stored = (await DatabaseService.getMessagesSince(since)).find( + (message: DatabaseMessage) => message.extra?.some((extra) => isPendingClip(extra, name)) + ); + + if (!stored) return false; + + await DatabaseService.updateMessage(stored.id, { + extra: withTranscript(stored.extra, name, transcript)! + }); + + return true; + } +} + +export const voiceTranscriptsStore = new VoiceTranscriptsStore(); diff --git a/ui/apps/chat/src/lib/types/chat.d.ts b/ui/apps/chat/src/lib/types/chat.d.ts index eca0dbec87f8..a72a8dfff984 100644 --- a/ui/apps/chat/src/lib/types/chat.d.ts +++ b/ui/apps/chat/src/lib/types/chat.d.ts @@ -48,6 +48,14 @@ export interface ChatUploadedFile { }; isLoading?: boolean; loadError?: string; + /** A voice message recorded in the chat form. */ + voice?: ChatUploadedVoice; +} + +/** What a voice message recorded in the chat form carries besides its WAV file. */ +export interface ChatUploadedVoice { + durationMs: number; + peaks: number[]; } export interface ChatAttachmentDisplayItem { diff --git a/ui/apps/chat/src/lib/types/database.d.ts b/ui/apps/chat/src/lib/types/database.d.ts index 50f54dc0b556..e47d60fffb7c 100644 --- a/ui/apps/chat/src/lib/types/database.d.ts +++ b/ui/apps/chat/src/lib/types/database.d.ts @@ -38,6 +38,13 @@ export interface DatabaseMessageExtraAudioFile { size?: number; base64Data: string; mimeType: string; + /** Recorded in the app as a voice message, as opposed to an uploaded audio file. */ + voice?: true; + /** What was said, for reading; a model without audio input gets it in place of the audio. */ + transcript?: string; + /** Normalised 0..1 waveform of the clip; computed from the audio when left out. */ + peaks?: number[]; + durationMs?: number; } export interface DatabaseMessageExtraVideoFile { diff --git a/ui/apps/chat/src/lib/types/index.ts b/ui/apps/chat/src/lib/types/index.ts index 68cd6e92f1e7..bfed7569c17f 100644 --- a/ui/apps/chat/src/lib/types/index.ts +++ b/ui/apps/chat/src/lib/types/index.ts @@ -57,6 +57,7 @@ export type { export type { AttachmentMenuItem, ChatUploadedFile, + ChatUploadedVoice, ChatAttachmentDisplayItem, ChatMessageSiblingInfo, ChatMessageActions, diff --git a/ui/apps/chat/src/lib/utils/audio-recording.ts b/ui/apps/chat/src/lib/utils/audio-recording.ts index d733fc4ef4b9..92f32e3c43ac 100644 --- a/ui/apps/chat/src/lib/utils/audio-recording.ts +++ b/ui/apps/chat/src/lib/utils/audio-recording.ts @@ -1,158 +1,6 @@ import { MICROPHONE_UNAVAILABLE_MESSAGE } from '#lib/constants/index.ts'; import { MimeTypeAudio } from '#lib/enums/index.ts'; -/** - * AudioRecorder - Browser-based audio recording with MediaRecorder API - * - * This class provides a complete audio recording solution using the browser's MediaRecorder API. - * It handles microphone access, recording state management, and audio format optimization. - * - * **Features:** - * - Automatic microphone permission handling - * - Audio enhancement (echo cancellation, noise suppression, auto gain) - * - Multiple format support with fallback (WAV, WebM, MP4, AAC) - * - Real-time recording state tracking - * - Proper cleanup and resource management - */ -export class AudioRecorder { - private audioChunks: Blob[] = []; - private mediaRecorder: MediaRecorder | null = null; - private recordingState: boolean = false; - private stream: MediaStream | null = null; - - cancelRecording(): void { - const recorder = this.mediaRecorder; - const stream = this.stream; - - this.mediaRecorder = null; - this.audioChunks = []; - this.stream = null; - this.recordingState = false; - - if (recorder && recorder.state !== 'inactive') { - // Drop the original handlers so the pending stop event does not touch the instance - recorder.onstop = null; - recorder.onerror = null; - recorder.stop(); - } - - if (stream) { - for (const track of stream.getTracks()) { - track.stop(); - } - } - } - - isRecording(): boolean { - return this.recordingState; - } - - async startRecording(): Promise { - try { - this.stream = await navigator.mediaDevices.getUserMedia({ - audio: { - autoGainControl: true, - echoCancellation: true, - noiseSuppression: true - } - }); - - this.initializeRecorder(this.stream); - - this.audioChunks = []; - // Start recording with a small timeslice to ensure we get data - this.mediaRecorder!.start(100); - this.recordingState = true; - } catch (error) { - console.error('Failed to start recording:', error); - - // the browser's own error, which says why: see describeMicrophoneError - throw error; - } - } - - async stopRecording(): Promise { - return new Promise((resolve, reject) => { - const recorder = this.mediaRecorder; - const chunks = this.audioChunks; - const stream = this.stream; - - if (!recorder || recorder.state === 'inactive') { - reject(new Error('No active recording to stop')); - - return; - } - - // Detach instance state right away so a new startRecording can take over without race - this.mediaRecorder = null; - this.audioChunks = []; - this.stream = null; - this.recordingState = false; - - recorder.onstop = () => { - const audioBlob = new Blob(chunks, { - type: recorder.mimeType || MimeTypeAudio.WAV - }); - - if (stream) { - for (const track of stream.getTracks()) { - track.stop(); - } - } - - resolve(audioBlob); - }; - - recorder.onerror = (event) => { - console.error('Recording error:', event); - - if (stream) { - for (const track of stream.getTracks()) { - track.stop(); - } - } - - reject(new Error('Recording failed')); - }; - - recorder.stop(); - }); - } - - private initializeRecorder(stream: MediaStream): void { - const options: MediaRecorderOptions = {}; - - if (MediaRecorder.isTypeSupported(MimeTypeAudio.WAV)) { - options.mimeType = MimeTypeAudio.WAV; - } else if (MediaRecorder.isTypeSupported(MimeTypeAudio.WEBM_OPUS)) { - options.mimeType = MimeTypeAudio.WEBM_OPUS; - } else if (MediaRecorder.isTypeSupported(MimeTypeAudio.WEBM)) { - options.mimeType = MimeTypeAudio.WEBM; - } else if (MediaRecorder.isTypeSupported(MimeTypeAudio.MP4)) { - options.mimeType = MimeTypeAudio.MP4; - } else { - console.warn('No preferred audio format supported, using default'); - } - - this.mediaRecorder = new MediaRecorder(stream, options); - - this.mediaRecorder.ondataavailable = (event) => { - if (event.data.size > 0) { - this.audioChunks.push(event.data); - } - }; - - this.mediaRecorder.onstop = () => { - this.recordingState = false; - }; - - this.mediaRecorder.onerror = (event) => { - console.error('MediaRecorder error:', event); - this.recordingState = false; - }; - } -} - export async function convertToWav(audioBlob: Blob): Promise { try { if (audioBlob.type.includes('wav')) { diff --git a/ui/apps/chat/src/lib/utils/browser-only.ts b/ui/apps/chat/src/lib/utils/browser-only.ts index 64eedc140dfb..066cfb653452 100644 --- a/ui/apps/chat/src/lib/utils/browser-only.ts +++ b/ui/apps/chat/src/lib/utils/browser-only.ts @@ -8,7 +8,6 @@ // Audio utilities (MediaRecorder API) export { - AudioRecorder, convertToWav, createAudioFile, describeMicrophoneError, diff --git a/ui/apps/chat/src/lib/utils/clipboard.ts b/ui/apps/chat/src/lib/utils/clipboard.ts index a5c75e873317..c2b05985e0e0 100644 --- a/ui/apps/chat/src/lib/utils/clipboard.ts +++ b/ui/apps/chat/src/lib/utils/clipboard.ts @@ -110,6 +110,16 @@ export function formatMessageForClipboard( extras?: DatabaseMessageExtra[], asPlainText: boolean = false ): string { + // what was said in a voice message reads as the message's text + const transcripts = + extras?.flatMap((extra) => + extra.type === AttachmentType.AUDIO && extra.transcript ? [extra.transcript] : [] + ) ?? []; + + if (transcripts.length > 0) { + content = [...transcripts, content].filter((part) => part.trim()).join('\n\n'); + } + // Filter text-like attachments (TEXT, LEGACY_CONTEXT, MCP_PROMPT, and MCP_RESOURCE types) const textAttachments = extras?.filter( diff --git a/ui/apps/chat/src/lib/utils/convert-files-to-extra.ts b/ui/apps/chat/src/lib/utils/convert-files-to-extra.ts index 43d29f43c119..4e50b089c5bc 100644 --- a/ui/apps/chat/src/lib/utils/convert-files-to-extra.ts +++ b/ui/apps/chat/src/lib/utils/convert-files-to-extra.ts @@ -6,6 +6,7 @@ import { SETTINGS_KEYS } from '#lib/constants/index.ts'; import { AttachmentType, FileTypeCategory, SpecialFileType } from '#lib/enums/index.ts'; import { modelsStore } from '#lib/stores/models/index.svelte.ts'; import { settingsStore } from '#lib/stores/settings/index.svelte.ts'; +import { voiceTranscriptsStore } from '#lib/stores/voice-transcripts.svelte.ts'; import type { ChatUploadedFile, DatabaseMessageExtra, @@ -83,13 +84,21 @@ export async function parseFilesToMessageExtras( // Process audio files (MP3 and WAV) try { const base64Data = await readFileAsBase64(file.file); + // a voice message keeps its waveform, and its transcript once it is there + const transcript = file.voice ? voiceTranscriptsStore.transcriptOf(file.name) : undefined; extras.push({ base64Data: base64Data, mimeType: file.type, name: file.name, size: file.size, - type: AttachmentType.AUDIO + type: AttachmentType.AUDIO, + ...(file.voice && { + durationMs: file.voice.durationMs, + peaks: file.voice.peaks, + voice: true as const, + ...(transcript !== undefined && { transcript }) + }) }); } catch (error) { console.error(`Failed to process audio file ${file.name}:`, error); diff --git a/ui/apps/chat/src/lib/utils/dictation-span.ts b/ui/apps/chat/src/lib/utils/dictation-span.ts new file mode 100644 index 000000000000..2305d4cb2010 --- /dev/null +++ b/ui/apps/chat/src/lib/utils/dictation-span.ts @@ -0,0 +1,110 @@ +/** + * The stretch of the message that dictation writes: it starts at the caret, and each + * new version of the dictated text replaces only that stretch, so what the user typed + * before and after it stays. An edit before the stretch moves it; an edit inside it + * hands that text over to the user, and dictation goes on after it. + */ +export class DictationSpan { + /** + * Edits inside the stretch stay in it, as when its text is a voice message's transcript: + * what it held is the user's from then on, and new text goes after it. + */ + keepsEdits = false; + /** Whether the user edited inside the stretch, see keepsEdits. */ + userEdited = false; + // characters of the dictated text already handed over to the user + private handedOver = 0; + // characters at the start of the stretch that the user edited, kept as they are + private kept = 0; + private length = 0; + // characters of the dictated text the stretch shows + private shown = 0; + private start: number; + + /** Where the dictated text ends, e.g. to put the caret after it. */ + get end(): number { + return this.start + this.length; + } + + /** Whether the stretch shows any text. */ + get isEmpty(): boolean { + return this.length === 0; + } + + /** + * Writes the dictated text so far into `value` in place of the stretch, with a space + * on either side where it would otherwise run into the neighbouring text. + */ + apply(value: string, text: string): string { + const body = text.slice(this.handedOver).trim(); + const before = value.slice(0, this.start + this.kept); + const after = value.slice(this.start + this.length); + // no space before text that goes on with punctuation, e.g. ", and then" + const lead = body && before && !/\s$/.test(before) && !/^[,.;:!?)\]}]/.test(body) ? ' ' : ''; + const trail = body && after && !/^\s/.test(after) ? ' ' : ''; + const inserted = lead + body + trail; + + this.length = this.kept + inserted.length; + this.shown = text.length; + + return before + inserted + after; + } + + constructor(value: string, caret: number) { + this.start = Math.max(0, Math.min(caret, value.length)); + } + + /** Follows an edit the user made, `previous` to `next`. */ + edited(previous: string, next: string): void { + const { from, insert, to } = changeBetween(previous, next); + const end = this.end; + const delta = insert.length - (to - from); + + if (to <= this.start) { + this.start += delta; + } else if (from >= end) { + return; + } else if (this.keepsEdits && from >= this.start && to <= end) { + this.length += delta; + this.kept = this.length; + this.handedOver = this.shown; + this.userEdited = true; + } else { + // typed into the dictated text: it is theirs now, what comes next goes after it + this.handedOver = this.shown; + this.start = to <= end ? end + delta : from + insert.length; + this.length = 0; + } + } + + /** Takes the stretch out of `value`, as a cancelled dictation leaves the message. */ + remove(value: string): string { + const next = value.slice(0, this.start) + value.slice(this.start + this.length); + + this.length = 0; + + return next; + } + + /** The text the stretch holds in `value`, with the user's edits inside it. */ + text(value: string): string { + return value.slice(this.start, this.start + this.length); + } +} + +/** The single change that turns `current` into `next`: their differing middle. */ +function changeBetween(current: string, next: string) { + let from = 0; + + while (from < current.length && from < next.length && current[from] === next[from]) from++; + + let currentTo = current.length; + let nextTo = next.length; + + while (currentTo > from && nextTo > from && current[currentTo - 1] === next[nextTo - 1]) { + currentTo--; + nextTo--; + } + + return { from, insert: next.slice(from, nextTo), to: currentTo }; +} diff --git a/ui/apps/chat/src/lib/utils/index.ts b/ui/apps/chat/src/lib/utils/index.ts index a0544551393e..0737b08f58d4 100644 --- a/ui/apps/chat/src/lib/utils/index.ts +++ b/ui/apps/chat/src/lib/utils/index.ts @@ -384,3 +384,11 @@ export { groupModelFamilies, modelFamilyKey, type ModelFamilyGroup } from './mod // Svelte actions export { nearViewport } from './near-viewport'; + +// Voice: what a recording becomes for the conversation's model +export { + resolveVoiceMode, + voiceModeLabel, + type VoiceModeInput, + type VoiceModeResolution +} from './voice-mode'; diff --git a/ui/apps/chat/src/lib/utils/voice-attachment.ts b/ui/apps/chat/src/lib/utils/voice-attachment.ts new file mode 100644 index 000000000000..79380ee7f7fb --- /dev/null +++ b/ui/apps/chat/src/lib/utils/voice-attachment.ts @@ -0,0 +1,108 @@ +import { AttachmentType } from '#lib/enums/index.ts'; +import type { ChatAttachmentDisplayItem, DatabaseMessageExtraAudioFile } from '#lib/types/index.ts'; +import { computePeaksFromAudio } from '#lib/voice/index.ts'; + +// peaks of clips stored without them, kept for the session only +const peaksCache = new Map>(); + +/** What a list item needs to draw an audio attachment as a clip. */ +export interface VoiceAttachmentSource { + /** Base64 or a URL of the audio, or the file itself. */ + audio: string | Blob | null; + durationMs?: number; + mimeType: string; + name: string; + peaks?: number[]; + /** Recorded in the app, as opposed to an uploaded file. */ + voice: boolean; + transcript?: string; +} + +/** The clip behind an audio item of the form or of a message. */ +export function voiceAttachmentSource(item: ChatAttachmentDisplayItem): VoiceAttachmentSource { + const uploaded = item.uploadedFile; + + if (uploaded) { + return { + audio: uploaded.preview ?? (uploaded.file.size > 0 ? uploaded.file : null), + durationMs: uploaded.voice?.durationMs, + mimeType: uploaded.type, + name: uploaded.name, + peaks: uploaded.voice?.peaks, + voice: !!uploaded.voice + }; + } + + const attachment = + item.attachment?.type === AttachmentType.AUDIO + ? (item.attachment as DatabaseMessageExtraAudioFile) + : null; + + return { + audio: attachment?.base64Data || null, + durationMs: attachment?.durationMs, + mimeType: attachment?.mimeType ?? '', + name: item.name, + peaks: attachment?.peaks, + transcript: attachment?.transcript, + voice: attachment?.voice === true + }; +} + +/** A URL an