Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions openless-all/app/e2e/live-input-settings.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import { expect, test } from '@playwright/test';

for (const [os, ua, platform] of [
['macOS', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0)', 'MacIntel'],
['Windows', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', 'Win32'],
] as const) {
test(`live transcript input has a dedicated advanced settings page on ${os}`, async ({
page,
}) => {
await page.addInitScript(
({ ua, platform }) => {
Object.defineProperty(navigator, 'userAgent', { value: ua });
Object.defineProperty(navigator, 'platform', { value: platform });
Object.defineProperty(navigator, 'userAgentData', { value: undefined });
},
{ ua, platform },
);
await page.goto('/');
await page.getByRole('button', { name: '设置', exact: true }).click();
const dialog = page.getByRole('dialog', { name: '设置', exact: true });
await dialog.getByRole('button', { name: '高级功能', exact: true }).click();
await dialog.locator('[data-ol-advanced-entry="liveInput"]').click();
const panel = dialog.locator('[data-ol-advanced-page="liveInput"]');
await expect(panel).toBeVisible();
await expect(panel.getByText('实时原文显示在输入框', { exact: true })).toBeVisible();
});
}
116 changes: 109 additions & 7 deletions openless-all/app/src-tauri/src/core_adapters.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5004,10 +5004,10 @@ impl TauriTextInsertionSession {
let live_final = final_text.clone();
let live_result = tauri::async_runtime::spawn_blocking(move || {
let mut guard = live.lock();
guard.take().map(|mut live| match live_final {
Some(text) => live.replace(&text),
None => live.cancel(),
})
match live_final {
Some(text) => Some(replace_live_composition(&mut guard, &text)),

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Gate final live-target capture on composition input being enabled

On macOS with live transcript input disabled (the default), live_composition is intentionally None, but this now captures a new AX target on every finish and always produces Some(live_result). Consequently, the later branch always returns before the ordinary insert_final path: unsupported AX editors receive only a clipboard fallback instead of normal insertion, while supported editors receive text at the current focus without restoring the captured insertion target. Streaming finalization also passes an empty string, which can now delete selected text rather than simply close the session. Only retry live capture when context.insertion.composition_input is enabled; otherwise preserve the existing non-composition finalization path.

Useful? React with 👍 / 👎.

None => guard.take().map(|mut live| live.cancel()),
}
})
.await
.map_err(|e| BackendError::new(BackendErrorCode::Platform, e.to_string()))?;
Expand Down Expand Up @@ -5067,6 +5067,41 @@ impl TauriTextInsertionSession {
}
}

#[cfg(target_os = "macos")]
fn replace_live_composition(
slot: &mut Option<crate::host_document::LiveComposition>,
text: &str,
) -> Result<(), String> {
replace_unwritten_target(
slot,
text,
crate::host_document::LiveComposition::capture,
|live, value| live.replace(value),
|live| live.has_written(),
)
}

#[cfg(any(test, target_os = "macos"))]
fn replace_unwritten_target<T>(
slot: &mut Option<T>,
text: &str,
capture: impl FnOnce() -> Result<T, String>,
mut replace: impl FnMut(&mut T, &str) -> Result<(), String>,
written: impl Fn(&T) -> bool,
) -> Result<(), String> {
if let Some(live) = slot.as_mut() {
match replace(live, text) {
Ok(()) => return Ok(()),
Err(error) if written(live) => return Err(error),
Err(_) => *slot = None,
}
}
let mut live = capture()?;
let result = replace(&mut live, text);
*slot = Some(live);
result
}

impl TextInsertionSession for TauriTextInsertionSession {
fn supports_streaming(&self) -> bool {
#[cfg(target_os = "macos")]
Expand Down Expand Up @@ -5104,9 +5139,10 @@ impl TextInsertionSession for TauriTextInsertionSession {
}
tauri::async_runtime::spawn_blocking(move || {
let mut live = session.live_composition.lock();
live.as_mut()
.ok_or_else(|| "live input is unavailable in this editor".to_string())?
.replace(&text)
match replace_live_composition(&mut live, &text) {
Err(_) if live.as_ref().is_none_or(|target| !target.has_written()) => Ok(()),
result => result,
}
})
.await
.map_err(|e| BackendError::new(BackendErrorCode::Platform, e.to_string()))?
Expand Down Expand Up @@ -5595,6 +5631,72 @@ fn map_tauri_error(error: impl std::fmt::Display) -> BackendError {

#[cfg(test)]
mod tests {
#[test]
fn live_input_retargets_only_before_any_write() {
#[derive(Debug)]
struct Target {
written: bool,
current: bool,
}
let mut slot = Some(Target {
written: false,
current: false,
});
let capture = || Ok(Target {
written: false,
current: true,
});
let replace = |target: &mut Target, _: &str| {
if !target.current {
return Err("target changed".into());
}
target.written = true;
Ok(())
};
let written = |target: &Target| target.written;
super::replace_unwritten_target(&mut slot, "text", capture, replace, written).unwrap();
assert!(slot.as_ref().unwrap().written);
slot.as_mut().unwrap().current = false;
let result = super::replace_unwritten_target(
&mut slot,
"more",
|| panic!("a written target must never be replaced"),
replace,
written,
);
assert_eq!(result.unwrap_err(), "target changed");
}

#[test]
fn live_input_keeps_target_when_write_outcome_is_uncertain() {
let mut slot = None;
let result = super::replace_unwritten_target(
&mut slot,
"text",
|| Ok(false),
|written, _| {
*written = true;
Err("verification failed".into())
},
|written| *written,
);
assert_eq!(result.unwrap_err(), "verification failed");
assert_eq!(slot, Some(true));
}

#[cfg(target_os = "macos")]
#[test]
#[ignore = "requires the foreground disposable native-live-input-fixture app"]
fn live_input_captures_late_focused_native_editor() {
let (_, bundle) = crate::selection::current_front_app_parts();
assert_eq!(bundle.as_deref(), Some("org.openless.live-input-fixture"));
let mut slot = None;
super::replace_live_composition(&mut slot, "实时原文").unwrap();
let mut live = slot.expect("captured live editor");
assert!(live.has_written());
live.cancel().unwrap();
}

use super::*;

#[cfg(target_os = "windows")]
Expand Down
18 changes: 17 additions & 1 deletion openless-all/app/src-tauri/src/host_document/macos.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1669,8 +1669,23 @@ pub(crate) struct LiveComposition {
}

impl LiveComposition {
pub(crate) fn has_written(&self) -> bool {
self.wrote
}

pub(crate) fn capture() -> Result<Self, String> {
match crate::selection::current_front_app_pid() {
Some(pid) if pid != std::process::id() as i32 => {}
_ => return Err("liveInputTargetUnavailable".into()),
}
let (_, _, target) = capture_voice_edit_target()?;
let mut owner_pid = 0;
if unsafe { AXUIElementGetPid(target.element.as_ref(), &mut owner_pid) } != AX_ERROR_SUCCESS
|| owner_pid <= 0
|| owner_pid == std::process::id() as i32
{
return Err("liveInputTargetUnavailable".into());
}
let start =
utf16_offset_to_char_offset(&target.text, target.selection_utf16.location as usize);
let end = utf16_offset_to_char_offset(
Expand Down Expand Up @@ -1735,13 +1750,14 @@ impl LiveComposition {
K_CF_STRING_ENCODING_UTF8,
);
let attr = cfstring_from_static(b"AXSelectedText\0").ok_or("liveInputUnavailable")?;
// A failed AX call can have an unknown outcome; never retarget after submission.
self.wrote = true;
let result = AXUIElementSetAttributeValue(self.target.element.as_ref(), attr, value);
CFRelease(attr);
CFRelease(value);
if result != AX_ERROR_SUCCESS {
return Err("liveInputUnsupportedEditor".into());
}
self.wrote = true;
let expected = format!("{start}{text}{suffix}");
if voice_edit_text(self.target.element.as_ref()).as_deref() != Some(&expected) {
return Err("liveInputEditorTransformedText".into());
Expand Down
6 changes: 4 additions & 2 deletions openless-all/app/src/i18n/de.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1208,6 +1208,7 @@ export const de: typeof zhCN = {
phoneDesc: 'Ein Telefon oder ein anderes Gerät für die Ferneingabe verbinden.',
},
advancedPages: {
liveInputTitle: 'Texteingabe',
experiments: 'Experimentelle Funktionen und Erweiterungen konfigurieren.',
voiceEnhancementTitle: 'Sprachverbesserung und Wiedergabeunterdrückung',
cursorContextTitle: 'Cursorkontext und kurze Eingaben',
Expand Down Expand Up @@ -2206,10 +2207,10 @@ export const de: typeof zhCN = {
'SendInput gibt Unicode direkt ein und umgeht TSF / IME. Ein Wechsel der Eingabemethode ist nicht erforderlich.',
streamingInsertHintLinux:
'Verwendet das fcitx5-Plugin zur Textübermittlung. Laufendes Einfügen simuliert Tasten über enigo + XTest.',
macosCompositionInputLabel: 'Live-Transkript im Eingabefeld (macOS)',
macosCompositionInputLabel: 'Live-Transkript im Eingabefeld',
macosCompositionInputDesc:
'Zeigt die Erkennung im ursprünglichen Eingabefeld und ersetzt sie anschließend durch den überarbeiteten oder übersetzten Text. Erfordert Bedienungshilfen und einen kompatiblen Editor. Cursorbewegungen oder manuelle Änderungen stoppen die Ersetzung.',
windowsCompositionInputLabel: 'Live-Transkript im Eingabefeld anzeigen (Windows)',
windowsCompositionInputLabel: 'Live-Transkript im Eingabefeld anzeigen',
windowsCompositionInputDesc:
'Während du sprichst, erscheint der erkannte Text unterstrichen im Eingabefeld und wird beim Beenden durch das Endergebnis ersetzt. Erfordert die OpenLess-TSF-Eingabekomponente und eine App mit Unterstützung für TSF.',
streamingInsertSaveClipboardLabel: 'In Zwischenablage kopieren',
Expand Down Expand Up @@ -2428,6 +2429,7 @@ export const de: typeof zhCN = {
backToAdvanced: 'Zurück zu erweiterten Funktionen',
back: 'Zurück',
advancedPages: {
liveInput: 'Live-Transkript im Eingabefeld konfigurieren.',
experiments: 'Experimentelle Funktionen und Erweiterungen konfigurieren.',
customSyncServer:
'Nur für selbst gehostete Dienste; die offizielle Synchronisierung benötigt keine Konfiguration.',
Expand Down
6 changes: 4 additions & 2 deletions openless-all/app/src/i18n/en.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1241,6 +1241,7 @@ export const en: typeof zhCN = {
phoneDesc: 'Connect a phone or another device for remote input.',
},
advancedPages: {
liveInputTitle: 'Text insertion',
experiments: 'Configure experimental features and extensions.',
voiceEnhancementTitle: 'Voice enhancement and playback suppression',
cursorContextTitle: 'Cursor context and short-input behavior',
Expand Down Expand Up @@ -2145,10 +2146,10 @@ export const en: typeof zhCN = {
'SendInput Unicode types directly, bypassing TSF / IME — no input-method switching needed.',
streamingInsertHintLinux:
'Uses fcitx5 plugin for text submission; streaming insertion uses enigo + XTest for keystroke synthesis.',
macosCompositionInputLabel: 'Live transcript in the input field (macOS)',
macosCompositionInputLabel: 'Live transcript in the input field',
macosCompositionInputDesc:
'Shows recognition in the captured editor, then replaces it with the final polished or translated text. Requires Accessibility and editor support. Moving the caret or editing stops replacement.',
windowsCompositionInputLabel: 'Show live transcript in the input box (Windows)',
windowsCompositionInputLabel: 'Show live transcript in the input box',
windowsCompositionInputDesc:
'While you speak, the recognized text appears underlined in the input box and is replaced by the final result when you finish. Requires the OpenLess TSF input component and a target app that supports the Text Services Framework (TSF).',
streamingInsertSaveClipboardLabel: 'Copy to clipboard',
Expand Down Expand Up @@ -2362,6 +2363,7 @@ export const en: typeof zhCN = {
backToAdvanced: 'Back to advanced features',
back: 'Back',
advancedPages: {
liveInput: 'Configure the live transcript in the input field.',
experiments: 'Configure experimental features and extensions.',
customSyncServer: 'For self-hosted services only; official sync needs no configuration.',
localModels: experimentalAsr.description,
Expand Down
6 changes: 4 additions & 2 deletions openless-all/app/src/i18n/es.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1201,6 +1201,7 @@ export const es: typeof zhCN = {
phoneDesc: 'Conecta un teléfono u otro dispositivo para la entrada remota.',
},
advancedPages: {
liveInputTitle: 'Inserción de texto',
experiments: 'Configura funciones experimentales y extensiones.',
voiceEnhancementTitle: 'Mejora de voz y supresión de reproducción',
cursorContextTitle: 'Contexto del cursor y frases cortas',
Expand Down Expand Up @@ -2185,10 +2186,10 @@ export const es: typeof zhCN = {
'SendInput Unicode escribe directamente, sin pasar por TSF / IME y sin cambiar de método de entrada.',
streamingInsertHintLinux:
'Usa el complemento fcitx5 para enviar texto; la inserción progresiva simula teclas mediante enigo + XTest.',
macosCompositionInputLabel: 'Transcripción en directo en el campo de entrada (macOS)',
macosCompositionInputLabel: 'Transcripción en directo en el campo de entrada',
macosCompositionInputDesc:
'Muestra el reconocimiento en el campo original y después lo sustituye por el texto corregido o traducido. Requiere permiso de Accesibilidad y un editor compatible. Mover el cursor o editar el texto detiene la sustitución.',
windowsCompositionInputLabel: 'Mostrar la transcripción en vivo en el campo (Windows)',
windowsCompositionInputLabel: 'Mostrar la transcripción en vivo en el campo',
windowsCompositionInputDesc:
'Mientras hablas, el texto reconocido aparece subrayado en el campo y se reemplaza por el resultado final al terminar. Requiere el componente de entrada TSF de OpenLess y una aplicación compatible con TSF.',
streamingInsertSaveClipboardLabel: 'Copiar al portapapeles',
Expand Down Expand Up @@ -2407,6 +2408,7 @@ export const es: typeof zhCN = {
backToAdvanced: 'Volver a funciones avanzadas',
back: 'Atrás',
advancedPages: {
liveInput: 'Configurar la transcripción en directo en el campo.',
experiments: 'Configura funciones experimentales y extensiones.',
customSyncServer:
'Solo para servicios propios; la sincronización oficial no requiere configuración.',
Expand Down
6 changes: 4 additions & 2 deletions openless-all/app/src/i18n/fr.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1218,6 +1218,7 @@ export const fr: typeof zhCN = {
phoneDesc: 'Connecter un téléphone ou un autre appareil pour la saisie à distance.',
},
advancedPages: {
liveInputTitle: 'Insertion de texte',
experiments: 'Configurer les fonctions expérimentales et les extensions.',
voiceEnhancementTitle: 'Amélioration vocale et suppression de lecture',
cursorContextTitle: 'Contexte du curseur et phrases courtes',
Expand Down Expand Up @@ -2212,10 +2213,10 @@ export const fr: typeof zhCN = {
'SendInput Unicode écrit directement, sans passer par TSF / IME et sans changer de méthode de saisie.',
streamingInsertHintLinux:
'Utilise le complément fcitx5 pour envoyer le texte ; l’insertion progressive simule les touches avec enigo + XTest.',
macosCompositionInputLabel: 'Transcription en direct dans le champ de saisie (macOS)',
macosCompositionInputLabel: 'Transcription en direct dans le champ de saisie',
macosCompositionInputDesc:
'Affiche la transcription dans le champ initial, puis la remplace par le résultat corrigé ou traduit. Nécessite les autorisations d’accessibilité et un éditeur compatible. Déplacer le curseur ou modifier le texte arrête le remplacement.',
windowsCompositionInputLabel: 'Afficher la transcription en direct dans le champ (Windows)',
windowsCompositionInputLabel: 'Afficher la transcription en direct dans le champ',
windowsCompositionInputDesc:
"Pendant la dictée, le texte reconnu apparaît souligné dans le champ puis est remplacé par le résultat final. Nécessite le composant de saisie TSF d'OpenLess et une application compatible avec TSF.",
streamingInsertSaveClipboardLabel: 'Copier dans le presse-papiers',
Expand Down Expand Up @@ -2437,6 +2438,7 @@ export const fr: typeof zhCN = {
backToAdvanced: 'Retour aux fonctions avancées',
back: 'Retour',
advancedPages: {
liveInput: 'Configurer la transcription en direct dans le champ.',
experiments: 'Configurer les fonctions expérimentales et les extensions.',
customSyncServer:
'Pour les services auto-hébergés uniquement ; la synchronisation officielle ne nécessite aucune configuration.',
Expand Down
6 changes: 4 additions & 2 deletions openless-all/app/src/i18n/ja.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1172,6 +1172,7 @@ export const ja: typeof zhCN = {
phoneDesc: 'スマホや別の端末を接続してリモート入力します。',
},
advancedPages: {
liveInputTitle: '文字入力',
experiments: '実験的な機能と拡張オプションを設定します。',
voiceEnhancementTitle: '音声強調と再生抑制',
cursorContextTitle: 'カーソル文脈と短文の動作',
Expand Down Expand Up @@ -2115,10 +2116,10 @@ export const ja: typeof zhCN = {
'SendInput Unicode で TSF / IME を迂回。入力ソースの切替は不要です。',
streamingInsertHintLinux:
'fcitx5 プラグインで文字を送信。ストリーミング入力は enigo + XTest でキー合成。',
macosCompositionInputLabel: '入力欄に認識中の原文を表示(macOS)',
macosCompositionInputLabel: '入力欄に認識中の原文を表示',
macosCompositionInputDesc:
'録音中の認識結果を元の入力欄に表示し、終了後に整形または翻訳した文章へ置き換えます。アクセシビリティ権限と対応エディタが必要です。カーソル移動や手動編集で置き換えを停止します。',
windowsCompositionInputLabel: 'リアルタイム原文を入力欄に表示(Windows)',
windowsCompositionInputLabel: 'リアルタイム原文を入力欄に表示',
windowsCompositionInputDesc:
'録音中、認識された文字を下線付きの変換中テキストとして入力欄に表示し、終了時に最終結果に置き換えます。OpenLess の TSF 入力コンポーネントと、テキストサービスフレームワーク(TSF)に対応したアプリが必要です。',
streamingInsertSaveClipboardLabel: 'クリップボードに保存',
Expand Down Expand Up @@ -2330,6 +2331,7 @@ export const ja: typeof zhCN = {
backToAdvanced: '高度な機能に戻る',
back: '戻る',
advancedPages: {
liveInput: '入力欄に表示するリアルタイム原文を設定します。',
experiments: '実験的な機能と拡張オプションを設定します。',
customSyncServer: 'セルフホスト専用です。公式の同期には設定は不要です。',
localModels: experimentalAsr.description,
Expand Down
Loading
Loading