From 66fcd151a308e48b79cb0d550f14a3c676d43ab9 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Tue, 20 Oct 2026 17:32:20 +0800 Subject: [PATCH] fix(editor): record voiceovers against the uncut timeline (#1195) --- .../NativeCompositorOverlay.test.tsx | 56 +++- .../ai-edition/NativeCompositorOverlay.tsx | 59 ++-- src/components/ai-edition/NewEditorShell.tsx | 37 ++- .../NewEditorShell.voiceover.test.tsx | 251 ++++++++++++++++++ src/components/ai-edition/Preview.tsx | 3 + src/components/ai-edition/PreviewCanvas.tsx | 11 +- .../VirtualPreview.playback.test.tsx | 42 ++- src/components/ai-edition/VirtualPreview.tsx | 31 ++- .../v4/AddAudioLayerDialog.test.tsx | 37 +++ .../ai-edition/v4/AddAudioLayerDialog.tsx | 13 +- src/native/sceneDescription.test.ts | 17 ++ 11 files changed, 509 insertions(+), 48 deletions(-) create mode 100644 src/components/ai-edition/NewEditorShell.voiceover.test.tsx diff --git a/src/components/ai-edition/NativeCompositorOverlay.test.tsx b/src/components/ai-edition/NativeCompositorOverlay.test.tsx index ff3522665..0f1d5c16e 100644 --- a/src/components/ai-edition/NativeCompositorOverlay.test.tsx +++ b/src/components/ai-edition/NativeCompositorOverlay.test.tsx @@ -18,6 +18,7 @@ import { publishNativePosition } from "@/native/nativeSync"; const native = vi.hoisted(() => ({ setActiveClip: vi.fn(async () => ({ ok: true })), setNativePlaying: vi.fn(), + setNativeScene: vi.fn(), })); vi.mock("@/native", () => ({ @@ -25,7 +26,7 @@ vi.mock("@/native", () => ({ setActiveClip: native.setActiveClip, setCurrentNativeViewId: vi.fn(), setNativePlaying: native.setNativePlaying, - setNativeScene: vi.fn(), + setNativeScene: native.setNativeScene, subscribeNativeCompositor: () => () => undefined, useIsCpuCompositor: () => false, useNativeCompositorView: () => ({ viewId: 7, error: null }), @@ -94,6 +95,29 @@ function makeDocument(): AxcutDocument { }; } +function makeTrimmedDocument(): AxcutDocument { + const document = makeDocument(); + return { + ...document, + timeline: { + ...document.timeline, + clips: [{ ...FIRST, sourceEndSec: 10, timelineEndSec: 10 }], + trimRanges: [ + { + id: "trim_a", + assetId: "asset_take", + clipId: "clip_a", + startSec: 4, + endSec: 6, + reason: "", + origin: "user", + }, + ], + }, + legacyEditor: { speedRegions: [{ id: "speed_a", startMs: 0, endMs: 10_000, speed: 2 }] }, + }; +} + function setPlayhead(currentTimeSec: number, playing: boolean) { act(() => { useProjectStore.setState({ currentTimeSec, playing }); @@ -248,4 +272,34 @@ describe("NativeCompositorOverlay while playing", () => { expect(native.setNativePlaying).toHaveBeenCalledWith(false); expect(native.setActiveClip).toHaveBeenCalledTimes(1); }); + + it("uses an untrimmed 1x scene while recording and restores the edited scene afterward", async () => { + useProjectStore.setState({ document: makeTrimmedDocument(), currentTimeSec: 8 }); + const view = render(); + await act(async () => { + await Promise.resolve(); + }); + native.setActiveClip.mockClear(); + native.setNativeScene.mockClear(); + + view.rerender(); + await act(async () => { + await Promise.resolve(); + }); + const recordingScene = JSON.parse(native.setNativeScene.mock.lastCall?.[0] ?? "{}"); + expect(recordingScene.clips).toHaveLength(1); + expect(recordingScene.speedRegions).toEqual([]); + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 0, 8); + + native.setActiveClip.mockClear(); + native.setNativeScene.mockClear(); + view.rerender(); + await act(async () => { + await Promise.resolve(); + }); + const editedScene = JSON.parse(native.setNativeScene.mock.lastCall?.[0] ?? "{}"); + expect(editedScene.clips).toHaveLength(2); + expect(editedScene.speedRegions.length).toBeGreaterThan(0); + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 1, 8); + }); }); diff --git a/src/components/ai-edition/NativeCompositorOverlay.tsx b/src/components/ai-edition/NativeCompositorOverlay.tsx index d60bbcd20..8d5326989 100644 --- a/src/components/ai-edition/NativeCompositorOverlay.tsx +++ b/src/components/ai-edition/NativeCompositorOverlay.tsx @@ -55,7 +55,11 @@ import { * calques interactifs (zoom gimbal, annotations, drag webcam) qui restent des * éléments DOM cliquables au-dessus. */ -export function NativeCompositorOverlay() { +export function NativeCompositorOverlay({ + playUneditedTimeline = false, +}: { + playUneditedTimeline?: boolean; +}) { const canvasRef = useRef(null); const previousActiveClipIdRef = useRef(null); const document = useProjectStore((s) => s.document); @@ -70,22 +74,25 @@ export function NativeCompositorOverlay() { () => 0, ); - // `currentTimeSec` est en temps RAW/document (la règle où les trims occupent encore leur - // place — même référentiel que le playhead V4 et l'overlay webcam web). Le natif, lui, joue - // les segments COMPACTÉS (`resolveVisibleClips`, trims retirés) — le même flux que la SCÈNE - // envoyée via `setNativeScene`. `resolveNativePosition` fait le pont : il mappe le playhead - // RAW → temps source via le layout RAW (`document.timeline.clips`) PUIS localise le segment - // compacté correspondant, d'où sortent le `clipIndex` natif et la bonne paire écran/webcam. - // Sans ça (ancien `resolveNativePlaybackPosition(nativeClips, currentTimeSec)`), un playhead - // RAW lu contre des clips compactés désignait le mauvais clip après un trim → mauvaise caméra - // + décalage écran/cam. - const nativeClips = useMemo(() => { - if (!document) return []; - return resolveVisibleClips(document); - }, [document]); + // `currentTimeSec` stays on the raw document ruler. During normal playback the native scene + // uses trim-compressed segments; while recording, `playbackDocument` removes trims and speed + // regions so the compositor shows the same unedited timeline as the video clock. + const playbackDocument = useMemo(() => { + if (!document || !playUneditedTimeline) return document; + return { + ...document, + timeline: { ...document.timeline, trimRanges: [] }, + legacyEditor: document.legacyEditor ? { ...document.legacyEditor, speedRegions: [] } : null, + }; + }, [document, playUneditedTimeline]); + const nativeClips = useMemo( + () => (playbackDocument ? resolveVisibleClips(playbackDocument) : []), + [playbackDocument], + ); const activePosition = useMemo( - () => resolveNativePosition(currentTimeSec, nativeClips, document?.timeline.clips ?? []), - [nativeClips, currentTimeSec, document], + () => + resolveNativePosition(currentTimeSec, nativeClips, playbackDocument?.timeline.clips ?? []), + [nativeClips, currentTimeSec, playbackDocument], ); const activeClip = activePosition?.clip ?? null; @@ -149,12 +156,12 @@ export function NativeCompositorOverlay() { // quand on pousse. // biome-ignore lint/correctness/useExhaustiveDependencies: size revision useEffect(() => { - if (viewId === null || !document) { + if (viewId === null || !playbackDocument) { return; } try { const webcamSourceSize = cameraPath ? getWebcamNativeSize(cameraPath) : null; - const scene = buildSceneDescription(document, webcamSourceSize); + const scene = buildSceneDescription(playbackDocument, webcamSourceSize); setNativeScene(JSON.stringify(scene)); } catch (error) { console.warn("[compositor-view] build/push scene failed:", error); @@ -164,7 +171,7 @@ export function NativeCompositorOverlay() { // re-read fresh via getWebcamNativeSize() above on every run (biome flags this as // an "unnecessary" dependency, but removing it would mean a probed webcam size // arriving after mount never gets pushed to native). - }, [viewId, document, sources, _webcamSizeRevision]); + }, [viewId, playbackDocument, sources, _webcamSizeRevision]); // SYNCHRO COMPLETE DES PARAMS, en un seul endroit. // @@ -195,14 +202,17 @@ export function NativeCompositorOverlay() { const activeSourceTimeSec = activePosition?.sourceTimeSec ?? null; // Matched on the raw ruler, like `currentTimeSec` and the regions' pills. const speedRegions = useMemo( - () => (document ? readSpeedRegions(document) : []), - [document], + () => (playbackDocument ? readSpeedRegions(playbackDocument) : []), + [playbackDocument], ); - const playbackSpeed = findActiveSpeedRegion(speedRegions, currentTimeSec * 1000)?.speed ?? 1; + const playbackSpeed = playUneditedTimeline + ? 1 + : (findActiveSpeedRegion(speedRegions, currentTimeSec * 1000)?.speed ?? 1); const pendingTargetClipIdRef = useRef(null); const playing = useProjectStore((s) => s.playing); const wasPlayingRef = useRef(playing); + const previousUneditedTimelineRef = useRef(playUneditedTimeline); // Change les décodeurs screen/webcam quand le playhead entre dans un autre clip, à l'arrêt. // @@ -214,6 +224,8 @@ export function NativeCompositorOverlay() { useEffect(() => { const justPaused = wasPlayingRef.current && !playing; wasPlayingRef.current = playing; + const playbackModeChanged = previousUneditedTimelineRef.current !== playUneditedTimeline; + previousUneditedTimelineRef.current = playUneditedTimeline; if ( viewId === null || !document || @@ -230,7 +242,7 @@ export function NativeCompositorOverlay() { // y retourne, sans quoi le recalage en temps (`setNativeTime`) chercherait dans le // mauvais fichier. const nativeElsewhere = justPaused && native !== null && native.clipIndex !== activeClipIndex; - if (!clipChanged && !nativeElsewhere) { + if (!clipChanged && !playbackModeChanged && !nativeElsewhere) { return; } const asset = document.assets.find((candidate) => candidate.id === activeClip.assetId); @@ -305,6 +317,7 @@ export function NativeCompositorOverlay() { activeClipIndex, activeSourceTimeSec, playing, + playUneditedTimeline, nativeClips, playbackSpeed, ]); diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index 13e5c1c47..cb835d20a 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -105,15 +105,22 @@ const NO_AUDIO_TRACKS: AxcutAudioTrack[] = []; function NativePlaybackSync({ visibleClips, clips, + playUneditedTimeline, }: { visibleClips: AxcutClip[]; clips: AxcutClip[]; + playUneditedTimeline: boolean; }) { const playing = useProjectStore((s) => s.playing); const currentTimeSec = useProjectStore((s) => s.currentTimeSec); - // visibleClips = trim-compressed native stream; `clips` = RAW layout currentTimeSec - // is measured against. resolveNativePosition needs both (see timelineMap). - useNativePlaybackSync(playing, currentTimeSec, visibleClips, clips); + // The native stream follows the edited timeline except while a voiceover records. + // `currentTimeSec` always stays on the raw ruler, including during playback. + useNativePlaybackSync( + playing, + currentTimeSec, + playUneditedTimeline ? clips : visibleClips, + clips, + ); return null; } @@ -1008,12 +1015,20 @@ export function NewEditorShell() { // where it is passed to the preview. const [voiceoverRecording, setVoiceoverRecording] = useState(false); - const handleVoiceoverRecordingStart = useCallback(() => { + const handleVoiceoverRecordingStart = useCallback(async () => { // Re-capture: the user may have scrubbed between opening the dialog and // hitting Record, and playback starts from wherever the playhead is now. - voiceoverStartSecRef.current = useProjectStore.getState().currentTimeSec; + const doc = useProjectStore.getState().document; + const playhead = useProjectStore.getState().currentTimeSec; + voiceoverStartSecRef.current = playhead; + if (doc) { + const total = doc.timeline.clips.reduce((max, c) => Math.max(max, c.timelineEndSec), 0); + setVoiceoverFlow({ maxDurationSec: Math.max(0.5, total - playhead) }); + } setVoiceoverRecording(true); - if (videoElement?.paused) void videoElement.play().catch(() => undefined); + if (!videoElement) throw new Error("The preview is not ready."); + videoElement.playbackRate = 1; + if (videoElement.paused) await videoElement.play(); }, [videoElement]); const handleVoiceoverRecordingStop = useCallback(() => { @@ -1024,7 +1039,8 @@ export function NewEditorShell() { const handleVoiceoverReady = useCallback( async (assetId: string, durationSec: number) => { setVoiceoverFlow(null); - await tl.addAudioTrack(assetId, voiceoverStartSecRef.current, { + const startSec = voiceoverStartSecRef.current; + await tl.addAudioTrack(assetId, startSec, { kind: "voiceover", durationSec, spanSec: durationSec, @@ -1548,7 +1564,11 @@ export function NewEditorShell() { className={v4.app} style={{ gridTemplateRows: `58px 1fr ${showTimeline ? timelineRow : "0px"}` }} > - + ({ + nativePlaybackSync: vi.fn(), + previewPlay: vi.fn(async () => undefined), +})); + +vi.mock("@/contexts/ShortcutsContext", async () => { + const { DEFAULT_SHORTCUTS } = await import("@/lib/shortcuts"); + return { + useShortcuts: () => ({ + shortcuts: DEFAULT_SHORTCUTS, + isMac: false, + isConfigOpen: false, + openConfig: vi.fn(), + closeConfig: vi.fn(), + setShortcuts: vi.fn(), + persistShortcuts: () => Promise.resolve(true), + }), + }; +}); + +vi.mock("@/contexts/I18nContext", () => ({ + useI18n: () => ({ locale: "en", setLocale: vi.fn() }), + useScopedT: () => (key: string) => key, +})); + +vi.mock("@/native/useNativePlaybackSync", () => ({ + useNativePlaybackSync: mocks.nativePlaybackSync, +})); + +vi.mock("./Preview", async () => { + const { createElement, useEffect } = await import("react"); + return { + Preview: ({ + playUneditedTimeline, + onVideoElement, + }: { + playUneditedTimeline?: boolean; + onVideoElement: (element: HTMLVideoElement | null) => void; + }) => { + useEffect(() => { + const element = document.createElement("video"); + Object.defineProperty(element, "paused", { configurable: true, value: true }); + element.play = mocks.previewPlay; + element.pause = vi.fn(); + onVideoElement(element); + return () => onVideoElement(null); + }, [onVideoElement]); + return createElement("div", { + "data-testid": "preview-recording-mode", + "data-unedited": String(playUneditedTimeline ?? false), + }); + }, + }; +}); + +vi.mock("./v4/V4Timeline", () => ({ + V4Timeline: ({ onAddVoiceover }: { onAddVoiceover: () => void }) => + createElement("button", { type: "button", onClick: onAddVoiceover }, "open-voiceover"), +})); + +vi.mock("./v4/AddAudioLayerDialog", () => ({ + AddAudioLayerDialog: ({ + open, + onComplete, + onRecordingStart, + onRecordingStop, + }: { + open: boolean; + onComplete: (assetId: string, durationSec: number) => void; + onRecordingStart: () => void | Promise; + onRecordingStop: () => void; + }) => + open + ? createElement( + "div", + null, + createElement("button", { type: "button", onClick: onRecordingStart }, "start-recording"), + createElement("button", { type: "button", onClick: onRecordingStop }, "stop-recording"), + createElement( + "button", + { type: "button", onClick: () => onComplete("voice", 4) }, + "complete-recording", + ), + ) + : null, +})); + +import { NewEditorShell } from "./NewEditorShell"; + +function makeDocument(): AxcutDocument { + const document = createEmptyDocument({ projectId: "project_voiceover", title: "Voiceover" }); + document.project.primaryAssetId = "screen"; + const screenAsset: AxcutAsset = { + id: "screen", + kind: "video", + label: "screen", + originalPath: "/screen.mp4", + durationSec: 10, + cameraTrack: null, + }; + const voiceAsset: AxcutAsset = { + id: "voice", + kind: "audio", + label: "voice", + originalPath: "/voice.webm", + durationSec: 4, + cameraTrack: null, + }; + const clip: AxcutClip = { + id: "clip", + assetId: screenAsset.id, + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + wordRefs: [], + origin: "system", + reason: "", + }; + document.assets = [screenAsset, voiceAsset]; + document.timeline.clips = [clip]; + document.timeline.trimRanges = [ + { + id: "trim", + assetId: screenAsset.id, + clipId: clip.id, + startSec: 3, + endSec: 5, + reason: "", + origin: "user", + }, + ]; + return document; +} + +function renderShell() { + return render( + + + + + , + ); +} + +describe("NewEditorShell voiceover placement", () => { + beforeEach(() => { + localStorage.clear(); + vi.clearAllMocks(); + (window as unknown as { electronAPI?: unknown }).electronAPI = { + onAiEditionChatEvent: () => () => undefined, + sendAiEditionChatPrompt: () => undefined, + setTitleBarOverlay: () => undefined, + setHasUnsavedChanges: () => undefined, + onRequestCloseConfirm: () => () => undefined, + onRequestSaveBeforeClose: () => () => undefined, + sendCloseConfirmResponse: () => undefined, + findRecordingCamera: () => Promise.resolve(null), + preparePreviewAudioTrack: () => Promise.resolve(null), + isAppPackaged: () => false, + }; + Element.prototype.scrollTo = () => undefined; + (globalThis as unknown as { ResizeObserver?: unknown }).ResizeObserver = class { + observe = vi.fn(); + unobserve = vi.fn(); + disconnect = vi.fn(); + }; + const document = makeDocument(); + const saveDocument = vi.fn(async (next: AxcutDocument) => { + useProjectStore.setState({ + document: next, + revision: 2, + dirty: true, + lastSavedAt: new Date(), + }); + return true; + }); + useProjectStore.setState({ + projectId: document.project.id, + document, + revision: 1, + status: "ready", + error: null, + sourceDurationSec: 10, + currentTimeSec: 2, + playing: false, + dirty: false, + lastSavedAt: new Date(), + saveDocument, + }); + }); + + afterEach(() => { + cleanup(); + localStorage.clear(); + useProjectStore.getState().clear(); + }); + + it("adds a full-duration plain track across a cut and lets the cut mute its middle", async () => { + const document = useProjectStore.getState().document; + if (!document) throw new Error("document was not initialized"); + const rawClips = document.timeline.clips; + renderShell(); + + expect(mocks.nativePlaybackSync.mock.lastCall?.[2]).toHaveLength(2); + fireEvent.click(screen.getByRole("button", { name: "open-voiceover" })); + fireEvent.click(screen.getByRole("button", { name: "start-recording" })); + + expect(screen.getByTestId("preview-recording-mode")).toHaveAttribute("data-unedited", "true"); + expect(mocks.previewPlay).toHaveBeenCalledOnce(); + expect(mocks.nativePlaybackSync).toHaveBeenLastCalledWith(false, 2, rawClips, rawClips); + + act(() => { + useProjectStore.setState({ currentTimeSec: 4 }); + }); + fireEvent.click(screen.getByRole("button", { name: "stop-recording" })); + expect(mocks.nativePlaybackSync.mock.lastCall?.[2]).toHaveLength(2); + fireEvent.click(screen.getByRole("button", { name: "complete-recording" })); + + await waitFor(() => { + expect(useProjectStore.getState().document?.audioTracks).toHaveLength(1); + }); + const updatedDocument = useProjectStore.getState().document; + if (!updatedDocument) throw new Error("voiceover document was not saved"); + const [track] = updatedDocument.audioTracks; + expect(track).toMatchObject({ + assetId: "voice", + kind: "voiceover", + startMs: 2_000, + endMs: 6_000, + durationSec: 4, + }); + expect(track).not.toHaveProperty("recordingPlaybackMap"); + expect(buildSceneDescription(updatedDocument).audioTracks).toMatchObject([ + { startSec: 2, trimStartSec: 0, trimEndSec: 1, kind: "voiceover" }, + { startSec: 3, trimStartSec: 3, trimEndSec: 4, kind: "voiceover" }, + ]); + }); +}); diff --git a/src/components/ai-edition/Preview.tsx b/src/components/ai-edition/Preview.tsx index f4ad0a5c5..c809a329b 100644 --- a/src/components/ai-edition/Preview.tsx +++ b/src/components/ai-edition/Preview.tsx @@ -37,6 +37,7 @@ interface PreviewProps { clips: AxcutClip[]; zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; + playUneditedTimeline?: boolean; cameraFullscreenRegions?: CameraFullscreenRegion[]; trimRanges?: AxcutTrimRange[]; transcripts?: AxcutTranscript[]; @@ -72,6 +73,7 @@ export function Preview({ clips, zoomRegions, speedRegions, + playUneditedTimeline, cameraFullscreenRegions, trimRanges, transcripts, @@ -215,6 +217,7 @@ export function Preview({ clips={clips} zoomRegions={zoomRegions} speedRegions={speedRegions} + playUneditedTimeline={playUneditedTimeline} cameraFullscreenRegions={cameraFullscreenRegions} trimRanges={trimRanges} transcripts={transcripts} diff --git a/src/components/ai-edition/PreviewCanvas.tsx b/src/components/ai-edition/PreviewCanvas.tsx index c817fde3c..557593f4c 100644 --- a/src/components/ai-edition/PreviewCanvas.tsx +++ b/src/components/ai-edition/PreviewCanvas.tsx @@ -85,6 +85,7 @@ interface PreviewCanvasProps { clips: AxcutClip[]; zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; + playUneditedTimeline?: boolean; cameraFullscreenRegions?: CameraFullscreenRegion[]; trimRanges?: AxcutTrimRange[]; /** Relayed to `VirtualPreview` so a clip carrying added words plays their extensions. */ @@ -124,6 +125,7 @@ interface PreviewCanvasProps { // (the