From e06e69bdf1eac0db5025729e0d60560fed89cf64 Mon Sep 17 00:00:00 2001 From: wuzixuan06 Date: Thu, 22 Oct 2026 16:39:33 +0800 Subject: [PATCH] feat: add waitForRecording MCP tool for user-controlled takes --- electron/electron-env.d.ts | 4 + electron/ipc/handlers.ts | 56 ++++- electron/ipc/recordingStateRouting.test.ts | 2 +- electron/mcp/app-recorder.test.ts | 14 ++ electron/mcp/app-recorder.ts | 13 +- electron/mcp/media-tools.test.ts | 144 +++++++++++ electron/mcp/media-tools.ts | 57 ++++- electron/mcp/openscreen-mcp-server.test.ts | 65 ++++- electron/mcp/recording-completion.test.ts | 228 ++++++++++++++++++ electron/mcp/recording-completion.ts | 121 ++++++++++ electron/preload.ts | 3 + .../ai-edition/recordingImport.test.ts | 46 ++++ src/components/ai-edition/recordingImport.ts | 153 ++++++++---- ...eScreenRecorder.nativeStopFailure.test.tsx | 20 ++ src/hooks/useScreenRecorder.ts | 32 +++ src/lib/recordingSession.ts | 5 + .../architecture/mcp-server.md | 40 ++- .../architecture/recording.md | 7 + .../testing/manual-e2e-checklist.md | 3 + 19 files changed, 948 insertions(+), 65 deletions(-) create mode 100644 electron/mcp/recording-completion.test.ts create mode 100644 electron/mcp/recording-completion.ts diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index fc6b5f911..562b51dc5 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -46,6 +46,10 @@ interface Window { meta: import("../src/native/contracts").CompositorSharedFrameMeta, ) => void, ) => () => void; + reportRecordingOutcome?: ( + outcome: import("../src/lib/recordingSession").RecordingOutcome, + ) => Promise<{ success: boolean }>; + getSources: (opts: Electron.SourcesOptions) => Promise; switchToEditor: () => Promise; switchToHud: () => Promise; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index ee36e105f..087801185 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -18,6 +18,7 @@ import { shell, systemPreferences, } from "electron"; +import { z } from "zod"; import { DEFAULT_WEBCAM_QUALITY, type WebcamQualityId } from "../../src/hooks/webcamCaptureTarget"; import { type AxcutDocument, @@ -88,6 +89,7 @@ import { McpController } from "../mcp/mcp-controller"; import { McpSettingsStore } from "../mcp/mcp-settings-store"; import { createMediaTools, type MediaToolDeps, type RecordRequest } from "../mcp/media-tools"; import { createToolRunner, type McpToolDeps } from "../mcp/openscreen-mcp-server"; +import { RecordingCompletion } from "../mcp/recording-completion"; import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks"; import { readCursorRecordingFile as readCursorRecordingFileFrom, @@ -1966,15 +1968,17 @@ export function registerIpcHandlers( ) { /** The HUD is recording a take, the user's or an MCP client's. Set by `setAppRecording` only. */ let appRecording = false; + const recordingCompletion = new RecordingCompletion(); /** * Every start and stop of a take goes through here: the browser path's `set-recording-state` * and the native paths, which start and stop in this process. The MCP recorder waits on it * for a client's take to start, and the tray shows it. A native path that called * `onRecordingStateChange` itself left the MCP recorder waiting on a take already running. */ - const setAppRecording = (recording: boolean, sourceName: string) => { + const setAppRecording = (recording: boolean, sourceName: string, recordingId?: number) => { + if (recording && recordingId !== undefined) recordingCompletion.started(recordingId); appRecording = recording; - mcpAppRecorder.recordingChanged(recording); + mcpAppRecorder.recordingChanged(recording, recordingId); onRecordingStateChange?.(recording, sourceName); }; const appSettings = new AppSettingsStore(app.getPath("userData")); @@ -2760,7 +2764,7 @@ export function registerIpcHandlers( // "Screen" }`, so the tray confidently displayed the name of a // window the capture had never been told about. linuxNativeCaptureSourceLabel = linuxSourceLabel(session.grantedSourceKind); - setAppRecording(true, linuxNativeCaptureSourceLabel); + setAppRecording(true, linuxNativeCaptureSourceLabel, recordingId); return { success: true, recordingId, path: outputPath }; } catch (error) { @@ -3056,7 +3060,7 @@ export function registerIpcHandlers( }); const source = selectedSource || { name: "Screen" }; - setAppRecording(true, source.name); + setAppRecording(true, source.name, recordingId); // Reported at start, not at stop: the helper decides the camera is a // lost cause during its own init — before it announces "Recording @@ -3318,7 +3322,7 @@ export function registerIpcHandlers( : 0; const source = selectedSource || { name: "Screen" }; - setAppRecording(true, source.name); + setAppRecording(true, source.name, recordingId); return { success: true, @@ -4113,6 +4117,8 @@ export function registerIpcHandlers( ipcMain.handle( "set-recording-state", async (_, recording: boolean, recordingId?: number, cursorCaptureMode?: CursorCaptureMode) => { + const source = selectedSource || { name: "Screen" }; + setAppRecording(recording, source.name, recordingId); const normalizedCursorCaptureMode = normalizeCursorCaptureMode(cursorCaptureMode) ?? "editable-overlay"; if (recording && normalizedCursorCaptureMode === "editable-overlay") { @@ -4120,9 +4126,6 @@ export function registerIpcHandlers( } else { await stopCursorRecording(); } - - const source = selectedSource || { name: "Screen" }; - setAppRecording(recording, source.name); }, ); @@ -4864,10 +4867,43 @@ export function registerIpcHandlers( setCurrentRecordingSessionState(normalizedSession); currentVideoPath = normalizedSession?.screenVideoPath ?? null; currentProjectPath = null; - if (normalizedSession) mcpAppRecorder.takeSaved(normalizedSession.screenVideoPath); + if (normalizedSession) { + mcpAppRecorder.takeSaved(normalizedSession.screenVideoPath); + recordingCompletion.saved(normalizedSession.createdAt, normalizedSession.screenVideoPath); + } return { success: true, session: currentRecordingSession }; }); + // A renderer acknowledgement of the actual hand-off, after the duration and + // automatic zoom saves. An imported/opened project never emits this signal. + const recordingOutcomeSchema = z.discriminatedUnion("status", [ + z.object({ + recordingId: z.number().finite(), + status: z.literal("ready"), + projectId: z.string(), + }), + z.object({ + recordingId: z.number().finite(), + status: z.enum(["failed", "cancelled"]), + message: z.string(), + }), + ]); + ipcMain.handle("recording-outcome", async (_, input: unknown) => { + const parsed = recordingOutcomeSchema.safeParse(input); + if (!parsed.success) return { success: false }; + const outcome = parsed.data; + if (outcome.status !== "ready") { + recordingCompletion.failed(outcome.recordingId, outcome.message); + mcpAppRecorder.recordingFinished(outcome.recordingId); + return { success: true }; + } + await recordingCompletion.projectReady(outcome.recordingId, outcome.projectId, (id) => + aiEditionDocuments.getProject(id), + ); + mcpAppRecorder.recordingFinished(outcome.recordingId); + return { success: true }; + }); + ipcMain.handle("get-current-recording-session", () => { if (!currentRecordingSession) { return { success: false }; @@ -5151,6 +5187,8 @@ export function registerIpcHandlers( host: mcpToolDeps.host, }); const mcpMedia = createMediaTools({ + recordingActive: () => recordingCompletion.isActive(), + waitForRecording: (timeoutMs, signal) => recordingCompletion.wait(timeoutMs, signal), transport: "http", // Recording and exporting change things too: the same switch as edits. editsAllowed: mcpToolDeps.editsAllowed, diff --git a/electron/ipc/recordingStateRouting.test.ts b/electron/ipc/recordingStateRouting.test.ts index 13c7193f1..96851dec7 100644 --- a/electron/ipc/recordingStateRouting.test.ts +++ b/electron/ipc/recordingStateRouting.test.ts @@ -36,7 +36,7 @@ describe("recording state routing", () => { const setter = source.slice(source.indexOf("const setAppRecording")); const body = setter.slice(0, setter.indexOf("};")); expect(body).toContain("appRecording = recording"); - expect(body).toContain("mcpAppRecorder.recordingChanged(recording)"); + expect(body).toContain("mcpAppRecorder.recordingChanged(recording, recordingId)"); expect(body).toContain("onRecordingStateChange?.(recording, sourceName)"); }); diff --git a/electron/mcp/app-recorder.test.ts b/electron/mcp/app-recorder.test.ts index 78b76be4e..78ebcb838 100644 --- a/electron/mcp/app-recorder.test.ts +++ b/electron/mcp/app-recorder.test.ts @@ -157,6 +157,20 @@ describe("an MCP take in the app", () => { expect(editor.applied).toEqual([]); }); + it("releases only the client take whose app lifecycle finished", async () => { + const { recorder, calls } = setup(); + const starting = recorder.start(REQUEST); + await new Promise((resolve) => setTimeout(resolve, 5)); + recorder.recordingChanged(true, 7); + await starting; + recorder.recordingFinished(6); + await expect(recorder.start(REQUEST)).rejects.toThrow("already recording"); + recorder.recordingChanged(false, 7); + recorder.recordingFinished(7); + expect(calls.filter((call) => call === "restore")).toHaveLength(1); + await expect(started(recorder, calls)).resolves.toBeDefined(); + }); + it("refuses a take while one records, or while the editor holds unsaved changes", async () => { const busy = setup({ isRecording: () => true }); await expect(busy.recorder.start(REQUEST)).rejects.toThrow("already recording"); diff --git a/electron/mcp/app-recorder.ts b/electron/mcp/app-recorder.ts index 6f2134147..dd4297f51 100644 --- a/electron/mcp/app-recorder.ts +++ b/electron/mcp/app-recorder.ts @@ -103,7 +103,9 @@ export interface AppRecorderDeps { export interface AppRecorder extends TakeRecorder { /** The app's recording state changed (`set-recording-state`). */ - recordingChanged(recording: boolean): void; + recordingChanged(recording: boolean, recordingId?: number): void; + /** A completed or failed app take no longer owns the recording bar. */ + recordingFinished(recordingId: number): void; /** The recording bar handed a saved take to the editor (`set-current-recording-session`). */ takeSaved(videoPath: string): void; } @@ -145,6 +147,7 @@ interface Deferred { } interface ClientTake { + recordingId?: number; phase: "starting" | "recording" | "saving"; /** Resolves with when capture started. */ started: Deferred; @@ -214,10 +217,16 @@ export function createAppRecorder(deps: AppRecorderDeps): AppRecorder { } return { - recordingChanged(recording) { + recordingFinished(recordingId) { + if (current?.recordingId !== recordingId) return; + current.restore(); + current = null; + }, + recordingChanged(recording, recordingId) { if (!current) return; if (recording && current.phase === "starting") { current.phase = "recording"; + current.recordingId = recordingId; current.started.resolve(Date.now()); } else if (!recording && current.phase === "recording") { // The take is captured: the bar's own settings can come back while it saves. diff --git a/electron/mcp/media-tools.test.ts b/electron/mcp/media-tools.test.ts index aaaf3cb3a..a27a350ce 100644 --- a/electron/mcp/media-tools.test.ts +++ b/electron/mcp/media-tools.test.ts @@ -26,6 +26,7 @@ import { createToolRunner, type McpToolDeps, } from "./openscreen-mcp-server"; +import { RecordingCompletion } from "./recording-completion"; /** The runner windows: records what was asked, answers like a runner would. */ class FakeRunners { @@ -1033,3 +1034,146 @@ describe("the media tools in the app", () => { expect(exportPath.description).not.toContain("working directory, ending"); }); }); + +describe("waitForRecording through MCP", () => { + const clients: Client[] = []; + afterEach(async () => { + for (const client of clients.splice(0)) await client.close(); + }); + async function observing( + completion: RecordingCompletion, + overrides: Partial = {}, + ) { + const deps: McpToolDeps = { + transport: "http", + host: { snapshot: async () => null, apply: async () => "no-editor" }, + projects: documents, + editsAllowed: () => false, + version: "test", + }; + let subscribed: () => void = () => undefined; + const subscription = new Promise((resolve) => { + subscribed = resolve; + }); + const media = createMediaTools({ + ...stdioDeps, + transport: "http", + recordingActive: () => completion.isActive(), + editsAllowed: () => false, + waitForRecording: (timeout, signal) => { + const waiting = completion.wait(timeout, signal); + subscribed(); + return waiting; + }, + ...overrides, + }); + const server = createOpenScreenMcpServer(deps, createToolRunner(deps), media); + const [clientSide, serverSide] = InMemoryTransport.createLinkedPair(); + await server.connect(serverSide); + const observer = new Client({ name: "observer", version: "1" }); + await observer.connect(clientSide); + clients.push(observer); + return { observer, subscription }; + } + + it("returns the stopRecording fields for a user take while MCP edits is off, without recording commands", async () => { + const project = await documents.createProject("User take"); + const completion = new RecordingCompletion(); + completion.started(42, 1_000); + const { observer, subscription } = await observing(completion); + const waiting = observer.callTool({ name: "waitForRecording", arguments: {} }); + await subscription; + completion.saved(42, videoPath); + completion.completed(42, { + projectId: project.project.id, + videoPath, + durationSec: 12, + autoZooms: 2, + }); + const result = await waiting; + expect(result.isError).toBeFalsy(); + expect(JSON.parse(textOf(result as CallToolResult))).toEqual({ + projectId: project.project.id, + videoPath, + durationSec: 12, + autoZooms: 2, + recordingStartedAt: 1_000, + checkpointId: expect.any(String), + }); + expect(runners.started).toEqual([]); + }); + + it("releases a client take after the user finishes it, so startRecording can record again", async () => { + const completion = new RecordingCompletion(); + let id = 0; + const recorder: TakeRecorder = { + start: async () => { + completion.started(++id); + return { + startedAt: id, + stop: async () => { + throw new Error("wait must never stop the take"); + }, + }; + }, + }; + const { observer, subscription } = await observing(completion, { + recorder, + editsAllowed: () => true, + }); + expect( + (await observer.callTool({ name: "startRecording", arguments: {} })).isError, + ).toBeFalsy(); + const waiting = observer.callTool({ name: "waitForRecording", arguments: {} }); + await subscription; + completion.saved(1, videoPath); + completion.completed(1, { projectId: "proj_take", videoPath, durationSec: 12, autoZooms: 0 }); + await waiting; + expect( + (await observer.callTool({ name: "startRecording", arguments: {} })).isError, + ).toBeFalsy(); + expect(id).toBe(2); + }); + + it.each([ + 0, + -1, + 1.5, + 3601, + "10", + ])("validates timeoutSec=%s before observing", async (timeoutSec) => { + const completion = new RecordingCompletion(); + const { observer } = await observing(completion); + const result = await observer.callTool({ name: "waitForRecording", arguments: { timeoutSec } }); + expect(result.isError).toBe(true); + expect(textOf(result as CallToolResult)).toContain("validation"); + }); + + it("reports no active recording as a tool error", async () => { + const { observer } = await observing(new RecordingCompletion()); + const result = await observer.callTool({ name: "waitForRecording", arguments: {} }); + expect(result.isError).toBe(true); + expect(textOf(result as CallToolResult)).toContain("No recording"); + }); + + it("propagates MCP cancellation and leaves the user's take available", async () => { + const completion = new RecordingCompletion(); + completion.started(42); + const { observer, subscription } = await observing(completion); + const abort = new AbortController(); + const waiting = expect( + observer.callTool({ name: "waitForRecording", arguments: {} }, undefined, { + signal: abort.signal, + }), + ).rejects.toThrow(); + await subscription; + abort.abort(); + await waiting; + // Flush the MCP cancellation notification, then the same take still completes. + await new Promise((resolve) => setTimeout(resolve, 0)); + const retry = completion.wait(1000, new AbortController().signal); + completion.saved(42, videoPath); + completion.completed(42, { projectId: "proj_take", videoPath, durationSec: 12, autoZooms: 0 }); + await expect(retry).resolves.toMatchObject({ projectId: "proj_take" }); + }); +}); diff --git a/electron/mcp/media-tools.ts b/electron/mcp/media-tools.ts index c112d2bda..6fce30ccf 100644 --- a/electron/mcp/media-tools.ts +++ b/electron/mcp/media-tools.ts @@ -89,6 +89,14 @@ export interface TakeRecorder { } export interface MediaToolDeps { + /** The app may finish a client take from its HUD, without stopRecording. */ + recordingActive?(): boolean; + /** Observes the app's current take without starting or stopping it. */ + waitForRecording?: ( + timeoutMs: number, + signal: AbortSignal, + ) => Promise; + /** The server these tools are served by: their texts say what is true on it. */ transport: McpTransport; /** Whether a client may record and export: the in-app server's "MCP edits". */ @@ -226,7 +234,7 @@ const TAKE_NOTES: Record = { export function mediaPreamble(transport: McpTransport): string { return [ "OpenScreen records the screen, turns each take into a project you edit with the document tools, and exports it as a video.", - "A demo goes: listSources, then startRecording, then drive the app, then stopRecording. It returns the projectId and a checkpointId. Edit with the document tools, always passing that projectId: every edit is saved to the project straight away, and restoreCheckpoint with the checkpointId takes the project back to the raw take in one step. Then exportVideo, and getFrames with the projectId to check the export before you hand it over: it takes the same ruler seconds as the document tools, so a trim or a speed region never shifts what you look at.", + "A demo goes: listSources, then startRecording, then drive the app, then stopRecording. To observe a take the user controls on the in-app HTTP server, call waitForRecording while it is recording or saving; the user stops it. It returns the projectId and a checkpointId. Edit with the document tools, always passing that projectId: every edit is saved to the project straight away, and restoreCheckpoint with the checkpointId takes the project back to the raw take in one step. Then exportVideo, and getFrames with the projectId to check the export before you hand it over: it takes the same ruler seconds as the document tools, so a trim or a speed region never shifts what you look at.", "To edit a video you already have instead, createProject with its videoPath; importMedia adds a video or an audio file to a project.", "Drive the app with the real mouse when you can: OpenScreen records the pointer, draws a clean cursor and zooms on the clicks as the editor does: clicks close in time and on screen share one zoom, and the first seconds and the end stay on the whole screen. If you drive it from a script instead (Playwright, CDP), the screen pointer never moves: log each action as the script runs it, then hand the log to setPointerTrack after stopRecording.", TAKE_NOTES[transport], @@ -592,6 +600,13 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { return text(error instanceof Error ? error.message : String(error), true); } startedAtByVideo.set(recorded.videoPath, running.startedAt); + return recordingResult(recorded, running.startedAt); + } + + async function recordingResult( + recorded: RecordedTake, + recordingStartedAt: number, + ): Promise { const { projectId } = recorded; return text( JSON.stringify({ @@ -599,7 +614,7 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { checkpointId: await checkpointOf(projectId), videoPath: recorded.videoPath, durationSec: recorded.durationSec, - recordingStartedAt: running.startedAt, + recordingStartedAt: recordingStartedAt, autoZooms: recorded.autoZooms, ...(recorded.warnings?.length ? { warnings: recorded.warnings } : {}), }), @@ -673,6 +688,7 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { }, async (args) => { if (!deps.editsAllowed()) return text(GATED_MESSAGE, true); + if (take?.running && !take.stopping && deps.recordingActive?.() === false) take = null; if (take?.stopping) { return text("The last take is still being saved. Try again in a moment.", true); } @@ -734,6 +750,43 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { }, ); + server.registerTool( + "waitForRecording", + { + description: + "Wait for the current recording in the OpenScreen app to finish and its project to be saved, including a take started by the user at the recording bar. Never starts or stops a recording. Binds to this take only, including while it saves; returns an error immediately if none is active. Returns the same recording fields as stopRecording. Timeout or request cancellation only ends the wait. Available on the in-app HTTP server; openscreen mcp cannot observe the app's recording bar.", + inputSchema: z.object({ + timeoutSec: z + .number() + .int() + .min(1) + .max(3600) + .default(300) + .describe( + "Maximum seconds to wait, including saving (1–3600, default 300). Set the client's tool-call timeout longer than this.", + ), + }), + annotations: READ_ONLY, + }, + async ({ timeoutSec }, { signal }) => { + if (!deps.waitForRecording) + return text( + "waitForRecording requires the in-app HTTP MCP server. Connect to it to observe the user's recording.", + true, + ); + const observedTake = take; + try { + const recorded = await deps.waitForRecording(timeoutSec * 1000, signal); + return recordingResult(recorded, recorded.recordingStartedAt); + } catch (error) { + return text(error instanceof Error ? error.message : String(error), true); + } finally { + if (take === observedTake && !take?.stopping && deps.recordingActive?.() === false) + take = null; + } + }, + ); + server.registerTool( "setPointerTrack", { diff --git a/electron/mcp/openscreen-mcp-server.test.ts b/electron/mcp/openscreen-mcp-server.test.ts index 85e0fbf2d..38877c99a 100644 --- a/electron/mcp/openscreen-mcp-server.test.ts +++ b/electron/mcp/openscreen-mcp-server.test.ts @@ -3,6 +3,7 @@ // DocumentService on a temp directory for the projects that are not open. import { spawnSync } from "node:child_process"; +import { getEventListeners } from "node:events"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import fsPromises from "node:fs/promises"; import os from "node:os"; @@ -18,7 +19,7 @@ import { import { OPENSCREEN_TOOL_NAMES } from "../ai-edition/agent-tools"; import { TOOL_DESCRIPTIONS } from "../ai-edition/deep-agent/service"; import { DocumentService } from "../ai-edition/document-service"; -import { createMediaTools } from "./media-tools"; +import { createMediaTools, type MediaToolDeps } from "./media-tools"; import { LIST_PROJECTS_TOOL, MCP_ENDPOINT_PATH, @@ -29,6 +30,7 @@ import { type RunningMcpServer, startMcpHttpServer, } from "./openscreen-mcp-server"; +import { RecordingCompletion } from "./recording-completion"; const TOKEN = "test-token-0123456789"; @@ -146,6 +148,7 @@ async function connect( editsAllowed?: boolean; headers?: Record; media?: Pick; + waitForRecording?: MediaToolDeps["waitForRecording"]; } = {}, ) { const editsAllowed = () => options.editsAllowed ?? true; @@ -166,6 +169,7 @@ async function connect( deps, // As the app serves them; media-tools.test.ts calls them. media: createMediaTools({ + waitForRecording: options.waitForRecording, transport: "http", editsAllowed, recorder: { start: unused }, @@ -207,6 +211,7 @@ describe("the MCP tool surface", () => { "listSources", "startRecording", "stopRecording", + "waitForRecording", "setPointerTrack", "exportVideo", "getFrames", @@ -987,3 +992,61 @@ describe("the HTTP guard", () => { expect((await post({ Authorization: `Bearer ${TOKEN}` }, "/other")).status).toBe(404); }); }); + +describe("HTTP recording waits", () => { + it("drops the wait's listener when its HTTP request disconnects, leaving the take alive", async () => { + const completion = new RecordingCompletion(); + completion.started(7); + let requestSignal: AbortSignal | undefined; + let subscribed: () => void = () => undefined; + const subscription = new Promise((resolve) => { + subscribed = resolve; + }); + await connect(new FakeEditor(), { + waitForRecording: (timeout, signal) => { + requestSignal = signal; + const waiting = completion.wait(timeout, signal); + subscribed(); + return waiting; + }, + }); + const controller = new AbortController(); + const request = fetch(`http://127.0.0.1:${running?.port}${MCP_ENDPOINT_PATH}`, { + method: "POST", + headers: { + Authorization: `Bearer ${TOKEN}`, + "Content-Type": "application/json", + Accept: "application/json, text/event-stream", + }, + body: JSON.stringify({ + jsonrpc: "2.0", + id: 77, + method: "tools/call", + params: { name: "waitForRecording", arguments: { timeoutSec: 3600 } }, + }), + signal: controller.signal, + }); + const rejected = expect(request).rejects.toThrow(); + await subscription; + expect(getEventListeners(requestSignal as AbortSignal, "abort")).toHaveLength(1); + controller.abort(); + await rejected; + await vi.waitFor(() => expect(requestSignal?.aborted).toBe(true)); + expect(getEventListeners(requestSignal as AbortSignal, "abort")).toHaveLength(0); + expect(completion.isActive()).toBe(true); + completion.failed(7, "test cleanup"); + }); + + it("returns a tool timeout over HTTP without ending the recording", async () => { + const completion = new RecordingCompletion(); + completion.started(7); + const mcp = await connect(new FakeEditor(), { + waitForRecording: (timeout, signal) => completion.wait(timeout, signal), + }); + const result = await mcp.callTool({ name: "waitForRecording", arguments: { timeoutSec: 1 } }); + expect(result.isError).toBe(true); + expect(resultText(result)).toContain("Timed out"); + expect(completion.isActive()).toBe(true); + completion.failed(7, "test cleanup"); + }); +}); diff --git a/electron/mcp/recording-completion.test.ts b/electron/mcp/recording-completion.test.ts new file mode 100644 index 000000000..213d5dc90 --- /dev/null +++ b/electron/mcp/recording-completion.test.ts @@ -0,0 +1,228 @@ +import { getEventListeners } from "node:events"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { replaceTimeline } from "../../src/lib/ai-edition/document/timeline"; +import { createEmptyDocument } from "../../src/lib/ai-edition/schema"; +import { RecordingCompletion } from "./recording-completion"; + +const RESULT = { projectId: "proj_take", videoPath: "/take.mp4", durationSec: 12, autoZooms: 2 }; +let completion: RecordingCompletion; +let abort: AbortController; +beforeEach(() => { + vi.useFakeTimers(); + completion = new RecordingCompletion(); + abort = new AbortController(); +}); +afterEach(() => { + expect(getEventListeners(abort.signal, "abort")).toHaveLength(0); + expect(vi.getTimerCount()).toBe(0); + vi.useRealTimers(); +}); + +describe("waiting on the app's recording", () => { + it("refuses idle calls and never returns an older take", async () => { + await expect(completion.wait(100, abort.signal)).rejects.toThrow("No recording"); + completion.started(1, 123); + completion.saved(1, RESULT.videoPath); + completion.completed(1, RESULT); + await expect(completion.wait(100, abort.signal)).rejects.toThrow("No recording"); + }); + + it("does not miss completion immediately after subscription, even before the first await", async () => { + completion.started(1, 123); + const waiting = completion.wait(100, abort.signal); + completion.saved(1, RESULT.videoPath); + completion.completed(1, RESULT); + expect(await waiting).toEqual({ ...RESULT, recordingStartedAt: 123 }); + }); + + it("requires that take's saved media and ignores imported projects and other recordings", async () => { + completion.started(1, 123); + completion.saved(99, "/import.mp4"); + const waiting = completion.wait(100, abort.signal); + completion.completed(1, RESULT); // no saved session yet + completion.saved(1, RESULT.videoPath); + completion.completed(99, RESULT); + completion.completed(1, { ...RESULT, videoPath: "/import.mp4" }); + expect(vi.getTimerCount()).toBe(1); + completion.completed(1, RESULT); + expect(await waiting).toMatchObject({ videoPath: RESULT.videoPath }); + }); + + it("binds late calls during saving and all concurrent waits to the same result", async () => { + completion.started(1, 123); + const first = completion.wait(100, abort.signal); + completion.saved(1, RESULT.videoPath); + const second = completion.wait(100, abort.signal); + completion.completed(1, RESULT); + expect(await first).toEqual(await second); + }); + + it("times out only the wait; a retry can still get the same take", async () => { + completion.started(1, 123); + const waiting = expect(completion.wait(10, abort.signal)).rejects.toThrow("Timed out"); + await vi.advanceTimersByTimeAsync(10); + await waiting; + const retry = completion.wait(100, abort.signal); + completion.saved(1, RESULT.videoPath); + completion.completed(1, RESULT); + expect(await retry).toMatchObject({ recordingStartedAt: 123 }); + }); + + it("cancels one request without affecting other waiters or the take", async () => { + completion.started(1); + const first = expect(completion.wait(100, abort.signal)).rejects.toThrow("cancelled"); + const other = new AbortController(); + const second = completion.wait(100, other.signal); + abort.abort(); + await first; + expect(getEventListeners(other.signal, "abort")).toHaveLength(1); + completion.saved(1, RESULT.videoPath); + completion.completed(1, RESULT); + await second; + expect(getEventListeners(other.signal, "abort")).toHaveLength(0); + }); + + it("cleans up a signal already cancelled when the call arrives", async () => { + completion.started(1); + abort.abort(); + await expect(completion.wait(100, abort.signal)).rejects.toThrow("cancelled"); + }); + + it.each([ + "Recording failed: disk full", + "The recording was cancelled or discarded.", + ])("ends all waits on %s", async (message) => { + completion.started(1); + const first = expect(completion.wait(100, abort.signal)).rejects.toThrow(message); + const second = expect(completion.wait(100, abort.signal)).rejects.toThrow(message); + completion.failed(99, "unrelated failure"); + completion.failed(1, message); + await Promise.all([first, second]); + await expect(completion.wait(100, abort.signal)).rejects.toThrow("No recording"); + }); + + it("fails a superseded target instead of waiting for the next take", async () => { + completion.started(1); + const first = expect(completion.wait(100, abort.signal)).rejects.toThrow("replaced"); + completion.started(2, 456); + await first; + const second = completion.wait(100, abort.signal); + completion.saved(1, "/old.mp4"); + completion.failed(1, "late error"); + completion.completed(1, { ...RESULT, videoPath: "/old.mp4" }); + completion.saved(2, RESULT.videoPath); + completion.completed(2, RESULT); + expect(await second).toMatchObject({ recordingStartedAt: 456 }); + }); +}); + +describe("the saved recording result", () => { + let dir: string; + beforeEach(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), "recording-ready-")); + }); + afterEach(async () => { + await fs.rm(dir, { recursive: true, force: true }); + }); + + async function savedProject(videoPath = path.join(dir, "take.mp4"), durationSec = 12) { + const base = createEmptyDocument({ projectId: "proj_saved", title: "Take" }); + const doc = replaceTimeline( + { + ...base, + project: { ...base.project, primaryAssetId: "asset_screen" }, + assets: [ + { + id: "asset_screen", + kind: "video", + label: "Take", + originalPath: videoPath, + durationSec, + cameraTrack: null, + }, + ], + }, + [{ startSec: 0, endSec: durationSec }], + "Recorded", + ); + const projectFile = path.join(dir, "project.openscreen"); + await fs.writeFile(projectFile, JSON.stringify(doc)); + return { + doc, + read: async () => JSON.parse(await fs.readFile(projectFile, "utf8")) as typeof doc, + videoPath, + }; + } + + it("reads the persisted project and verifies its actual media before resolving", async () => { + const { read, videoPath } = await savedProject(); + await fs.writeFile(videoPath, "test media"); + completion.started(1, 123); + completion.saved(1, videoPath); + const waiting = completion.wait(100, abort.signal); + await completion.projectReady(1, "proj_saved", read); + expect(await waiting).toEqual({ + projectId: "proj_saved", + videoPath, + durationSec: 12, + autoZooms: 0, + recordingStartedAt: 123, + }); + }); + + it.each([ + "missing media", + "empty media", + "unmeasured duration", + "different asset", + "unreadable project", + ])("refuses success for %s", async (condition) => { + const { read, videoPath } = await savedProject( + condition === "different asset" ? path.join(dir, "import.mp4") : undefined, + condition === "unmeasured duration" ? 60 : 12, + ); + const target = path.join(dir, "take.mp4"); + if (condition !== "missing media") + await fs.writeFile(videoPath, condition === "empty media" ? "" : "media"); + completion.started(1); + completion.saved(1, target); + const waiting = expect(completion.wait(100, abort.signal)).rejects.toThrow( + "could not be prepared", + ); + await completion.projectReady( + 1, + "proj_saved", + condition === "unreadable project" + ? async () => { + throw new Error("disk failure"); + } + : read, + ); + await waiting; + }); + + it("does not deliver an old result if a new take starts while the project is being read", async () => { + const { read, videoPath } = await savedProject(); + await fs.writeFile(videoPath, "media"); + completion.started(1); + completion.saved(1, videoPath); + const waiting = expect(completion.wait(100, abort.signal)).rejects.toThrow("replaced"); + let release: () => void = () => undefined; + const opening = completion.projectReady(1, "proj_saved", async () => { + await new Promise((resolve) => { + release = resolve; + }); + return read(); + }); + completion.started(2); + await waiting; + release(); + await opening; + const next = expect(completion.wait(1, abort.signal)).rejects.toThrow("Timed out"); + await vi.advanceTimersByTimeAsync(1); + await next; + }); +}); diff --git a/electron/mcp/recording-completion.ts b/electron/mcp/recording-completion.ts new file mode 100644 index 000000000..2f48751ff --- /dev/null +++ b/electron/mcp/recording-completion.ts @@ -0,0 +1,121 @@ +// Tracks the app's actual take, including takes started by the person at the HUD. +// No project-folder watching: only that take's saved session and editor hand-off +// can complete it. Waiters own their deadlines, never the recording's lifetime. +import fs from "node:fs/promises"; +import { clipAwaitsProbedDuration } from "../../src/lib/ai-edition/document/timeline"; +import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; +import type { RecordedTake } from "./media-tools"; + +export type CompletedRecording = RecordedTake & { recordingStartedAt: number }; +type Outcome = { result: CompletedRecording } | { error: string }; +interface Take { + id: number; + startedAt: number; + videoPath?: string; + listeners: Set<(outcome: Outcome) => void>; +} + +export class RecordingCompletion { + private current: Take | null = null; + + isActive(): boolean { + return this.current !== null; + } + + started(id: number, startedAt = Date.now()): void { + if (this.current?.id === id) return; + if (this.current) this.failed(this.current.id, "The recording was replaced by another take."); + this.current = { id, startedAt, listeners: new Set() }; + } + + saved(id: number, videoPath: string): void { + if (this.current?.id === id) this.current.videoPath = videoPath; + } + + videoPath(id: number): string | undefined { + return this.current?.id === id ? this.current.videoPath : undefined; + } + + completed(id: number, result: RecordedTake): void { + const take = this.current; + if (!take || take.id !== id || take.videoPath !== result.videoPath) return; + this.finish(take, { result: { ...result, recordingStartedAt: take.startedAt } }); + } + + /** The editor has acknowledged its saves. Verify against the persisted project. */ + async projectReady( + id: number, + projectId: string, + readProject: (id: string) => Promise, + ): Promise { + const videoPath = this.videoPath(id); + if (!videoPath) return; + try { + const document = await readProject(projectId); + const asset = document.assets.find((asset) => asset.originalPath === videoPath); + if ( + !asset?.durationSec || + !document.timeline.clips.some((clip) => clip.assetId === asset.id) || + document.timeline.clips.some((clip) => clipAwaitsProbedDuration(clip, asset.id)) + ) { + throw new Error("The recording's project has no measured video clip."); + } + for (const filePath of [videoPath, asset.cameraTrack?.sourcePath]) { + if (!filePath) continue; + const stat = await fs.stat(filePath); + if (!stat.isFile() || stat.size === 0) + throw new Error("A recording media file is empty or missing."); + } + this.completed(id, { + projectId: document.project.id, + videoPath, + durationSec: asset.durationSec, + autoZooms: document.zoomRanges.length, + }); + } catch (error) { + this.failed(id, `The recording could not be prepared: ${String(error)}`); + } + } + + failed(id: number, error: string): void { + if (this.current?.id === id) this.finish(this.current, { error }); + } + + private finish(take: Take, outcome: Outcome): void { + this.current = null; + for (const listener of take.listeners) listener(outcome); + take.listeners.clear(); + } + + wait(timeoutMs: number, signal: AbortSignal): Promise { + const take = this.current; + if (!take) return Promise.reject(new Error("No recording is running or being saved.")); + return new Promise((resolve, reject) => { + const cleanup = () => { + clearTimeout(timer); + take.listeners.delete(finished); + signal.removeEventListener("abort", aborted); + }; + const finished = (outcome: Outcome) => { + cleanup(); + if ("error" in outcome) reject(new Error(outcome.error)); + else resolve(outcome.result); + }; + const aborted = () => + finished({ error: "Waiting for the recording was cancelled. The recording is unchanged." }); + const timer = setTimeout( + () => + finished({ + error: + "Timed out waiting for the recording. The recording is unchanged; call waitForRecording again while it is running or being saved.", + }), + timeoutMs, + ); + // Capture the target and subscribe synchronously, with no await between them. + // Completion cannot fall between a state read and listener registration. + take.listeners.add(finished); + signal.addEventListener("abort", aborted, { once: true }); + if (signal.aborted) aborted(); + }); + } +} diff --git a/electron/preload.ts b/electron/preload.ts index 23160bcaa..f018e8f18 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -236,6 +236,9 @@ contextBridge.exposeInMainWorld("electronAPI", { getRecordedVideoPath: () => { return ipcRenderer.invoke("get-recorded-video-path"); }, + reportRecordingOutcome: (outcome: import("../src/lib/recordingSession").RecordingOutcome) => + ipcRenderer.invoke("recording-outcome", outcome), + setRecordingState: ( recording: boolean, recordingId?: number, diff --git a/src/components/ai-edition/recordingImport.test.ts b/src/components/ai-edition/recordingImport.test.ts index 4b1a314a6..7f8b00403 100644 --- a/src/components/ai-edition/recordingImport.test.ts +++ b/src/components/ai-edition/recordingImport.test.ts @@ -227,6 +227,52 @@ describe("what the recording import leaves on the undo stack", () => { stubElectronApi(SCREEN_PATH); }); + it("acknowledges the exact HUD take only after its measured duration is saved", async () => { + const reportRecordingOutcome = vi.fn(async () => ({ success: true })); + Object.assign(window.electronAPI, { reportRecordingOutcome }); + await importPendingRecording(); + expect(reportRecordingOutcome).not.toHaveBeenCalled(); + const loaded = useProjectStore.getState().document as AxcutDocument; + const measured = replaceTimelineOp( + { ...loaded, assets: loaded.assets.map((a) => ({ ...a, durationSec: 12 })) }, + [{ startSec: 0, endSec: 12 }], + "Measured", + ); + let release: () => void = () => undefined; + bridge.save.mockImplementationOnce(async (document: unknown) => { + await new Promise((resolve) => { + release = resolve; + }); + return { success: true, document }; + }); + const saving = useProjectStore.getState().saveDocument(measured, { history: false }); + const ready = maybeSaveFreshRecordingAutoZooms(measured, { enabled: false }); + await Promise.resolve(); + expect(reportRecordingOutcome).not.toHaveBeenCalled(); + release(); + await saving; + await ready; + expect(reportRecordingOutcome).toHaveBeenCalledExactlyOnceWith({ + recordingId: 0, + status: "ready", + projectId: PROJECT_ID, + }); + await maybeSaveFreshRecordingAutoZooms(measured); + expect(reportRecordingOutcome).toHaveBeenCalledTimes(1); + }); + + it("reports an import failure against the originating take", async () => { + const reportRecordingOutcome = vi.fn(async () => ({ success: true })); + Object.assign(window.electronAPI, { reportRecordingOutcome }); + bridge.create.mockRejectedValueOnce(new Error("disk full")); + await expect(importPendingRecording()).rejects.toThrow("disk full"); + expect(reportRecordingOutcome).toHaveBeenCalledExactlyOnceWith({ + recordingId: 0, + status: "failed", + message: expect.stringContaining("disk full"), + }); + }); + it("leaves it empty: the user has not edited anything yet", async () => { await importPendingRecording(); diff --git a/src/components/ai-edition/recordingImport.ts b/src/components/ai-edition/recordingImport.ts index b6b3e119a..69b3f421c 100644 --- a/src/components/ai-edition/recordingImport.ts +++ b/src/components/ai-edition/recordingImport.ts @@ -28,6 +28,7 @@ import { appendAutoZoomSuggestions, collectAutoZoomSuggestionsForLatestDocument, } from "@/lib/ai-edition/timeline/apply-auto-zooms"; +import type { RecordingSession } from "@/lib/recordingSession"; import { nativeBridgeClient } from "@/native/client"; // Fresh recordings used to get cursor-dwell zooms on load (legacy editor @@ -48,6 +49,40 @@ let pendingFreshRecordingAutoZoomPath: string | null = null; let appliedFreshRecordingAutoZoomProjectId: string | null = null; let freshRecordingAutoZoomSaveChain: Promise = Promise.resolve(); +// Kept only for the HUD hand-off, never for ordinary media imports or reopened +// projects. Metadata may arrive after importPendingRecording has returned. +let pendingCompletion: { session: RecordingSession; projectId: string | null } | null = null; + +async function reportReadyRecording(): Promise { + const pending = pendingCompletion; + const api = window.electronAPI; + if (!pending || !api?.reportRecordingOutcome) return; + if ((await waitForDocumentSaves()) === "timeout") return; + const state = useProjectStore.getState(); + const document = state.document; + if (!document || state.dirty || document.project.id !== pending.projectId) return; + const asset = document.assets.find( + (asset) => asset.originalPath === pending.session.screenVideoPath, + ); + if ( + !asset?.durationSec || + !document.timeline.clips.some((clip) => clip.assetId === asset.id) || + document.timeline.clips.some((clip) => clipAwaitsProbedDuration(clip, asset.id)) || + pendingFreshRecordingAutoZoom + ) + return; + try { + await api.reportRecordingOutcome({ + recordingId: pending.session.createdAt, + status: "ready", + projectId: document.project.id, + }); + if (pendingCompletion === pending) pendingCompletion = null; + } catch (error) { + console.warn("[recording] could not report saved project:", error); + } +} + export function markFreshRecordingAutoZoomPending(assetPath: string): void { pendingFreshRecordingAutoZoom = true; pendingFreshRecordingAutoZoomPath = assetPath; @@ -311,7 +346,10 @@ export async function maybeSaveFreshRecordingAutoZooms( () => undefined, () => undefined, ); - return done; + return done.then(async (saved) => { + await reportReadyRecording(); + return saved; + }); } /** @@ -336,53 +374,74 @@ export async function importPendingRecording( const result = await api.getCurrentRecordingSession(); const screenPath = result.success ? result.session?.screenVideoPath : undefined; if (!screenPath) return false; - const cursorCaptureMode = result.success ? result.session?.cursorCaptureMode : undefined; - const warning = result.success ? result.warning : undefined; - - const label = screenPath.split(/[\\/]/).pop() || "Recording"; - await useProjectStore.getState().createProject(`Recording ${new Date().toLocaleString()}`); - await useProjectStore.getState().addAsset(screenPath, label); - // Mark before the video element can fire `loadedmetadata`. The asset path is - // already on the document; waiting until the 60s seed finished let the first - // metadata pass consume nothing and the second never arrive. - // - // Except for a system-cursor take, which writes no `.cursor.json` at all: the - // toggle stays on in prefs (it is only disabled in the UI while that mode is - // picked), so without this the flag is set for a recording that can never produce - // a click. What governs is the mode THIS take was recorded in, not the current - // preference. - if (cursorCaptureMode !== "system") { - markFreshRecordingAutoZoomPending(screenPath); - } - // Consumed: the recording now lives in a project. Cleared here rather than - // after the timeline seed below so a failure down there can't hand the same - // recording to the next editor window. - await api.setCurrentRecordingSession(null); + const pending: typeof pendingCompletion = result.session + ? { session: result.session, projectId: null } + : null; + pendingCompletion = pending; + try { + const cursorCaptureMode = result.success ? result.session?.cursorCaptureMode : undefined; + const warning = result.success ? result.warning : undefined; - // ponytail: MediaRecorder WebMs ship with duration = NaN until - // fix-webm-duration patches the EBML header; until that flows through the - // asset, drop a default 60s clip into the timeline so the editor isn't stuck - // on "No clips yet" the moment the user lands in the project. Real duration - // overwrites this when handleLoadedMetadata fires with a finite value. - const doc = useProjectStore.getState().document; - if (doc && doc.timeline.clips.length === 0 && doc.assets.length > 0) { - // `history: false`. Nothing here is an edit: the user finished a recording and the - // editor built them a project around it, unattended, on mount. Recording it left a - // brand-new project sitting at `past.length === 1` before the user had touched - // anything, so their FIRST Ctrl+Z restored the state before the seed -- an empty - // timeline -- and the persist that follows an undo wrote that empty timeline to disk. - await useProjectStore + const label = screenPath.split(/[\\/]/).pop() || "Recording"; + const created = await useProjectStore .getState() - .replaceTimeline([{ startSec: 0, endSec: 60 }], "Auto-imported recording", { - history: false, - }); - } - const latest = useProjectStore.getState().document; - if (latest) { - await maybeSaveFreshRecordingAutoZooms(latest); - } - if (warning) { - onWarning?.(warning); + .createProject(`Recording ${new Date().toLocaleString()}`); + if (pending) pending.projectId = created.project.id; + await useProjectStore.getState().addAsset(screenPath, label); + // Mark before the video element can fire `loadedmetadata`. The asset path is + // already on the document; waiting until the 60s seed finished let the first + // metadata pass consume nothing and the second never arrive. + // + // Except for a system-cursor take, which writes no `.cursor.json` at all: the + // toggle stays on in prefs (it is only disabled in the UI while that mode is + // picked), so without this the flag is set for a recording that can never produce + // a click. What governs is the mode THIS take was recorded in, not the current + // preference. + if (cursorCaptureMode !== "system") { + markFreshRecordingAutoZoomPending(screenPath); + } + // Consumed: the recording now lives in a project. Cleared here rather than + // after the timeline seed below so a failure down there can't hand the same + // recording to the next editor window. + await api.setCurrentRecordingSession(null); + + // ponytail: MediaRecorder WebMs ship with duration = NaN until + // fix-webm-duration patches the EBML header; until that flows through the + // asset, drop a default 60s clip into the timeline so the editor isn't stuck + // on "No clips yet" the moment the user lands in the project. Real duration + // overwrites this when handleLoadedMetadata fires with a finite value. + const doc = useProjectStore.getState().document; + if (doc && doc.timeline.clips.length === 0 && doc.assets.length > 0) { + // `history: false`. Nothing here is an edit: the user finished a recording and the + // editor built them a project around it, unattended, on mount. Recording it left a + // brand-new project sitting at `past.length === 1` before the user had touched + // anything, so their FIRST Ctrl+Z restored the state before the seed -- an empty + // timeline -- and the persist that follows an undo wrote that empty timeline to disk. + await useProjectStore + .getState() + .replaceTimeline([{ startSec: 0, endSec: 60 }], "Auto-imported recording", { + history: false, + }); + } + const latest = useProjectStore.getState().document; + if (latest) { + await maybeSaveFreshRecordingAutoZooms(latest); + } + if (warning) { + onWarning?.(warning); + } + return true; + } catch (error) { + if (pendingCompletion === pending) pendingCompletion = null; + if (pending) { + await api + .reportRecordingOutcome?.({ + recordingId: pending.session.createdAt, + status: "failed", + message: `The recording's project could not be prepared: ${String(error)}`, + }) + .catch(() => undefined); + } + throw error; } - return true; } diff --git a/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx b/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx index 928ac5886..df1f4ff3a 100644 --- a/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx +++ b/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx @@ -25,6 +25,7 @@ let api: Record>; */ function stubElectronAPI(overrides: Record = {}) { api = { + reportRecordingOutcome: vi.fn(async () => ({ success: true })), getRecordingPrefs: vi.fn(async () => null), getPlatform: vi.fn(() => "win32"), getSelectedSource: vi.fn(async () => SOURCE), @@ -104,6 +105,9 @@ describe("useScreenRecorder native Windows stop failure", () => { expect(view.result.current.saving).toBe(false); // The failure must not be mistaken for a successful recording. expect(api.switchToEditor).not.toHaveBeenCalled(); + expect(api.reportRecordingOutcome).toHaveBeenCalledWith( + expect.objectContaining({ recordingId: 7, status: "failed" }), + ); // The next press starts a NEW recording instead of re-sending a stop. api.stopNativeWindowsRecording.mockClear(); @@ -122,6 +126,21 @@ describe("useScreenRecorder native Windows stop failure", () => { expect(view.result.current.recording).toBe(false); expect(toast.error).toHaveBeenCalledWith("IPC channel closed"); + expect(api.reportRecordingOutcome).toHaveBeenCalledWith( + expect.objectContaining({ recordingId: 7, status: "failed" }), + ); + }); + + it("reports a discarded take as cancelled, never as a completed project", async () => { + api.stopNativeWindowsRecording.mockResolvedValue({ success: true, discarded: true }); + const view = renderHook(() => useScreenRecorder()); + await startNativeRecording(view); + await act(async () => view.result.current.cancelRecording()); + await settle(); + expect(api.reportRecordingOutcome).toHaveBeenCalledWith( + expect.objectContaining({ recordingId: 7, status: "cancelled" }), + ); + expect(api.switchToEditor).not.toHaveBeenCalled(); }); it("still opens the editor when the stop succeeds", async () => { @@ -135,6 +154,7 @@ describe("useScreenRecorder native Windows stop failure", () => { await pressStop(view); expect(api.switchToEditor).toHaveBeenCalled(); + expect(api.reportRecordingOutcome).not.toHaveBeenCalled(); expect(view.result.current.recording).toBe(false); }); }); diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index c5156749b..48e7afa37 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -234,6 +234,20 @@ export async function finalizeWebcamAsset( } } +async function reportUnfinishedRecording(recordingId: number, cancelled: boolean) { + try { + await window.electronAPI?.reportRecordingOutcome?.({ + recordingId, + status: cancelled ? "cancelled" : "failed", + message: cancelled + ? "The recording was cancelled or discarded." + : "The recording could not be saved and opened. The recording bar says why.", + }); + } catch (error) { + console.warn("Failed to report recording outcome:", error); + } +} + export function useScreenRecorder(): UseScreenRecorderReturn { const t = useScopedT("editor"); const tLaunch = useScopedT("launch"); @@ -602,6 +616,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // store-recorded-session finalizes the streams included in a successful // save; the finally block discards everything else. let storeSucceeded = false; + let handedOver = false; let webcamIncludedInSave = false; try { const screenBlob = await activeScreenRecorder.recordedBlobPromise; @@ -674,9 +689,15 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } await window.electronAPI.switchToEditor(); + handedOver = true; } catch (error) { console.error("Error saving recording:", error); } finally { + if (!handedOver) + await reportUnfinishedRecording( + activeRecordingId, + discardRecordingId.current === activeRecordingId, + ); // Discard any recorder whose data wasn't part of a successful save (discarded // run, failed save, or a webcam whose disk write failed while the screen still // saved) so no stream or partial file is left open or orphaned. @@ -719,6 +740,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { segmentStartedAt.current = null; }; + let handedOver = false; try { const result = await window.electronAPI.stopNativeWindowsRecording(discard); if (discard || result.discarded) { @@ -758,6 +780,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } await window.electronAPI.switchToEditor(); + handedOver = true; return true; } catch (error) { console.error("Error saving native Windows recording:", error); @@ -767,6 +790,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { clearNativeRecordingState(); return true; } finally { + if (!handedOver) await reportUnfinishedRecording(activeNativeRecording.recordingId, discard); if (discardRecordingId.current === activeNativeRecording.recordingId) { discardRecordingId.current = null; } @@ -818,6 +842,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }; let webcamSaved = false; + let handedOver = false; try { const result = await window.electronAPI.stopNativeMacRecording(discard); const webcamResult = await webcamResultPromise; @@ -868,6 +893,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } await window.electronAPI.switchToEditor(); + handedOver = true; return true; } catch (error) { console.error("Error saving native macOS recording:", error); @@ -877,6 +903,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { clearNativeRecordingState(); return true; } finally { + if (!handedOver) + await reportUnfinishedRecording(activeNativeRecording.recordingId, discard); // A webcam stream that wasn't folded into a saved session has to be closed // and its partial file removed, or a discarded or failed take orphans a // half-written .webm now that the bytes go to disk as they arrive. @@ -936,6 +964,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }; let webcamSaved = false; + let handedOver = false; try { const result = await window.electronAPI.stopNativeLinuxRecording(discard); const webcamResult = await webcamResultPromise; @@ -986,6 +1015,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } await window.electronAPI.switchToEditor(); + handedOver = true; return true; } catch (error) { console.error("Error saving native Linux recording:", error); @@ -995,6 +1025,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { clearNativeRecordingState(); return true; } finally { + if (!handedOver) + await reportUnfinishedRecording(activeNativeRecording.recordingId, discard); // A webcam stream that wasn't folded into a saved session has to be closed // and its partial file removed, or a discarded or failed take orphans a // half-written .webm now that the bytes go to disk as they arrive. diff --git a/src/lib/recordingSession.ts b/src/lib/recordingSession.ts index 5fd06bc25..413ba9cbf 100644 --- a/src/lib/recordingSession.ts +++ b/src/lib/recordingSession.ts @@ -99,3 +99,8 @@ export function normalizeRecordingSession(candidate: unknown): RecordingSession : Date.now(), }; } + +/** Internal recorder/editor hand-off, never an MCP server notification. */ +export type RecordingOutcome = + | { recordingId: number; status: "ready"; projectId: string } + | { recordingId: number; status: "failed" | "cancelled"; message: string }; diff --git a/technical-documentation/architecture/mcp-server.md b/technical-documentation/architecture/mcp-server.md index e9ac92564..0df4c0853 100644 --- a/technical-documentation/architecture/mcp-server.md +++ b/technical-documentation/architecture/mcp-server.md @@ -12,7 +12,7 @@ The same tools are served over two transports, by two different processes: | Edits, recording, export | Off until the user turns on **MCP edits** | Always allowed | | Project open in the editor | Edited live through the editor, one undo step per call | The same while the app runs: the call is forwarded to it. With no app answering, refused ([the marker](#the-open-in-editor-marker)) | | Other projects | Read from and saved to their file, a checkpoint first | Same | -| Tools | `listProjects`, the agent's tools, `transcribe`, the project tools (`setProjectInfo`, `importMedia`, `removeMedia`), the checkpoint tools, and the media tools: `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames`, `createProject` | The same | +| Tools | `listProjects`, the agent's tools, `transcribe`, the project tools (`setProjectInfo`, `importMedia`, `removeMedia`), the checkpoint tools, and the media tools: `listSources`, `startRecording`, `stopRecording`, `waitForRecording`, `setPointerTrack`, `exportVideo`, `getFrames`, `createProject` | The same | | A take | Recorded by the app's recording bar, as its Record button records; the editor opens it ([below](#the-media-tools)) | Recorded by a hidden runner of its own process, and made a project there | | Checkpoints | In the app's memory, shared with forwarded `openscreen mcp` calls, lost when the app quits | In the app's while it runs; otherwise in this process's, lost when it exits, which it does when the client disconnects | @@ -157,7 +157,7 @@ Three tools of the runner's own ([`project-tools.ts`](../../electron/mcp/project ## The media tools -`createMediaTools` builds the tools that make a video around the edits, once per server, and `createOpenScreenMcpServer` registers them. Both servers serve the same seven, from the same code. Their state (the take running, the export running, each take's start time, each project's last export) lives in what `createMediaTools` returns, so the in-app server, which builds a fresh `McpServer` per request, keeps it across requests and across a restart of its port. They share the runner the document tools use: the app's one runner in the app, the routed one of `stdio-server.ts` over stdio, so their document calls go to the app while it runs. +`createMediaTools` builds the tools that make a video around the edits, once per server, and `createOpenScreenMcpServer` registers them. Both servers register the same tools, from the same code; `waitForRecording` observes the GUI only on the in-app HTTP server. Over stdio it returns an unsupported-tool error explaining which server to connect to. Their state (the take running, the export running, each take's start time, each project's last export) lives in what `createMediaTools` returns, so the in-app server, which builds a fresh `McpServer` per request, keeps it across requests and across a restart of its port. They share the runner the document tools use: the app's one runner in the app, the routed one of `stdio-server.ts` over stdio, so their document calls go to the app while it runs. What differs between the two servers is who records, handed in as a `TakeRecorder`: @@ -168,6 +168,40 @@ What differs between the two servers is who records, handed in as a `TakeRecorde | `stopRecording` | Presses the HUD's Stop, as the tray does. The HUD saves the take and hands it to the editor (`set-current-recording-session`), which imports it as for any take: one clip, measured, and the zooms on its clicks when Record mode's *Auto-zoom after recording* is on. The recorder waits for the editor's document to hold the take with its real length, then for its revision to hold still for 1.5 s, and returns that project, open in the editor. A `title` is applied through the editor. | Stops the runner and makes the project here (`createProjectFromVideo`): a fresh id no editor can hold, the zooms planned from the clicks, and the webcam, when one was recorded, linked to the take's asset as the editor links it (its offset and size). | | The user's settings | The source, microphone, camera, system audio and cursor mode the user had (`TAKE_PREF_KEYS`) come back once the take is captured, or as the app quits mid-take. A request that fails before the take changes none of them. | Untouched: the runner never saves them. | +### Waiting for a user-controlled take + +On the **in-app HTTP server**, `waitForRecording({ timeoutSec: 300 })` observes the +current take, whether the user or `startRecording` began it. It never starts or stops +capture. `timeoutSec` is an integer from 1 to 3600, default 300; configure the client's +own tool-call timeout to be longer. The server's existing bearer token, Host and Origin +checks apply. Observation is read-only and works with **MCP edits** off. + +- With no active take (recording or saving), the call immediately returns `isError`. + It does not return the last recording or wait for the next one. Call it after the + HUD starts recording; the countdown is not an active take. +- `RecordingCompletion` (`electron/mcp/recording-completion.ts`) binds each wait to + the recording ID reported by the capture path. A new take, or Cancel/Restart at + the HUD, fails waits on the old one. An imported project cannot finish a wait. +- Capture finalizes the files and cursor sidecar before publishing the recording + session. The editor then acknowledges that specific session's project after its + real duration and automatic-zoom writes have settled. The main process reads the + saved project and verifies its measured clip and nonempty screen/camera files. + Unlike `stopRecording`'s quiet-period check, this acknowledgement is explicit. +- Success uses `stopRecording`'s result builder: `projectId`, `checkpointId` (null + if checkpoint creation fails), `videoPath`, `durationSec`, `recordingStartedAt`, + `autoZooms`, and optional `warnings`. It does not wait for transcription or export. +- Save/import failures and discarded takes return tool errors. A missing duration + or unfinished auto-zoom save cannot produce a success; the wait can time out. +- Timeout, MCP cancellation, HTTP disconnect, and server shutdown remove only that + wait's timer, listener and abort subscription. They leave the recording running. + Several clients may wait independently on the same take, each with its own timeout. + A timed-out caller can retry while that take is still recording or saving. + +The completed take releases the app recorder's claim. A client that used +`startRecording` and then waited for the user's stop can start another take without +an extra `stopRecording` call. The acknowledgement is an internal renderer-to-main +IPC message, not an MCP notification or a command hook. + Then both servers carry on alike: - `stopRecording` creates the take's checkpoint through the runner, and records when the take started, by its video, for `setPointerTrack`. @@ -228,7 +262,7 @@ codex mcp add openscreen -- "" mcp ## Known gaps - **`openscreen mcp` records in its own process, app or no app.** Only its document calls are forwarded; its takes, exports and frames stay in the CLI process, and its take lands in the project list rather than in the editor. The in-app server is the way to record through the app's recording bar. -- **An in-app take is ready by a quiet period.** The editor lays a take out in several saves and says none of them is the last: the recorder waits for its revision to hold for 1.5 s. A save later than that (a slow auto-zoom pass) lands after the checkpoint, so `restoreCheckpoint` takes it back with the client's edits. +- **`stopRecording` considers an in-app take ready by a quiet period.** The editor lays a take out in several saves and says none of them is the last: the recorder waits for its revision to hold for 1.5 s. A save later than that (a slow auto-zoom pass) lands after the checkpoint, so `restoreCheckpoint` takes it back with the client's edits. - **No Ctrl+Z for edits to closed projects.** Their undo is the checkpoint the write requires, which lives in memory: it is gone once its server goes away or 20 newer ones push it out, and the edits stay. - **An editor that stops answering looks closed.** A snapshot unanswered for 30 s reads as "no project open", so an edit naming the project that editor holds would go to its file, and the editor's next save would drop it. It takes a renderer hung for 30 s. - **No project deletion**, on purpose ([above](#the-project-and-its-media)). diff --git a/technical-documentation/architecture/recording.md b/technical-documentation/architecture/recording.md index 2f643ec22..a07060b5d 100644 --- a/technical-documentation/architecture/recording.md +++ b/technical-documentation/architecture/recording.md @@ -16,6 +16,13 @@ flowchart LR The HUD starts and controls a recording. The source selector chooses a display or window and the countdown gives the user time to prepare. On Linux there is no in-app source selector: the compositor's portal picker takes its place. It is raised before the countdown, not during it — the helper negotiates the portal, reports `source-selected`, and then waits for `record` on stdin while the app counts down. Electron resolves the source and output paths, starts the selected capture backend, and records cursor telemetry alongside media. Stop finalizes the media and session files; the resulting paths are passed to the editor as recording assets. +The in-app MCP server can observe this lifecycle through +[`waitForRecording`](mcp-server.md#waiting-for-a-user-controlled-take). Its target is +the current recording ID, not the current project or a newly created file. The +recorder reports failure/discard explicitly; the editor acknowledges completion +after the project's duration and automatic-zoom saves. Waiting never controls +capture, and timing out or cancelling a request leaves the take running. + ## The HUD `LaunchWindow` is the recording controller. Its control row exposes recording, pause, source, microphone, system-audio, webcam, cursor, settings, and stop/restart actions through the HUD controls. The overlay is click-through, so it does not intercept the application being recorded — but it is never *born* that way: the renderer asks for it on mount over `hud-overlay-ignore-mouse-events`, and `electron/windows.ts` applies it as `setIgnoreMouseEvents(ignore)`, with no `{ forward: true }`. While the window is input-transparent Chromium delivers it no pointer events at all, so the way back out is a poll: the main process samples the OS cursor and pushes the window-relative point over `hud-overlay-cursor`, and the renderer hit-tests it to decide when the HUD's own controls temporarily opt into pointer handling. On Linux none of this runs: `setIgnoreMouseEvents` is a no-op there, so the renderer never asks for click-through and the HUD stays interactive — which is why its drag region sits on the grab handle rather than on the transparent root. (Until #385 the lift was Electron's `{ forward: true }` — a global `WH_MOUSE_LL` hook on Windows that Windows can revoke without telling the app, leaving the HUD painted and permanently dead. The poll replaced it.) diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 340df1480..fca138d0c 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -580,6 +580,7 @@ Two servers. The **in-app** one is HTTP, in *AI settings* → *MCP server*, off - [ ] In *AI settings*, confirm *Enable MCP server* and *Allow MCP clients to edit your projects, record and export* are off by default; enable the server and confirm the status reads *Running* with a *Port*, an *Access token* (Show, Copy, Regenerate) and the Claude Code and Codex snippets. Change the port, *Apply*, and confirm a client reconnects on it. - [ ] With edits not allowed, confirm an edit tool, `startRecording` and `exportVideo` are refused, naming *MCP edits*; allow them and confirm the edit lands in the open editor and undoes with `Ctrl/Cmd+Z`. +- [ ] On the **in-app HTTP MCP server**, start a take from the HUD, call `waitForRecording`, then stop it from the HUD. Verify that the returned `projectId` is the saved project in the editor and `videoPath` is its completed media. Repeat with `timeoutSec: 1` while recording: the wait returns a tool error and the HUD keeps recording; a second wait can receive that same take after the user stops it. While idle the tool must immediately return a tool error. Cancel/Restart must fail waits on the discarded take; test an MCP request cancellation independently. This tool does not observe the HUD through the stdio server. - [ ] From a client of the **in-app** server, `startRecording` on a window: the editor closes and the recording bar records, its countdown first, with no second recorder process. Click in the window, `stopRecording`: the editor opens on the take, laid out and zoomed as after a take started by hand, and the returned `projectId` is that project. An edit to it lands live and undoes with `Ctrl/Cmd+Z`; `exportVideo` to an absolute path and `getFrames` on it work. Confirm the recording bar's microphone, camera, system audio and cursor mode are back to what they were before the take, and that `startRecording` is refused while the editor has unsaved changes. - [ ] Confirm `"" help` lists `mcp`, register the stdio server in an MCP client, and confirm `listSources` lists displays, windows, microphones and cameras. - [ ] Run `startRecording` on a window, click inside it a few times, `stopRecording`, and confirm the returned project is in the editor's project list with one clip and the zooms its `autoZooms` count announces. Clicks close in time (3 s or less) and on screen share one zoom, and a click in the first seconds or the stopping click gets none, so there can be fewer zooms than clicks. On macOS 15.2+, confirm Apple's picker never appears; on Linux, confirm the portal asks. @@ -994,3 +995,5 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-17 | PR #1319 (`2236580`), artifact `openscreen-linux` of build run 39046512213, AppImage run directly. Reference: the `v2.1.0` AppImage from the GitHub release | Ubuntu 24.04.4, GNOME 46 Wayland, AMD Radeon 610M (RADV); `[d3d] … -> backend Hardware`, no CPU fallback | **Partial — fix slice only, no defect** | Same MCP pass and project as the rows above, on the hardware Vulkan backend. `orbit` (12 frames) and `left` (4 s, 6 s): the whole card with four rounded corners, no flat cut. `v2.1.0` reproduces the bug (top and bottom edges flat, corners cut from 1.5 s). `none` and the flat frames at 1.0 and 9.5 s are byte-identical to `v2.1.0` (equal md5). **Not covered:** the editor preview; `right`; a full MP4 export. | | 2026-10-17 | `v2.1.1-rc.1`, `release/v2.1.1` @ `6ec38bf` (= `v2.1.0` + #1319), assets of the GitHub pre-release (build run 39069024444). Windows: `Openscreen.Setup.2.1.1-rc.1-x64.exe`, sha512 matched `latest.yml`, unpacked with 7-Zip and run in place, version 2.1.1. macOS: `Openscreen-macOS-Apple-Silicon-2.1.1-rc.1.dmg`, sha256 matched the asset digest; `spctl` accepted, Notarized Developer ID, on the DMG and the app; ticket stapled to the DMG only, as on 2.1.0-rc.6; `CFBundleShortVersionString` 2.1.1-rc.1. Linux: `Openscreen-Linux-2.1.1-rc.1.AppImage`, sha512 matched `latest-linux.yml` | Windows 11 Home 26200, RTX 4070 Ti; macOS 27.0.1 (26A434), Mac mini M1 8 GB; Ubuntu 24.04.4, GNOME 46 Wayland, Radeon 610M, hardware Vulkan backend | **Partial — patch pass toward stable: no defect on any of the three** | Scoped to what the patch changes (the compositor geometry of a tilted filled card), in MCP over stdio with an isolated `--user-data-dir`, on the project of the PR rows above. On all three: `orbit` (12 frames) and `left` show the whole card with four rounded corners and no flat cut; `none` is a flat zoom in a rounded card (byte-identical to `v2.1.0` on Windows and Linux; on Windows every checked frame is byte-identical to the PR build). `exportVideo` at 1080p30: 10.00 s, 1920×1080 H.264 with an AAC track, decodes without error, and its frame at 5 s shows the tilted card with rounded corners. Before the cut, on a dev build of the PR: the four device frames, dual-frame and vertical-stack, under `orbit`, `left` and `none`, rendered byte-identical to `v2.1.0` (49 of 49 frames). **Not covered:** recording, playback and the editor preview, which the patch does not touch; the rest of the checklist (the `v2.1.0-rc.6` rows). | | 2026-10-19 | `v2.1.2-rc.1`, `release/v2.1.2` @ `c4ef8fd` (= `v2.1.1` + #1344), assets of the GitHub pre-release (build run 39224311283). Windows: `Openscreen.Setup.2.1.2-rc.1-x64.exe` unpacked with 7-Zip and run in place. macOS: `Openscreen-Mac-arm64-2.1.2-rc.1.zip`, run in place. Linux: the AppImage, run directly with `--no-sandbox` | Windows 11 Home 26200; macOS 27.0.1, Mac mini M1; Ubuntu 24.04 GNOME Wayland (Xwayland `:0`) | **Partial: fix slice only, no defect** | `openscreen mcp` over stdio, scripted, with a fresh `--user-data-dir` per run. On all three platforms it answered `initialize`, and `usage-ping.json` held today's day, which is saved only once PostHog accepted the event. With `DO_NOT_TRACK=1` no state file was written. stdout held only JSON-RPC on macOS and Linux. On Windows it starts with one `\r\n` that Electron 41 itself writes during browser-process startup, before `main.js` loads: a bare `app.exit(0)` script prints it too, `ELECTRON_RUN_AS_NODE` does not. It predates this change, and the MCP SDK clients skip it with one parse error. Not covered: everything else in this file, including recording, editing, export, the GUI ping and the tray toggle. These runs sent eight real `surface: mcp` events to PostHog on 19/10. | + +| 2026-10-22 | #1188 local worktree, `feat/wait-for-recording-1188`, based on `main` @ `d8bf2d81`, development build (Electron 41.2.1) | macOS 15.8 arm64 | **Partial — HTTP idle behavior passed; real capture blocked** | Launched this worktree with Vite port 5188, MCP port 47888, CDP port 9388 and isolated userData `/tmp/os1188-3cvs73hn/profile`; `OPENSCREEN_DISABLE_CONTENT_PROTECTION=1`, `OPENSCREEN_MAC_SOURCE_PICKER=legacy`, `DO_NOT_TRACK=1`. Computer-use selected Screen 1 from the HUD and clicked Start recording. The app displayed **Native macOS capture helper is not available.** The full borrowed native library directory has the compositor and ffmpeg but no capture helper; `npm run build:native:mac` cannot build it because only Command Line Tools is active, not full Xcode. A real authenticated HTTP `waitForRecording` call returned HTTP 200 with `isError: true` and **No recording is running or being saved.** in 679 ms. **Skipped:** successful HUD recording → wait → manual stop → returned recording, and timeout while a real recording keeps running; no real recording began. Unit/SDK/HTTP fixture tests are separate evidence and do not count as real capture. Editor, export, webcam, other platforms and the rest of this checklist were not exercised. Quit only this isolated app and stopped only this worktree's Vite process. |