Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions electron/electron-env.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ interface Window {
meta: import("../src/native/contracts").CompositorSharedFrameMeta,
) => void,
) => () => void;
reportRecordingOutcome?: (
outcome: import("../src/lib/recordingSession").RecordingOutcome,
) => Promise<{ success: boolean }>;

getSources: (opts: Electron.SourcesOptions) => Promise<ProcessedDesktopSource[]>;
switchToEditor: () => Promise<void>;
switchToHud: () => Promise<void>;
Expand Down
56 changes: 47 additions & 9 deletions electron/ipc/handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
shell,
systemPreferences,
} from "electron";
import { z } from "zod";
import { DEFAULT_WEBCAM_QUALITY, type WebcamQualityId } from "../../src/hooks/webcamCaptureTarget";
import {
type AxcutDocument,
Expand Down Expand Up @@ -88,6 +89,7 @@ import { McpController } from "../mcp/mcp-controller";
import { McpSettingsStore } from "../mcp/mcp-settings-store";
import { createMediaTools, type MediaToolDeps, type RecordRequest } from "../mcp/media-tools";
import { createToolRunner, type McpToolDeps } from "../mcp/openscreen-mcp-server";
import { RecordingCompletion } from "../mcp/recording-completion";
import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks";
import {
readCursorRecordingFile as readCursorRecordingFileFrom,
Expand Down Expand Up @@ -1966,15 +1968,17 @@ export function registerIpcHandlers(
) {
/** The HUD is recording a take, the user's or an MCP client's. Set by `setAppRecording` only. */
let appRecording = false;
const recordingCompletion = new RecordingCompletion();
/**
* Every start and stop of a take goes through here: the browser path's `set-recording-state`
* and the native paths, which start and stop in this process. The MCP recorder waits on it
* for a client's take to start, and the tray shows it. A native path that called
* `onRecordingStateChange` itself left the MCP recorder waiting on a take already running.
*/
const setAppRecording = (recording: boolean, sourceName: string) => {
const setAppRecording = (recording: boolean, sourceName: string, recordingId?: number) => {
if (recording && recordingId !== undefined) recordingCompletion.started(recordingId);
appRecording = recording;
mcpAppRecorder.recordingChanged(recording);
mcpAppRecorder.recordingChanged(recording, recordingId);
onRecordingStateChange?.(recording, sourceName);
};
const appSettings = new AppSettingsStore(app.getPath("userData"));
Expand Down Expand Up @@ -2760,7 +2764,7 @@ export function registerIpcHandlers(
// "Screen" }`, so the tray confidently displayed the name of a
// window the capture had never been told about.
linuxNativeCaptureSourceLabel = linuxSourceLabel(session.grantedSourceKind);
setAppRecording(true, linuxNativeCaptureSourceLabel);
setAppRecording(true, linuxNativeCaptureSourceLabel, recordingId);

return { success: true, recordingId, path: outputPath };
} catch (error) {
Expand Down Expand Up @@ -3056,7 +3060,7 @@ export function registerIpcHandlers(
});

const source = selectedSource || { name: "Screen" };
setAppRecording(true, source.name);
setAppRecording(true, source.name, recordingId);

// Reported at start, not at stop: the helper decides the camera is a
// lost cause during its own init — before it announces "Recording
Expand Down Expand Up @@ -3318,7 +3322,7 @@ export function registerIpcHandlers(
: 0;

const source = selectedSource || { name: "Screen" };
setAppRecording(true, source.name);
setAppRecording(true, source.name, recordingId);

return {
success: true,
Expand Down Expand Up @@ -4113,16 +4117,15 @@ export function registerIpcHandlers(
ipcMain.handle(
"set-recording-state",
async (_, recording: boolean, recordingId?: number, cursorCaptureMode?: CursorCaptureMode) => {
const source = selectedSource || { name: "Screen" };
setAppRecording(recording, source.name, recordingId);
const normalizedCursorCaptureMode =
normalizeCursorCaptureMode(cursorCaptureMode) ?? "editable-overlay";
if (recording && normalizedCursorCaptureMode === "editable-overlay") {
await startCursorRecording(recordingId);
} else {
await stopCursorRecording();
}

const source = selectedSource || { name: "Screen" };
setAppRecording(recording, source.name);
},
);

Expand Down Expand Up @@ -4864,10 +4867,43 @@ export function registerIpcHandlers(
setCurrentRecordingSessionState(normalizedSession);
currentVideoPath = normalizedSession?.screenVideoPath ?? null;
currentProjectPath = null;
if (normalizedSession) mcpAppRecorder.takeSaved(normalizedSession.screenVideoPath);
if (normalizedSession) {
mcpAppRecorder.takeSaved(normalizedSession.screenVideoPath);
recordingCompletion.saved(normalizedSession.createdAt, normalizedSession.screenVideoPath);
}
return { success: true, session: currentRecordingSession };
});

// A renderer acknowledgement of the actual hand-off, after the duration and
// automatic zoom saves. An imported/opened project never emits this signal.
const recordingOutcomeSchema = z.discriminatedUnion("status", [
z.object({
recordingId: z.number().finite(),
status: z.literal("ready"),
projectId: z.string(),
}),
z.object({
recordingId: z.number().finite(),
status: z.enum(["failed", "cancelled"]),
message: z.string(),
}),
]);
ipcMain.handle("recording-outcome", async (_, input: unknown) => {
const parsed = recordingOutcomeSchema.safeParse(input);
if (!parsed.success) return { success: false };
const outcome = parsed.data;
if (outcome.status !== "ready") {
recordingCompletion.failed(outcome.recordingId, outcome.message);
mcpAppRecorder.recordingFinished(outcome.recordingId);
return { success: true };
}
await recordingCompletion.projectReady(outcome.recordingId, outcome.projectId, (id) =>
aiEditionDocuments.getProject(id),
);
mcpAppRecorder.recordingFinished(outcome.recordingId);
return { success: true };
});

ipcMain.handle("get-current-recording-session", () => {
if (!currentRecordingSession) {
return { success: false };
Expand Down Expand Up @@ -5151,6 +5187,8 @@ export function registerIpcHandlers(
host: mcpToolDeps.host,
});
const mcpMedia = createMediaTools({
recordingActive: () => recordingCompletion.isActive(),
waitForRecording: (timeoutMs, signal) => recordingCompletion.wait(timeoutMs, signal),
transport: "http",
// Recording and exporting change things too: the same switch as edits.
editsAllowed: mcpToolDeps.editsAllowed,
Expand Down
2 changes: 1 addition & 1 deletion electron/ipc/recordingStateRouting.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ describe("recording state routing", () => {
const setter = source.slice(source.indexOf("const setAppRecording"));
const body = setter.slice(0, setter.indexOf("};"));
expect(body).toContain("appRecording = recording");
expect(body).toContain("mcpAppRecorder.recordingChanged(recording)");
expect(body).toContain("mcpAppRecorder.recordingChanged(recording, recordingId)");
expect(body).toContain("onRecordingStateChange?.(recording, sourceName)");
});

Expand Down
14 changes: 14 additions & 0 deletions electron/mcp/app-recorder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,20 @@ describe("an MCP take in the app", () => {
expect(editor.applied).toEqual([]);
});

it("releases only the client take whose app lifecycle finished", async () => {
const { recorder, calls } = setup();
const starting = recorder.start(REQUEST);
await new Promise((resolve) => setTimeout(resolve, 5));
recorder.recordingChanged(true, 7);
await starting;
recorder.recordingFinished(6);
await expect(recorder.start(REQUEST)).rejects.toThrow("already recording");
recorder.recordingChanged(false, 7);
recorder.recordingFinished(7);
expect(calls.filter((call) => call === "restore")).toHaveLength(1);
await expect(started(recorder, calls)).resolves.toBeDefined();
});

it("refuses a take while one records, or while the editor holds unsaved changes", async () => {
const busy = setup({ isRecording: () => true });
await expect(busy.recorder.start(REQUEST)).rejects.toThrow("already recording");
Expand Down
13 changes: 11 additions & 2 deletions electron/mcp/app-recorder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,9 @@ export interface AppRecorderDeps {

export interface AppRecorder extends TakeRecorder {
/** The app's recording state changed (`set-recording-state`). */
recordingChanged(recording: boolean): void;
recordingChanged(recording: boolean, recordingId?: number): void;
/** A completed or failed app take no longer owns the recording bar. */
recordingFinished(recordingId: number): void;
/** The recording bar handed a saved take to the editor (`set-current-recording-session`). */
takeSaved(videoPath: string): void;
}
Expand Down Expand Up @@ -145,6 +147,7 @@ interface Deferred<T> {
}

interface ClientTake {
recordingId?: number;
phase: "starting" | "recording" | "saving";
/** Resolves with when capture started. */
started: Deferred<number>;
Expand Down Expand Up @@ -214,10 +217,16 @@ export function createAppRecorder(deps: AppRecorderDeps): AppRecorder {
}

return {
recordingChanged(recording) {
recordingFinished(recordingId) {
if (current?.recordingId !== recordingId) return;
current.restore();
current = null;
},
recordingChanged(recording, recordingId) {
if (!current) return;
if (recording && current.phase === "starting") {
current.phase = "recording";
current.recordingId = recordingId;
current.started.resolve(Date.now());
} else if (!recording && current.phase === "recording") {
// The take is captured: the bar's own settings can come back while it saves.
Expand Down
144 changes: 144 additions & 0 deletions electron/mcp/media-tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {
createToolRunner,
type McpToolDeps,
} from "./openscreen-mcp-server";
import { RecordingCompletion } from "./recording-completion";

/** The runner windows: records what was asked, answers like a runner would. */
class FakeRunners {
Expand Down Expand Up @@ -1033,3 +1034,146 @@ describe("the media tools in the app", () => {
expect(exportPath.description).not.toContain("working directory, ending");
});
});

describe("waitForRecording through MCP", () => {
const clients: Client[] = [];
afterEach(async () => {
for (const client of clients.splice(0)) await client.close();
});
async function observing(
completion: RecordingCompletion,
overrides: Partial<MediaToolDeps> = {},
) {
const deps: McpToolDeps = {
transport: "http",
host: { snapshot: async () => null, apply: async () => "no-editor" },
projects: documents,
editsAllowed: () => false,
version: "test",
};
let subscribed: () => void = () => undefined;
const subscription = new Promise<void>((resolve) => {
subscribed = resolve;
});
const media = createMediaTools({
...stdioDeps,
transport: "http",
recordingActive: () => completion.isActive(),
editsAllowed: () => false,
waitForRecording: (timeout, signal) => {
const waiting = completion.wait(timeout, signal);
subscribed();
return waiting;
},
...overrides,
});
const server = createOpenScreenMcpServer(deps, createToolRunner(deps), media);
const [clientSide, serverSide] = InMemoryTransport.createLinkedPair();
await server.connect(serverSide);
const observer = new Client({ name: "observer", version: "1" });
await observer.connect(clientSide);
clients.push(observer);
return { observer, subscription };
}

it("returns the stopRecording fields for a user take while MCP edits is off, without recording commands", async () => {
const project = await documents.createProject("User take");
const completion = new RecordingCompletion();
completion.started(42, 1_000);
const { observer, subscription } = await observing(completion);
const waiting = observer.callTool({ name: "waitForRecording", arguments: {} });
await subscription;
completion.saved(42, videoPath);
completion.completed(42, {
projectId: project.project.id,
videoPath,
durationSec: 12,
autoZooms: 2,
});
const result = await waiting;
expect(result.isError).toBeFalsy();
expect(JSON.parse(textOf(result as CallToolResult))).toEqual({
projectId: project.project.id,
videoPath,
durationSec: 12,
autoZooms: 2,
recordingStartedAt: 1_000,
checkpointId: expect.any(String),
});
expect(runners.started).toEqual([]);
});

it("releases a client take after the user finishes it, so startRecording can record again", async () => {
const completion = new RecordingCompletion();
let id = 0;
const recorder: TakeRecorder = {
start: async () => {
completion.started(++id);
return {
startedAt: id,
stop: async () => {
throw new Error("wait must never stop the take");
},
};
},
};
const { observer, subscription } = await observing(completion, {
recorder,
editsAllowed: () => true,
});
expect(
(await observer.callTool({ name: "startRecording", arguments: {} })).isError,
).toBeFalsy();
const waiting = observer.callTool({ name: "waitForRecording", arguments: {} });
await subscription;
completion.saved(1, videoPath);
completion.completed(1, { projectId: "proj_take", videoPath, durationSec: 12, autoZooms: 0 });
await waiting;
expect(
(await observer.callTool({ name: "startRecording", arguments: {} })).isError,
).toBeFalsy();
expect(id).toBe(2);
});

it.each([
0,
-1,
1.5,
3601,
"10",
])("validates timeoutSec=%s before observing", async (timeoutSec) => {
const completion = new RecordingCompletion();
const { observer } = await observing(completion);
const result = await observer.callTool({ name: "waitForRecording", arguments: { timeoutSec } });
expect(result.isError).toBe(true);
expect(textOf(result as CallToolResult)).toContain("validation");
});

it("reports no active recording as a tool error", async () => {
const { observer } = await observing(new RecordingCompletion());
const result = await observer.callTool({ name: "waitForRecording", arguments: {} });
expect(result.isError).toBe(true);
expect(textOf(result as CallToolResult)).toContain("No recording");
});

it("propagates MCP cancellation and leaves the user's take available", async () => {
const completion = new RecordingCompletion();
completion.started(42);
const { observer, subscription } = await observing(completion);
const abort = new AbortController();
const waiting = expect(
observer.callTool({ name: "waitForRecording", arguments: {} }, undefined, {
signal: abort.signal,
}),
).rejects.toThrow();
await subscription;
abort.abort();
await waiting;
// Flush the MCP cancellation notification, then the same take still completes.
await new Promise((resolve) => setTimeout(resolve, 0));
const retry = completion.wait(1000, new AbortController().signal);
completion.saved(42, videoPath);
completion.completed(42, { projectId: "proj_take", videoPath, durationSec: 12, autoZooms: 0 });
await expect(retry).resolves.toMatchObject({ projectId: "proj_take" });
});
});
Loading
Loading