From 631ca69500277e92648a6e85de465db831895727 Mon Sep 17 00:00:00 2001 From: Satyam Raj Date: Fri, 16 Oct 2026 14:43:21 +0530 Subject: [PATCH 1/6] feat: add floating camera preview on macOS --- electron/cameraPreview.test.ts | 103 +++++++++++++++++ electron/cameraPreview.ts | 84 ++++++++++++++ electron/electron-env.d.ts | 6 + electron/ipc/handlers.ts | 33 +++++- .../ScreenCaptureRecorder.swift | 6 +- electron/preload.ts | 17 +++ electron/windows.ts | 65 +++++++++++ src/App.tsx | 7 +- .../launch/FloatingCameraPreview.module.css | 88 ++++++++++++++ .../launch/FloatingCameraPreview.test.tsx | 37 ++++++ .../launch/FloatingCameraPreview.tsx | 59 ++++++++++ src/components/launch/HudControls.tsx | 34 +++++- src/components/launch/LaunchWindow.test.tsx | 45 ++++++++ src/components/launch/LaunchWindow.tsx | 14 +++ src/hooks/useFloatingCameraPreview.test.tsx | 108 ++++++++++++++++++ src/hooks/useFloatingCameraPreview.ts | 63 ++++++++++ .../useScreenRecorder.cameraPrefs.test.tsx | 97 ++++++++++++++++ ...eenRecorder.nativeMacStartWarning.test.tsx | 7 +- ...eScreenRecorder.nativeStopFailure.test.tsx | 7 +- src/hooks/useScreenRecorder.noCamera.test.tsx | 11 +- .../useScreenRecorder.prefsRace.test.tsx | 7 +- .../useScreenRecorder.restartFailure.test.tsx | 7 +- src/hooks/useScreenRecorder.ts | 27 ++++- src/hooks/webcamCaptureTarget.test.ts | 4 + src/hooks/webcamCaptureTarget.ts | 2 + src/i18n/locales/ar/launch.json | 7 ++ src/i18n/locales/cs/launch.json | 7 ++ src/i18n/locales/de/launch.json | 7 ++ src/i18n/locales/en/launch.json | 7 ++ src/i18n/locales/es/launch.json | 7 ++ src/i18n/locales/fr/launch.json | 7 ++ src/i18n/locales/it/launch.json | 7 ++ src/i18n/locales/ja-JP/launch.json | 7 ++ src/i18n/locales/ko-KR/launch.json | 7 ++ src/i18n/locales/pt-BR/launch.json | 7 ++ src/i18n/locales/ru/launch.json | 7 ++ src/i18n/locales/tr/launch.json | 7 ++ src/i18n/locales/vi/launch.json | 7 ++ src/i18n/locales/zh-CN/launch.json | 7 ++ src/i18n/locales/zh-TW/launch.json | 7 ++ src/lib/cameraPreview.test.ts | 35 ++++++ src/lib/cameraPreview.ts | 37 ++++++ .../architecture/recording.md | 27 +++++ .../testing/manual-e2e-checklist.md | 4 + 44 files changed, 1120 insertions(+), 26 deletions(-) create mode 100644 electron/cameraPreview.test.ts create mode 100644 electron/cameraPreview.ts create mode 100644 src/components/launch/FloatingCameraPreview.module.css create mode 100644 src/components/launch/FloatingCameraPreview.test.tsx create mode 100644 src/components/launch/FloatingCameraPreview.tsx create mode 100644 src/hooks/useFloatingCameraPreview.test.tsx create mode 100644 src/hooks/useFloatingCameraPreview.ts create mode 100644 src/hooks/useScreenRecorder.cameraPrefs.test.tsx create mode 100644 src/lib/cameraPreview.test.ts create mode 100644 src/lib/cameraPreview.ts diff --git a/electron/cameraPreview.test.ts b/electron/cameraPreview.test.ts new file mode 100644 index 000000000..b393f7fce --- /dev/null +++ b/electron/cameraPreview.test.ts @@ -0,0 +1,103 @@ +import { EventEmitter } from "node:events"; +import type { BrowserWindow, WebContents } from "electron"; +import { describe, expect, it, vi } from "vitest"; +import { CameraPreviewController } from "./cameraPreview"; + +function setup() { + const owner = Object.assign(new EventEmitter(), { send: vi.fn() }); + const overlay = Object.assign(new EventEmitter(), { + webContents: { send: vi.fn() }, + showInactive: vi.fn(), + hide: vi.fn(), + destroy: vi.fn(), + }); + const create = vi.fn(() => overlay as unknown as BrowserWindow); + const resize = vi.fn(); + const preview = new CameraPreviewController(create, resize); + const sender = owner as unknown as WebContents; + const overlaySender = overlay.webContents as unknown as WebContents; + return { owner, overlay, create, resize, preview, sender, overlaySender }; +} + +const jpeg = "data:image/jpeg;base64,/9j/"; + +describe("floating camera preview lifecycle", () => { + it("waits for the window to paint, then shows without taking focus", () => { + const { preview, sender, overlay } = setup(); + preview.configure(sender, true, true); + expect(overlay.showInactive).not.toHaveBeenCalled(); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + }); + + it("hides and restores the same window, retaining its capture exclusion id", () => { + const { preview, sender, overlaySender, overlay, owner, create } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + preview.hide(overlaySender); + expect(owner.send).toHaveBeenCalledWith("camera-preview:hidden"); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.destroy).not.toHaveBeenCalled(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + }); + + it("does not flash a preview disabled before its renderer is ready", () => { + const { preview, sender, overlay } = setup(); + preview.configure(sender, true, true); + preview.configure(sender, true, false); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).not.toHaveBeenCalled(); + }); + + it("ignores other renderers and rejects oversized or non-JPEG frames", () => { + const { preview, sender, overlay, create } = setup(); + const stranger = {} as WebContents; + preview.configure(sender, true, true); + preview.configure(stranger, false, false); + preview.hide(stranger); + for (const invalid of [undefined, "https://example.com/camera", jpeg + "x".repeat(128_000)]) { + preview.frame(sender, invalid); + } + preview.frame(stranger, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + expect(overlay.destroy).not.toHaveBeenCalled(); + expect(create).toHaveBeenCalledOnce(); + }); + + it("allows only the preview to resize to a supported diameter without replacing its window", () => { + const { preview, sender, overlaySender, resize, create, overlay } = setup(); + preview.configure(sender, true, true); + preview.resize(sender, 320); + preview.resize(overlaySender, Number.NaN); + preview.resize(overlaySender, 10000); + expect(resize).not.toHaveBeenCalled(); + preview.resize(overlaySender, 320); + expect(resize).toHaveBeenCalledWith(overlay, 320); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.destroy).not.toHaveBeenCalled(); + }); + + it("destroys the auxiliary window when the HUD is destroyed", () => { + const { preview, sender, owner, overlay } = setup(); + preview.configure(sender, true, true); + owner.emit("destroyed"); + expect(overlay.destroy).toHaveBeenCalledOnce(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + }); + + it("releases listeners on camera-off rather than accumulating them between takes", () => { + const { preview, sender, owner, overlay } = setup(); + for (let i = 0; i < 15; i++) { + preview.configure(sender, true, false); + overlay.emit("ready-to-show"); + preview.configure(sender, false, false); + } + expect(owner.listenerCount("destroyed")).toBe(0); + expect(overlay.destroy).toHaveBeenCalledTimes(15); + }); +}); diff --git a/electron/cameraPreview.ts b/electron/cameraPreview.ts new file mode 100644 index 000000000..dafa59813 --- /dev/null +++ b/electron/cameraPreview.ts @@ -0,0 +1,84 @@ +import type { BrowserWindow, WebContents } from "electron"; +import { CAMERA_PREVIEW_SIZES } from "../src/lib/cameraPreview"; + +/** One stable window per HUD: hiding it must not change the native capture exclusion id. */ +export class CameraPreviewController { + private window: BrowserWindow | null = null; + private owner: WebContents | null = null; + private visible = false; + private ready = false; + private onOwnerDestroyed: (() => void) | null = null; + + constructor( + private readonly createWindow: () => BrowserWindow, + private readonly resizeWindow: (window: BrowserWindow, diameter: number) => void, + ) {} + + configure(owner: WebContents, enabled: boolean, visible: boolean): void { + if (this.owner && this.owner !== owner) return; + if (!enabled) { + this.dispose(); + return; + } + if (!this.owner) { + this.owner = owner; + this.onOwnerDestroyed = () => { + if (this.owner === owner) this.dispose(); + }; + owner.once("destroyed", this.onOwnerDestroyed); + } + if (!this.window) { + this.window = this.createWindow(); + const previewWindow = this.window; + this.window.once("ready-to-show", () => { + if (this.window !== previewWindow) return; + this.ready = true; + if (this.visible) this.window?.showInactive(); + }); + } + this.visible = visible; + if (visible && this.ready) this.window.showInactive(); + else if (!visible) this.window.hide(); + } + + hide(sender: WebContents): void { + if (sender !== this.window?.webContents) return; + this.visible = false; + this.window.hide(); + this.owner?.send("camera-preview:hidden"); + } + + resize(sender: WebContents, diameter: unknown): void { + if ( + sender !== this.window?.webContents || + typeof diameter !== "number" || + !CAMERA_PREVIEW_SIZES.some((size) => size === diameter) + ) + return; + this.resizeWindow(this.window, diameter); + } + + frame(sender: WebContents, jpeg: unknown): void { + if (sender !== this.owner || !this.visible || !this.window) return; + // A 320px JPEG is usually under 20KB. Never relay arbitrary URLs or unbounded IPC data. + if ( + typeof jpeg !== "string" || + jpeg.length > 128_000 || + !jpeg.startsWith("data:image/jpeg;base64,") + ) + return; + this.window.webContents.send("camera-preview:frame", jpeg); + } + + dispose(): void { + if (this.owner && this.onOwnerDestroyed) { + this.owner.removeListener("destroyed", this.onOwnerDestroyed); + } + this.onOwnerDestroyed = null; + this.ready = false; + this.window?.destroy(); + this.window = null; + this.owner = null; + this.visible = false; + } +} diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index fc6b5f911..aea81b57c 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -58,6 +58,12 @@ interface Window { usesSystemSourcePicker?: () => Promise; /** Name of the last source picked in Apple's picker, for display only: it is not selected. */ getLastPickedSource?: () => Promise; + configureCameraPreview?: (enabled: boolean, visible: boolean) => Promise; + sendCameraPreviewFrame?: (jpeg: string) => Promise; + resizeCameraPreview?: (diameter: number) => Promise; + hideCameraPreview?: () => Promise; + onCameraPreviewFrame?: (callback: (jpeg: string) => void) => () => void; + onCameraPreviewHidden?: (callback: () => void) => () => void; openNotes: () => Promise<{ opened: boolean; reason?: string; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index ee36e105f..b1fdd8dbe 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -69,6 +69,7 @@ import { DocumentService } from "../ai-edition/document-service"; import { LlmConfigStore } from "../ai-edition/llm-config-store"; import { StylePresetService } from "../ai-edition/style-preset-service"; import { AppSettingsStore } from "../app-settings"; +import { CameraPreviewController } from "../cameraPreview"; import { createRunnerHost } from "../cli/runnerHost"; import { hideCountdownOverlay } from "../countdownOverlay"; import { isDiagnosticModeEnabled, mainLogBuffer } from "../diagnostics/main-log-buffer"; @@ -161,7 +162,12 @@ import { shouldPersistSelectedSource, } from "../recording-source-settings"; import { getSttManager } from "../stt"; -import { createRunnerWindow } from "../windows"; +import { + createCameraPreviewWindow, + createRunnerWindow, + getCameraPreviewWindow, + resizeCameraPreviewWindow, +} from "../windows"; import { getSourcesByType, shouldEnumerateSourceTypesSeparately } from "./desktopSourceTypes"; import { registerNativeBridgeHandlers } from "./nativeBridge"; import { createNativeMacMidCaptureErrorWatch } from "./nativeMacMidCaptureErrorWatch"; @@ -2141,7 +2147,7 @@ export function registerIpcHandlers( * pick would record them unless the picker is told to leave them out. */ function macPickerExcludedWindowIds() { - const appWindowSourceIds = [getMainWindow(), getNotesWindow()] + const appWindowSourceIds = [getMainWindow(), getNotesWindow(), getCameraPreviewWindow()] .filter((window): window is BrowserWindow => !!window && !window.isDestroyed()) .map((window) => window.getMediaSourceId()); return collectMacCaptureExcludedWindowIds(appWindowSourceIds); @@ -2470,6 +2476,27 @@ export function registerIpcHandlers( return { opened: true }; }); + const cameraPreview = new CameraPreviewController( + createCameraPreviewWindow, + resizeCameraPreviewWindow, + ); + ipcMain.handle("camera-preview:configure", (event, enabled: unknown, visible: unknown) => { + if ( + process.platform !== "darwin" || + event.sender !== getMainWindow()?.webContents || + new URL(event.sender.getURL()).searchParams.get("windowType") !== "hud-overlay" + ) + return; + cameraPreview.configure(event.sender, enabled === true, visible === true); + }); + ipcMain.handle("camera-preview:frame", (event, jpeg: unknown) => + cameraPreview.frame(event.sender, jpeg), + ); + ipcMain.handle("camera-preview:hide", (event) => cameraPreview.hide(event.sender)); + ipcMain.handle("camera-preview:resize", (event, diameter: unknown) => + cameraPreview.resize(event.sender, diameter), + ); + ipcMain.handle("open-notes", async () => { const notesSelectorWin = getNotesWindow(); if (notesSelectorWin) { @@ -3192,7 +3219,7 @@ export function registerIpcHandlers( pick?.bounds ?? request.source.bounds ?? sourceDisplay?.bounds ?? getSelectedSourceBounds(); const captureExcludedWindowSourceIds: string[] = []; if (request.source.type === "display") { - for (const window of [getMainWindow(), getNotesWindow()]) { + for (const window of [getMainWindow(), getNotesWindow(), getCameraPreviewWindow()]) { if (window && !window.isDestroyed()) { captureExcludedWindowSourceIds.push(window.getMediaSourceId()); } diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift index 1450f130d..e204556bf 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift @@ -215,9 +215,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { if let picked { target = makeCaptureTarget(picked: picked) } else { + // Keep hidden recording overlays in the filter, so showing a camera preview + // during the take never exposes it while the exclusion-refresh poll catches up. let content = try await SCShareableContent.excludingDesktopWindows( false, - onScreenWindowsOnly: true + onScreenWindowsOnly: false ) target = try makeCaptureTarget(from: content) if recordsSystemAudioFromDisplay, let display = content.displays.first { @@ -671,7 +673,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { guard let self, let stream = self.stream, let content = try? await SCShareableContent.excludingDesktopWindows( false, - onScreenWindowsOnly: true + onScreenWindowsOnly: false ), let display = content.displays.first(where: { $0.displayID == displayID }) else { diff --git a/electron/preload.ts b/electron/preload.ts index 23160bcaa..c80e50d4f 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -169,6 +169,23 @@ contextBridge.exposeInMainWorld("electronAPI", { getLastPickedSource: (): Promise => { return ipcRenderer.invoke("get-last-picked-source"); }, + configureCameraPreview: (enabled: boolean, visible: boolean): Promise => + ipcRenderer.invoke("camera-preview:configure", enabled, visible), + sendCameraPreviewFrame: (jpeg: string): Promise => + ipcRenderer.invoke("camera-preview:frame", jpeg), + resizeCameraPreview: (diameter: number): Promise => + ipcRenderer.invoke("camera-preview:resize", diameter), + hideCameraPreview: (): Promise => ipcRenderer.invoke("camera-preview:hide"), + onCameraPreviewFrame: (callback: (jpeg: string) => void) => { + const listener = (_event: Electron.IpcRendererEvent, jpeg: string) => callback(jpeg); + ipcRenderer.on("camera-preview:frame", listener); + return () => ipcRenderer.removeListener("camera-preview:frame", listener); + }, + onCameraPreviewHidden: (callback: () => void) => { + const listener = () => callback(); + ipcRenderer.on("camera-preview:hidden", listener); + return () => ipcRenderer.removeListener("camera-preview:hidden", listener); + }, openNotes: () => { return ipcRenderer.invoke("open-notes"); }, diff --git a/electron/windows.ts b/electron/windows.ts index 8db7d9cd4..3557ac0de 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { app, BrowserWindow, ipcMain, screen } from "electron"; +import { cameraPreviewBounds, resizedCameraPreviewBounds } from "../src/lib/cameraPreview"; import { fitEditorRect, loadEditorWindowState, @@ -860,6 +861,70 @@ export function createPermissionsWindow(): BrowserWindow { return win; } +let cameraPreviewWindow: BrowserWindow | null = null; + +export function getCameraPreviewWindow(): BrowserWindow | null { + return cameraPreviewWindow && !cameraPreviewWindow.isDestroyed() ? cameraPreviewWindow : null; +} + +/** The camera stream belongs to the HUD; this window only displays preview frames. */ +export function createCameraPreviewWindow(): BrowserWindow { + const initialBounds = () => { + const bar = + hudOverlayWindow && !hudOverlayWindow.isDestroyed() && hudContentRect + ? hudContentScreenRect(hudOverlayWindow.getBounds(), hudContentRect) + : undefined; + const workArea = bar + ? screen.getDisplayMatching(bar).workArea + : screen.getPrimaryDisplay().workArea; + return cameraPreviewBounds(workArea, bar); + }; + const win = new BrowserWindow({ + ...initialBounds(), + title: "OpenScreen - Camera Preview", + hasShadow: false, + frame: false, + transparent: true, + backgroundColor: "#00000000", + resizable: false, + maximizable: false, + minimizable: false, + closable: false, + alwaysOnTop: true, + skipTaskbar: true, + show: false, + webPreferences: { + preload: path.join(__dirname, "preload.mjs"), + additionalArguments: [ASSET_BASE_URL_ARG], + nodeIntegration: false, + contextIsolation: true, + backgroundThrottling: false, + }, + }); + win.once("ready-to-show", () => win.setBounds(initialBounds())); + applyContentProtection(win, "Camera Preview"); + followAcrossSpaces(win); + cameraPreviewWindow = win; + win.on("closed", () => { + if (cameraPreviewWindow === win) cameraPreviewWindow = null; + }); + if (VITE_DEV_SERVER_URL) { + win.loadURL(VITE_DEV_SERVER_URL + "?windowType=camera-preview"); + } else { + win.loadFile(path.join(RENDERER_DIST, "index.html"), { + query: { windowType: "camera-preview" }, + }); + } + return win; +} + +export function resizeCameraPreviewWindow(win: BrowserWindow, diameter: number): void { + const bounds = win.getBounds(); + win.setBounds( + resizedCameraPreviewBounds(bounds, screen.getDisplayMatching(bounds).workArea, diameter), + ); +} + // Frameless Notes Window for taking notes during a recording. export function createNotesWindow(): BrowserWindow { const win = new BrowserWindow({ diff --git a/src/App.tsx b/src/App.tsx index 681bb5993..c326d3b83 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -6,6 +6,7 @@ import { installBrowserShims } from "./native/browserShim"; installBrowserShims(); import { CountdownOverlay } from "./components/launch/CountdownOverlay.tsx"; +import { FloatingCameraPreview } from "./components/launch/FloatingCameraPreview"; import { LaunchWindow } from "./components/launch/LaunchWindow"; import { NotesWindow } from "./components/launch/NotesWindow.tsx"; import { SourceSelector } from "./components/launch/SourceSelector"; @@ -52,7 +53,7 @@ export default function App() { setWindowType(type); } - if (type === "hud-overlay" || type === "source-selector" || type === "countdown-overlay") { + if (["hud-overlay", "source-selector", "countdown-overlay", "camera-preview"].includes(type)) { document.body.style.background = "transparent"; document.documentElement.style.background = "transparent"; document.getElementById("root")?.style.setProperty("background", "transparent"); @@ -60,7 +61,7 @@ export default function App() { // HUD is a fixed-size BrowserWindow; pin the document shell and hide overflow // so the renderer can't introduce scrollbars (see issue #305). - if (type === "hud-overlay") { + if (type === "hud-overlay" || type === "camera-preview") { document.documentElement.style.height = "100%"; document.documentElement.style.overflow = "hidden"; document.body.style.height = "100%"; @@ -85,6 +86,8 @@ export default function App() { switch (windowType) { case "hud-overlay": return ; + case "camera-preview": + return ; case "source-selector": return ; case "countdown-overlay": diff --git a/src/components/launch/FloatingCameraPreview.module.css b/src/components/launch/FloatingCameraPreview.module.css new file mode 100644 index 000000000..9239c7e43 --- /dev/null +++ b/src/components/launch/FloatingCameraPreview.module.css @@ -0,0 +1,88 @@ +.preview { + -webkit-app-region: drag; + position: relative; + display: grid; + place-items: center; + width: calc(100% - 16px); + height: calc(100% - 16px); + margin: 8px; + overflow: hidden; + border-radius: 50%; + background: #09090b; + color: #a1a1aa; + font-size: 12px; + cursor: grab; +} + +.preview img { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + object-fit: cover; + object-position: center; + transform: scaleX(-1); +} + +.hide { + -webkit-app-region: no-drag; + position: absolute; + top: 9%; + right: 13%; + display: grid; + place-items: center; + width: 28px; + height: 28px; + border-radius: 50%; + background: #09090bbf; + color: #f4f4f5; + cursor: pointer; + opacity: 0; + transition: opacity 120ms ease; +} + +.preview:hover .hide, +.preview:focus-within .hide { + opacity: 1; +} + +.hide:focus-visible { + outline: 2px solid #10b981; +} + +.sizeControls { + -webkit-app-region: no-drag; + position: absolute; + bottom: 9%; + left: 50%; + display: flex; + gap: 4px; + transform: translateX(-50%); + opacity: 0; + transition: opacity 120ms ease; +} + +.sizeControls button { + display: grid; + place-items: center; + width: 28px; + height: 28px; + border-radius: 50%; + background: #09090bbf; + color: #f4f4f5; + cursor: pointer; +} + +.sizeControls button:disabled { + opacity: 0.4; + cursor: default; +} + +.preview:hover .sizeControls, +.preview:focus-within .sizeControls { + opacity: 1; +} + +.sizeControls button:focus-visible { + outline: 2px solid #10b981; +} diff --git a/src/components/launch/FloatingCameraPreview.test.tsx b/src/components/launch/FloatingCameraPreview.test.tsx new file mode 100644 index 000000000..08194f994 --- /dev/null +++ b/src/components/launch/FloatingCameraPreview.test.tsx @@ -0,0 +1,37 @@ +// @vitest-environment jsdom +import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, expect, it, vi } from "vitest"; +import { FloatingCameraPreview } from "./FloatingCameraPreview"; + +vi.mock("../../contexts/I18nContext", () => ({ useScopedT: () => (key: string) => key })); +afterEach(cleanup); + +it("displays incoming frames and offers three bounded preview sizes", async () => { + let frame: ((jpeg: string) => void) | undefined; + const resize = vi.fn().mockResolvedValue(undefined); + const unsubscribe = vi.fn(); + window.electronAPI = { + onCameraPreviewFrame: (callback: (jpeg: string) => void) => { + frame = callback; + return unsubscribe; + }, + resizeCameraPreview: resize, + } as unknown as Window["electronAPI"]; + const view = render(); + act(() => frame?.("data:image/jpeg;base64,frame")); + expect(screen.getByAltText("floatingCamera.title").getAttribute("src")).toBe( + "data:image/jpeg;base64,frame", + ); + const larger = screen.getByRole("button", { name: "floatingCamera.larger" }); + const smaller = screen.getByRole("button", { name: "floatingCamera.smaller" }); + fireEvent.click(larger); + await waitFor(() => expect((larger as HTMLButtonElement).disabled).toBe(true)); + expect(resize).toHaveBeenLastCalledWith(320); + fireEvent.click(smaller); + await waitFor(() => expect((larger as HTMLButtonElement).disabled).toBe(false)); + fireEvent.click(smaller); + await waitFor(() => expect((smaller as HTMLButtonElement).disabled).toBe(true)); + expect(resize).toHaveBeenLastCalledWith(160); + view.unmount(); + expect(unsubscribe).toHaveBeenCalledOnce(); +}); diff --git a/src/components/launch/FloatingCameraPreview.tsx b/src/components/launch/FloatingCameraPreview.tsx new file mode 100644 index 000000000..cdd21f351 --- /dev/null +++ b/src/components/launch/FloatingCameraPreview.tsx @@ -0,0 +1,59 @@ +import { Minus, Plus, X } from "lucide-react"; +import { useEffect, useRef, useState } from "react"; +import { useScopedT } from "../../contexts/I18nContext"; +import { CAMERA_PREVIEW_SIZES } from "../../lib/cameraPreview"; +import styles from "./FloatingCameraPreview.module.css"; + +export function FloatingCameraPreview() { + const t = useScopedT("launch"); + const image = useRef(null); + const [ready, setReady] = useState(false); + const [sizeIndex, setSizeIndex] = useState(1); + const resize = (next: number) => { + void window.electronAPI + ?.resizeCameraPreview?.(CAMERA_PREVIEW_SIZES[next]) + .then(() => setSizeIndex(next)) + .catch((error: unknown) => console.warn("Could not resize camera preview:", error)); + }; + useEffect( + () => + window.electronAPI?.onCameraPreviewFrame?.((jpeg) => { + if (image.current) image.current.src = jpeg; + setReady(true); + }), + [], + ); + + return ( +
+ + {!ready && {t("deviceSettings.previewUnavailable")}} + +
+ + +
+
+ ); +} diff --git a/src/components/launch/HudControls.tsx b/src/components/launch/HudControls.tsx index 2c5fa7df1..93322a7bd 100644 --- a/src/components/launch/HudControls.tsx +++ b/src/components/launch/HudControls.tsx @@ -1,4 +1,4 @@ -import { Check, Languages, NotepadText, Settings } from "lucide-react"; +import { Check, Eye, EyeOff, Languages, NotepadText, Settings } from "lucide-react"; import { createContext, memo, @@ -297,6 +297,38 @@ export const HudMicButton = memo(function HudMicButton({ ); }); +export function HudCameraPreviewButton({ + visible, + disabled, + label, + onClick, +}: { + visible: boolean; + disabled: boolean; + label: string; + onClick: () => void; +}) { + return ( + + + + ); +} + export const HudCameraButton = memo(function HudCameraButton({ enabled, locked, diff --git a/src/components/launch/LaunchWindow.test.tsx b/src/components/launch/LaunchWindow.test.tsx index 8cd92de95..64dcdb410 100644 --- a/src/components/launch/LaunchWindow.test.tsx +++ b/src/components/launch/LaunchWindow.test.tsx @@ -2374,3 +2374,48 @@ describe("LaunchWindow toasts", () => { }); }); }); + +describe("macOS floating camera preview control", () => { + beforeEach(() => { + platformState.value = "darwin"; + resetLaunchMocks(); + }); + + it("keeps hide/show available during recording without changing camera capture", () => { + recorderState.value.webcamEnabled = true; + recorderState.value.recording = true; + renderLaunchWindow(); + const button = screen.getByTestId("launch-camera-preview-button"); + expect(button).toHaveAttribute("aria-pressed", "true"); + fireEvent.click(button); + expect(button).toHaveAttribute("aria-pressed", "false"); + fireEvent.click(button); + expect(button).toHaveAttribute("aria-pressed", "true"); + expect(recorderState.value.setWebcamEnabled).not.toHaveBeenCalled(); + }); + + it("keeps the preview button visible but disabled when the camera is off", () => { + const view = renderLaunchWindow(); + const button = screen.getByTestId("launch-camera-preview-button"); + expect(button).toHaveAttribute("aria-disabled", "true"); + expect(button).toHaveAttribute("aria-pressed", "false"); + fireEvent.click(button); + expect(button).toHaveAttribute("aria-pressed", "false"); + expect(recorderState.value.setWebcamEnabled).not.toHaveBeenCalled(); + recorderState.value.webcamEnabled = true; + view.rerender( + + + , + ); + expect(button).not.toHaveAttribute("aria-disabled"); + expect(button).toHaveAttribute("aria-pressed", "true"); + }); + + it("does not offer the preview on Windows", () => { + platformState.value = "win32"; + recorderState.value.webcamEnabled = true; + renderLaunchWindow(); + expect(screen.queryByTestId("launch-camera-preview-button")).not.toBeInTheDocument(); + }); +}); diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index d4b402b84..ffd3132eb 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -8,6 +8,7 @@ import { nativeBridgeClient } from "@/native"; import { canRecordMicrophone, getPlatform } from "@/utils/platformUtils"; import { type CameraDevice } from "../../hooks/useCameraDevices"; import { useCameraHudSync } from "../../hooks/useCameraHudSync"; +import { useFloatingCameraPreview } from "../../hooks/useFloatingCameraPreview"; import { isPlaceholderMicrophoneLabel, type MicrophoneDevice, @@ -21,6 +22,7 @@ import { requestCameraAccess } from "../../lib/requestCameraAccess"; import { Toaster } from "../ui/sonner"; import { HudCameraButton, + HudCameraPreviewButton, HudCursorButton, HudDivider, HudDragHandle, @@ -104,6 +106,7 @@ export function LaunchWindow() { const languageCode = locale.split("-")[0].toUpperCase(); const { + webcamPreviewStream, recording, paused, saving, @@ -137,6 +140,9 @@ export function LaunchWindow() { } = useScreenRecorder(); // Choosing a device and switching one on are deliberately separate concerns. + const cameraPreview = useFloatingCameraPreview(webcamPreviewStream, webcamEnabled && !saving); + const cameraPreviewShown = webcamEnabled && !saving && cameraPreview.visible; + // The mic and camera buttons are plain on/off toggles that use whatever device // is currently selected (the system default until the user says otherwise); // picking a different device — and checking it actually works — happens in the @@ -1279,6 +1285,14 @@ export function LaunchWindow() { tooltip={t("tooltips.camera")} onClick={toggleWebcam} /> + {cameraPreview.supported && ( + + )} ({ getPlatform: () => platform })); +let platform = "darwin"; +let hidden: (() => void) | undefined; +const configure = vi.fn().mockResolvedValue(undefined); +const send = vi.fn().mockResolvedValue(undefined); +const unsubscribe = vi.fn(); +const stop = vi.fn(); +const stream = { getTracks: () => [{ stop }] } as unknown as MediaStream; + +beforeEach(() => { + vi.useFakeTimers(); + platform = "darwin"; + vi.clearAllMocks(); + Object.defineProperty(window, "electronAPI", { + configurable: true, + value: { + configureCameraPreview: configure, + sendCameraPreviewFrame: send, + onCameraPreviewHidden: (callback: () => void) => { + hidden = callback; + return unsubscribe; + }, + }, + }); + vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(undefined); + vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => undefined); + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({ + drawImage: vi.fn(), + } as unknown as CanvasRenderingContext2D); + vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue( + "data:image/jpeg;base64,frame", + ); + vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockReturnValue(2); + vi.spyOn(HTMLVideoElement.prototype, "videoWidth", "get").mockReturnValue(1280); + vi.spyOn(HTMLVideoElement.prototype, "videoHeight", "get").mockReturnValue(720); +}); + +afterEach(() => { + cleanup(); + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +describe("floating camera publisher", () => { + it("shares frames without acquiring or stopping the recording camera", async () => { + const { unmount } = renderHook(() => useFloatingCameraPreview(stream, true)); + expect(configure).toHaveBeenLastCalledWith(true, true); + expect(send).toHaveBeenCalledOnce(); + await act(() => vi.advanceTimersByTimeAsync(300)); + expect(send).toHaveBeenCalledTimes(4); + unmount(); + await vi.advanceTimersByTimeAsync(500); + expect(send).toHaveBeenCalledTimes(4); + expect(stop).not.toHaveBeenCalled(); + expect(configure).toHaveBeenLastCalledWith(false, false); + expect(unsubscribe).toHaveBeenCalledOnce(); + }); + + it("hides without disabling the camera, then resumes from the HUD control", () => { + const { result } = renderHook(() => useFloatingCameraPreview(stream, true)); + act(() => hidden?.()); + expect(configure).toHaveBeenLastCalledWith(true, false); + expect(result.current.visible).toBe(false); + act(() => result.current.toggle()); + expect(configure).toHaveBeenLastCalledWith(true, true); + expect(stop).not.toHaveBeenCalled(); + }); + + it("applies backpressure while a previous frame is still being delivered", async () => { + let delivered: (() => void) | undefined; + send.mockImplementationOnce( + () => + new Promise((resolve) => { + delivered = resolve; + }), + ); + renderHook(() => useFloatingCameraPreview(stream, true)); + await act(() => vi.advanceTimersByTimeAsync(1000)); + expect(send).toHaveBeenCalledOnce(); + await act(async () => { + delivered?.(); + }); + await act(() => vi.advanceTimersByTimeAsync(100)); + expect(send).toHaveBeenCalledTimes(2); + }); + + it("waits for the recorder stream and never opens a preview on other platforms", () => { + const { rerender, unmount } = renderHook( + ({ enabled }) => useFloatingCameraPreview(null, enabled), + { initialProps: { enabled: true } }, + ); + expect(configure).toHaveBeenLastCalledWith(true, false); + expect(send).not.toHaveBeenCalled(); + rerender({ enabled: false }); + expect(configure).toHaveBeenLastCalledWith(false, false); + unmount(); + configure.mockClear(); + platform = "win32"; + renderHook(() => useFloatingCameraPreview(stream, true)); + expect(configure).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); +}); diff --git a/src/hooks/useFloatingCameraPreview.ts b/src/hooks/useFloatingCameraPreview.ts new file mode 100644 index 000000000..d6bb2d012 --- /dev/null +++ b/src/hooks/useFloatingCameraPreview.ts @@ -0,0 +1,63 @@ +import { useEffect, useState } from "react"; +import { getPlatform } from "../utils/platformUtils"; + +/** A small self-view from the recorder's stream; never acquires or stops the camera. */ +export function useFloatingCameraPreview(stream: MediaStream | null, enabled: boolean) { + const supported = getPlatform() === "darwin"; + const [visible, setVisible] = useState(true); + + useEffect(() => window.electronAPI?.onCameraPreviewHidden?.(() => setVisible(false)), []); + + useEffect(() => { + if (!supported) return; + void window.electronAPI + ?.configureCameraPreview?.(enabled, enabled && visible && !!stream) + .catch((error: unknown) => console.warn("Could not configure camera preview:", error)); + }, [enabled, visible, stream, supported]); + + useEffect(() => { + if (!supported || !enabled || !visible || !stream) return; + const video = document.createElement("video"); + video.muted = true; + video.playsInline = true; + video.srcObject = stream; + void video.play().catch(() => undefined); + const canvas = document.createElement("canvas"); + canvas.width = 320; + canvas.height = 180; + const context = canvas.getContext("2d"); + let cancelled = false; + let timer: ReturnType; + const draw = async () => { + try { + if (context && video.readyState >= 2 && video.videoWidth > 0) { + canvas.height = Math.max(1, Math.round((320 * video.videoHeight) / video.videoWidth)); + context.drawImage(video, 0, 0, canvas.width, canvas.height); + // Await delivery: a slow renderer must not build a queue of stale frames. + await window.electronAPI?.sendCameraPreviewFrame?.(canvas.toDataURL("image/jpeg", 0.7)); + } + } catch (error) { + console.warn("Could not update camera preview:", error); + } finally { + if (!cancelled) timer = setTimeout(() => void draw(), 100); + } + }; + void draw(); + return () => { + cancelled = true; + clearTimeout(timer); + video.pause(); + video.srcObject = null; + }; + }, [enabled, visible, stream, supported]); + + useEffect( + () => () => { + if (supported) + void window.electronAPI?.configureCameraPreview?.(false, false).catch(() => undefined); + }, + [supported], + ); + + return { supported, visible, toggle: () => setVisible((value) => !value) }; +} diff --git a/src/hooks/useScreenRecorder.cameraPrefs.test.tsx b/src/hooks/useScreenRecorder.cameraPrefs.test.tsx new file mode 100644 index 000000000..b7f4f4129 --- /dev/null +++ b/src/hooks/useScreenRecorder.cameraPrefs.test.tsx @@ -0,0 +1,97 @@ +// @vitest-environment jsdom +import { act, cleanup, renderHook, waitFor } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useCameraHudSync } from "./useCameraHudSync"; +import { useScreenRecorder } from "./useScreenRecorder"; + +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); +vi.mock("sonner", () => ({ toast: { error: vi.fn(), warning: vi.fn(), info: vi.fn() } })); +vi.mock("./useCameraDevices", () => ({ + useCameraDevices: (_enabled: boolean, preferred?: string) => ({ + selectedDeviceId: preferred ?? "cam-1", + selectedDevice: { label: preferred === "cam-2" ? "Camera 2" : "Camera 1" }, + isReady: true, + }), +})); + +type Prefs = Awaited>; +const initial: Prefs = { + micEnabled: false, + micDeviceId: null, + micDeviceName: null, + camEnabled: true, + camDeviceId: null, + camDeviceName: null, + camQuality: "2160p", + systemAudioEnabled: false, + cursorCaptureMode: "editable-overlay", + hideDesktopIcons: false, + autoZoomEnabled: true, +}; + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +function setup() { + let publish: ((prefs: Prefs) => void) | undefined; + const stops: Array> = []; + const getUserMedia = vi.fn(async () => { + const stop = vi.fn(); + stops.push(stop); + const track = { stop, onended: null }; + return { getTracks: () => [track], getVideoTracks: () => [track] } as unknown as MediaStream; + }); + Object.defineProperty(navigator, "mediaDevices", { configurable: true, value: { getUserMedia } }); + window.electronAPI = { + getPlatform: () => "darwin", + getRecordingPrefs: async () => initial, + onRecordingPrefsChanged: (callback: (prefs: Prefs) => void) => { + publish = callback; + return () => undefined; + }, + hideCountdownOverlay: vi.fn().mockResolvedValue(undefined), + } as unknown as Window["electronAPI"]; + const view = renderHook(() => { + const recorder = useScreenRecorder(); + useCameraHudSync(recorder); + return recorder; + }); + return { view, getUserMedia, stops, publish: (prefs: Prefs) => publish?.(prefs) }; +} + +describe("camera preview across recording preference broadcasts", () => { + it("keeps the resolved default camera stream alive when audio toggles publish unchanged camera preferences", async () => { + const { view, getUserMedia, stops, publish } = setup(); + await waitFor(() => expect(view.result.current.webcamPreviewStream).not.toBeNull()); + const stream = view.result.current.webcamPreviewStream; + const calls = getUserMedia.mock.calls.length; + const stop = stops.at(-1); + await act(async () => { + view.result.current.setSystemAudioEnabled(true); + publish({ ...initial, systemAudioEnabled: true }); + }); + await act(async () => { + view.result.current.setMicrophoneEnabled(true); + publish({ ...initial, micEnabled: true, systemAudioEnabled: true }); + }); + expect(view.result.current.webcamPreviewStream).toBe(stream); + expect(getUserMedia).toHaveBeenCalledTimes(calls); + expect(stop).not.toHaveBeenCalled(); + }); + + it("still applies a changed camera selection and an explicit reset to the default", async () => { + const { view, publish } = setup(); + await waitFor(() => expect(view.result.current.webcamPreviewStream).not.toBeNull()); + const first = view.result.current.webcamPreviewStream; + await act(async () => publish({ ...initial, camDeviceId: "cam-2", camDeviceName: "Camera 2" })); + expect(view.result.current.webcamDeviceId).toBe("cam-2"); + expect(view.result.current.webcamPreviewStream).not.toBe(first); + await act(async () => publish(initial)); + expect(view.result.current.webcamDeviceId).toBe("cam-1"); + }); +}); diff --git a/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx b/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx index 817a008f9..65176bcb4 100644 --- a/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx +++ b/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx @@ -2,9 +2,10 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx b/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx index 928ac5886..caad8e673 100644 --- a/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx +++ b/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx @@ -2,9 +2,10 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.noCamera.test.tsx b/src/hooks/useScreenRecorder.noCamera.test.tsx index cd75f16b8..4958a8fbf 100644 --- a/src/hooks/useScreenRecorder.noCamera.test.tsx +++ b/src/hooks/useScreenRecorder.noCamera.test.tsx @@ -2,9 +2,10 @@ import { act, renderHook, waitFor } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, @@ -101,7 +102,9 @@ describe("useScreenRecorder on a machine without a camera (#967)", () => { expect(ok).toBe(true); expect(view.result.current.webcamEnabled).toBe(true); - await waitFor(() => expect(getUserMedia).toHaveBeenCalled()); + await waitFor(() => expect(view.result.current.webcamPreviewStream).not.toBeNull()); + expect(view.result.current.webcamPreviewStream).toBe(await getUserMedia.mock.results[0].value); + expect(getUserMedia).toHaveBeenCalledTimes(1); expect(setRecordingPrefs).not.toHaveBeenCalled(); }); diff --git a/src/hooks/useScreenRecorder.prefsRace.test.tsx b/src/hooks/useScreenRecorder.prefsRace.test.tsx index ae84c2cd7..edb9c5859 100644 --- a/src/hooks/useScreenRecorder.prefsRace.test.tsx +++ b/src/hooks/useScreenRecorder.prefsRace.test.tsx @@ -2,9 +2,10 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.restartFailure.test.tsx b/src/hooks/useScreenRecorder.restartFailure.test.tsx index 63c78d916..764338e5b 100644 --- a/src/hooks/useScreenRecorder.restartFailure.test.tsx +++ b/src/hooks/useScreenRecorder.restartFailure.test.tsx @@ -2,9 +2,10 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index c5156749b..37264921b 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -88,6 +88,7 @@ const AUDIO_BITRATE_VOICE = 128_000; const AUDIO_BITRATE_SYSTEM = 192_000; type UseScreenRecorderReturn = { + webcamPreviewStream: MediaStream | null; recording: boolean; paused: boolean; saving: boolean; @@ -285,6 +286,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn { useEffect(() => { let cancelled = false; let receivedNewerSnapshot = false; + let appliedCameraSelection: { + deviceId: string | undefined; + deviceName: string | undefined; + } | null = null; const applyPrefs = (prefs: { micEnabled: boolean; micDeviceId?: string | null; @@ -307,8 +312,20 @@ export function useScreenRecorder(): UseScreenRecorderReturn { new URLSearchParams(window.location.search).get("windowType") === "cli-record"; if (!isCliRecord) { setWebcamEnabledState(prefs.camEnabled); - setWebcamDeviceId(prefs.camDeviceId ?? undefined); - setWebcamDeviceName(prefs.camDeviceName ?? undefined); + const deviceId = prefs.camDeviceId ?? undefined; + const deviceName = prefs.camDeviceName ?? undefined; + // The HUD resolves a stored default (null) to a concrete device locally. + // An audio-only save republishes that same null: applying it again would + // reset the resolved device, stop its stream, and visibly blink the preview. + if ( + !appliedCameraSelection || + appliedCameraSelection.deviceId !== deviceId || + appliedCameraSelection.deviceName !== deviceName + ) { + setWebcamDeviceId(deviceId); + setWebcamDeviceName(deviceName); + appliedCameraSelection = { deviceId, deviceName }; + } } setWebcamQuality(webcamQualityFrom(prefs.camQuality)); setSystemAudioEnabled(prefs.systemAudioEnabled); @@ -351,6 +368,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const screenStream = useRef(null); const microphoneStream = useRef(null); const webcamStream = useRef(null); + const [webcamPreviewStream, setWebcamPreviewStream] = useState(null); const mixingContext = useRef(null); const recordingId = useRef(0); const accumulatedDurationMs = useRef(0); @@ -467,6 +485,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { track.stop(); }); webcamStream.current = null; + setWebcamPreviewStream(null); webcamReady.current = true; }, []); @@ -514,6 +533,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { stream.getVideoTracks().forEach((track) => { track.onended = () => { webcamStream.current = null; + setWebcamPreviewStream(null); if (!restarting.current) { setWebcamEnabledState(false); toast.error(t("recording.cameraDisconnected")); @@ -521,6 +541,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }; }); webcamStream.current = stream; + setWebcamPreviewStream(stream); webcamReady.current = true; } catch (cameraError) { if (isRecoverableWebcamConstraintError(cameraError, webcamDeviceName)) { @@ -560,6 +581,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { track.stop(); }); webcamStream.current = null; + setWebcamPreviewStream(null); } }; }, [webcamEnabled, webcamDeviceId, webcamDeviceName, webcamQuality, t]); @@ -2459,6 +2481,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }; return { + webcamPreviewStream, recording, paused, saving, diff --git a/src/hooks/webcamCaptureTarget.test.ts b/src/hooks/webcamCaptureTarget.test.ts index 25f83efdd..0de5da047 100644 --- a/src/hooks/webcamCaptureTarget.test.ts +++ b/src/hooks/webcamCaptureTarget.test.ts @@ -16,10 +16,12 @@ describe("webcamVideoConstraints", () => { expect(webcamVideoConstraints(undefined, "2160p")).toMatchObject({ width: { ideal: 3840 }, height: { ideal: 2160 }, + aspectRatio: { ideal: 16 / 9 }, }); expect(webcamVideoConstraints(undefined, "1080p")).toMatchObject({ width: { ideal: 1920 }, height: { ideal: 1080 }, + aspectRatio: { ideal: 16 / 9 }, }); }); @@ -33,6 +35,8 @@ describe("webcamVideoConstraints", () => { expect(constraints.height).not.toHaveProperty("exact"); expect(constraints.width).not.toHaveProperty("min"); expect(constraints.height).not.toHaveProperty("min"); + expect(constraints.aspectRatio).not.toHaveProperty("exact"); + expect(constraints.aspectRatio).not.toHaveProperty("min"); }); it("still pins the chosen device and caps the frame rate", () => { diff --git a/src/hooks/webcamCaptureTarget.ts b/src/hooks/webcamCaptureTarget.ts index 906412b19..fa960469f 100644 --- a/src/hooks/webcamCaptureTarget.ts +++ b/src/hooks/webcamCaptureTarget.ts @@ -74,6 +74,8 @@ export function webcamVideoConstraints( ...(deviceId ? { deviceId: { exact: deviceId } } : {}), width: { ideal: preset.width }, height: { ideal: preset.height }, + // Width/height alone can select a portrait mode on cameras offering both orientations. + aspectRatio: { ideal: preset.width / preset.height }, frameRate: { ideal: WEBCAM_TARGET_FRAME_RATE, max: WEBCAM_TARGET_FRAME_RATE }, }; } diff --git a/src/i18n/locales/ar/launch.json b/src/i18n/locales/ar/launch.json index ef613d15d..538b34e26 100644 --- a/src/i18n/locales/ar/launch.json +++ b/src/i18n/locales/ar/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "معاينة الكاميرا", + "show": "إظهار معاينة الكاميرا", + "hide": "إخفاء معاينة الكاميرا", + "smaller": "تصغير المعاينة", + "larger": "تكبير المعاينة" + }, "tooltips": { "hideHUD": "إخفاء واجهة العرض", "hideHUDTip": "إخفاء الشريط. يمكنك إظهاره مجددًا من أيقونة علبة النظام.", diff --git a/src/i18n/locales/cs/launch.json b/src/i18n/locales/cs/launch.json index d9912da51..a5a63939a 100644 --- a/src/i18n/locales/cs/launch.json +++ b/src/i18n/locales/cs/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Náhled kamery", + "show": "Zobrazit náhled kamery", + "hide": "Skrýt náhled kamery", + "smaller": "Zmenšit náhled", + "larger": "Zvětšit náhled" + }, "tooltips": { "hideHUD": "Skrýt HUD", "hideHUDTip": "Skrýt panel. Znovu ho zobrazíte z ikony v oznamovací oblasti.", diff --git a/src/i18n/locales/de/launch.json b/src/i18n/locales/de/launch.json index b4dab82a9..204b6e858 100644 --- a/src/i18n/locales/de/launch.json +++ b/src/i18n/locales/de/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Kameravorschau", + "show": "Kameravorschau anzeigen", + "hide": "Kameravorschau ausblenden", + "smaller": "Vorschau verkleinern", + "larger": "Vorschau vergrößern" + }, "tooltips": { "hideHUD": "HUD ausblenden", "hideHUDTip": "Aufnahmeleiste ausblenden. Über das Symbol im Infobereich wieder einblenden.", diff --git a/src/i18n/locales/en/launch.json b/src/i18n/locales/en/launch.json index 97263f184..4987f2396 100644 --- a/src/i18n/locales/en/launch.json +++ b/src/i18n/locales/en/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Camera preview", + "show": "Show camera preview", + "hide": "Hide camera preview", + "smaller": "Decrease preview size", + "larger": "Increase preview size" + }, "tooltips": { "hideHUD": "Hide recording bar", "hideHUDTip": "Hide the recording bar. Show it again from the tray icon.", diff --git a/src/i18n/locales/es/launch.json b/src/i18n/locales/es/launch.json index 3114921ec..fc230bba7 100644 --- a/src/i18n/locales/es/launch.json +++ b/src/i18n/locales/es/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Vista previa de cámara", + "show": "Mostrar vista previa de cámara", + "hide": "Ocultar vista previa de cámara", + "smaller": "Reducir vista previa", + "larger": "Ampliar vista previa" + }, "tooltips": { "hideHUD": "Ocultar HUD", "hideHUDTip": "Ocultar la barra. Vuelve a mostrarla desde el icono de la bandeja del sistema.", diff --git a/src/i18n/locales/fr/launch.json b/src/i18n/locales/fr/launch.json index c956fb37a..b768831ed 100644 --- a/src/i18n/locales/fr/launch.json +++ b/src/i18n/locales/fr/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Aperçu de la caméra", + "show": "Afficher l’aperçu de la caméra", + "hide": "Masquer l’aperçu de la caméra", + "smaller": "Réduire l’aperçu", + "larger": "Agrandir l’aperçu" + }, "tooltips": { "hideHUD": "Masquer le HUD", "hideHUDTip": "Masquer la barre. Réaffichez-la depuis l'icône de la zone de notification.", diff --git a/src/i18n/locales/it/launch.json b/src/i18n/locales/it/launch.json index 9897b92eb..cad6d1551 100644 --- a/src/i18n/locales/it/launch.json +++ b/src/i18n/locales/it/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Anteprima della videocamera", + "show": "Mostra anteprima della videocamera", + "hide": "Nascondi anteprima della videocamera", + "smaller": "Riduci anteprima", + "larger": "Ingrandisci anteprima" + }, "tooltips": { "hideHUD": "Nascondi", "hideHUDTip": "Nascondi la barra. Puoi mostrarla di nuovo dall'icona nell'area di notifica.", diff --git a/src/i18n/locales/ja-JP/launch.json b/src/i18n/locales/ja-JP/launch.json index 9f2323eec..a18f71ded 100644 --- a/src/i18n/locales/ja-JP/launch.json +++ b/src/i18n/locales/ja-JP/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "カメラプレビュー", + "show": "カメラプレビューを表示", + "hide": "カメラプレビューを非表示", + "smaller": "プレビューを縮小", + "larger": "プレビューを拡大" + }, "tooltips": { "hideHUD": "HUDを隠す", "hideHUDTip": "録画バーを隠します。通知領域のアイコンから再表示できます。", diff --git a/src/i18n/locales/ko-KR/launch.json b/src/i18n/locales/ko-KR/launch.json index 5ce2b8e72..455a84146 100644 --- a/src/i18n/locales/ko-KR/launch.json +++ b/src/i18n/locales/ko-KR/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "카메라 미리보기", + "show": "카메라 미리보기 표시", + "hide": "카메라 미리보기 숨기기", + "smaller": "미리보기 축소", + "larger": "미리보기 확대" + }, "tooltips": { "hideHUD": "HUD 숨기기", "hideHUDTip": "녹화 바를 숨깁니다. 알림 영역 아이콘에서 다시 표시할 수 있습니다.", diff --git a/src/i18n/locales/pt-BR/launch.json b/src/i18n/locales/pt-BR/launch.json index 886615088..893576b96 100644 --- a/src/i18n/locales/pt-BR/launch.json +++ b/src/i18n/locales/pt-BR/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Prévia da câmera", + "show": "Mostrar prévia da câmera", + "hide": "Ocultar prévia da câmera", + "smaller": "Diminuir prévia", + "larger": "Aumentar prévia" + }, "tooltips": { "hideHUD": "Ocultar HUD", "hideHUDTip": "Ocultar a barra. Mostre-a de novo pelo ícone da bandeja do sistema.", diff --git a/src/i18n/locales/ru/launch.json b/src/i18n/locales/ru/launch.json index cdc103c0b..a34023430 100644 --- a/src/i18n/locales/ru/launch.json +++ b/src/i18n/locales/ru/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Предпросмотр камеры", + "show": "Показать предпросмотр камеры", + "hide": "Скрыть предпросмотр камеры", + "smaller": "Уменьшить предпросмотр", + "larger": "Увеличить предпросмотр" + }, "tooltips": { "hideHUD": "Скрыть HUD", "hideHUDTip": "Скрыть панель. Вернуть её можно через значок в области уведомлений.", diff --git a/src/i18n/locales/tr/launch.json b/src/i18n/locales/tr/launch.json index 530157dd2..929709e36 100644 --- a/src/i18n/locales/tr/launch.json +++ b/src/i18n/locales/tr/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Kamera önizlemesi", + "show": "Kamera önizlemesini göster", + "hide": "Kamera önizlemesini gizle", + "smaller": "Önizlemeyi küçült", + "larger": "Önizlemeyi büyüt" + }, "tooltips": { "hideHUD": "Kontrol panelini gizle", "hideHUDTip": "Araç çubuğunu gizle. Bildirim alanındaki simgeden yeniden gösterebilirsiniz.", diff --git a/src/i18n/locales/vi/launch.json b/src/i18n/locales/vi/launch.json index 53f452e27..fe3ce29fa 100644 --- a/src/i18n/locales/vi/launch.json +++ b/src/i18n/locales/vi/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Xem trước camera", + "show": "Hiện bản xem trước camera", + "hide": "Ẩn bản xem trước camera", + "smaller": "Thu nhỏ bản xem trước", + "larger": "Phóng to bản xem trước" + }, "tooltips": { "hideHUD": "Ẩn HUD", "hideHUDTip": "Ẩn thanh ghi hình. Hiện lại từ biểu tượng ở khay hệ thống.", diff --git a/src/i18n/locales/zh-CN/launch.json b/src/i18n/locales/zh-CN/launch.json index a4bbf2c57..b53025f9c 100644 --- a/src/i18n/locales/zh-CN/launch.json +++ b/src/i18n/locales/zh-CN/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "摄像头预览", + "show": "显示摄像头预览", + "hide": "隐藏摄像头预览", + "smaller": "缩小预览", + "larger": "放大预览" + }, "tooltips": { "hideHUD": "隐藏录制栏", "hideHUDTip": "隐藏录制栏，可通过系统托盘图标重新显示", diff --git a/src/i18n/locales/zh-TW/launch.json b/src/i18n/locales/zh-TW/launch.json index 2e460592e..03d134c20 100644 --- a/src/i18n/locales/zh-TW/launch.json +++ b/src/i18n/locales/zh-TW/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "攝影機預覽", + "show": "顯示攝影機預覽", + "hide": "隱藏攝影機預覽", + "smaller": "縮小預覽", + "larger": "放大預覽" + }, "tooltips": { "hideHUD": "隱藏控制面板", "hideHUDTip": "隱藏控制面板。可從系統匣圖示重新顯示。", diff --git a/src/lib/cameraPreview.test.ts b/src/lib/cameraPreview.test.ts new file mode 100644 index 000000000..c98f8548f --- /dev/null +++ b/src/lib/cameraPreview.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { cameraPreviewBounds, resizedCameraPreviewBounds } from "./cameraPreview"; + +const workArea = { x: 0, y: 25, width: 1512, height: 900 }; +const bar = { x: 300, y: 840, width: 900, height: 64 }; + +describe("camera preview placement", () => { + it("starts just above the recording bar's right end", () => { + const preview = cameraPreviewBounds(workArea, bar); + expect(preview.x + preview.width).toBe(bar.x + bar.width); + expect(preview.y + preview.height).toBe(bar.y - 8); + }); + + it("keeps the bottom centre fixed when resized near the bar", () => { + const before = cameraPreviewBounds(workArea, bar); + const after = resizedCameraPreviewBounds(before, workArea, 320); + expect(after.width).toBe(336); + expect(after.height).toBe(336); + expect(after.y + after.height).toBe(before.y + before.height); + expect(after.x + after.width / 2).toBe(before.x + before.width / 2); + }); + + it("keeps a preview on-screen on negative-origin displays and near the screen edge", () => { + const display = { x: -1280, y: -100, width: 1280, height: 720 }; + const preview = resizedCameraPreviewBounds( + { x: -1280, y: -100, width: 176, height: 176 }, + display, + 320, + ); + expect(preview.x).toBeGreaterThanOrEqual(display.x); + expect(preview.y).toBeGreaterThanOrEqual(display.y); + expect(preview.x + preview.width).toBeLessThanOrEqual(0); + expect(preview.y + preview.height).toBeLessThanOrEqual(620); + }); +}); diff --git a/src/lib/cameraPreview.ts b/src/lib/cameraPreview.ts new file mode 100644 index 000000000..613d0e410 --- /dev/null +++ b/src/lib/cameraPreview.ts @@ -0,0 +1,37 @@ +export const CAMERA_PREVIEW_SIZES = [160, 224, 320] as const; +export const CAMERA_PREVIEW_DEFAULT_SIZE = 224; +const WINDOW_PADDING = 16; + +type Rect = { x: number; y: number; width: number; height: number }; + +function clampBounds(x: number, y: number, diameter: number, workArea: Rect): Rect { + const size = Math.min(diameter + WINDOW_PADDING, workArea.width, workArea.height); + return { + x: Math.round(Math.max(workArea.x, Math.min(x, workArea.x + workArea.width - size))), + y: Math.round(Math.max(workArea.y, Math.min(y, workArea.y + workArea.height - size))), + width: size, + height: size, + }; +} + +/** Place the circle just above the bar's right end, including the transparent window padding. */ +export function cameraPreviewBounds(workArea: Rect, bar?: Rect): Rect { + const size = CAMERA_PREVIEW_DEFAULT_SIZE + WINDOW_PADDING; + return clampBounds( + bar ? bar.x + bar.width - size : workArea.x + workArea.width / 2 + 160, + (bar?.y ?? workArea.y + workArea.height - 84) - size - 8, + CAMERA_PREVIEW_DEFAULT_SIZE, + workArea, + ); +} + +/** Grow upward from the bottom centre, retaining its distance from the recording bar. */ +export function resizedCameraPreviewBounds(bounds: Rect, workArea: Rect, diameter: number): Rect { + const size = diameter + WINDOW_PADDING; + return clampBounds( + bounds.x + (bounds.width - size) / 2, + bounds.y + bounds.height - size, + diameter, + workArea, + ); +} diff --git a/technical-documentation/architecture/recording.md b/technical-documentation/architecture/recording.md index 2f643ec22..7e70f7819 100644 --- a/technical-documentation/architecture/recording.md +++ b/technical-documentation/architecture/recording.md @@ -22,6 +22,33 @@ The HUD starts and controls a recording. The source selector chooses a display o Electron applies `setContentProtection(true)` to the HUD window (`applyContentProtection` in `electron/windows.ts`). This keeps the controller out of captures and also makes it invisible in screenshots. It is skipped on macOS 26 and later, which never displays a protected window, and on Windows before 11 22H2 (build 22621), where it leaves the click-through HUD unable to take a click (#1105). On those Windows versions the HUD can appear in recordings; on macOS the ScreenCaptureKit helper still excludes it by window id. For a testing session only, `OPENSCREEN_DISABLE_CONTENT_PROTECTION=1` disables the protection; the code warns that the HUD then appears in captures. The tray icon is the reliable way to refocus OpenScreen or stop a recording when the click-through HUD is not convenient or is not visible. +## Presenter camera preview (macOS) + +When the HUD camera is enabled, a draggable circular self-view floats above other windows. +It starts just above the bar’s right end, centres the camera image to fill the circle +without stretching or letterboxing, and offers hover/focus minus/plus controls for +160, 224 and 320px diameters. +Resizing retains the window id and anchors its bottom centre, clamped to its display. +Camera constraints explicitly prefer the landscape aspect ratio of the selected quality +preset; width/height preferences alone can select a portrait mode on some cameras. +The eye button on the HUD and the circle's hover/focus hide button toggle the preview without +changing camera recording. The eye button stays visible but disabled when the camera is off. +Audio preference broadcasts retain the resolved camera selection unless the stored camera +identity changes, so toggling audio does not reopen its stream. This first implementation is macOS-only; Windows and Linux keep +the existing recording controls. + +`useScreenRecorder` exposes its existing camera stream to `useFloatingCameraPreview`, which +publishes 320px JPEG frames at up to 10 fps with one IPC request in flight. The preview never +opens a second camera session or stops the recorder's tracks. The main process accepts frames +only from the owning HUD and limits the payload size. `CameraPreviewController` owns the +auxiliary window and destroys it when the camera is disabled or its HUD is destroyed. + +Hiding the circle retains the window and its capture exclusion id. Both macOS source-picker +and direct-display requests exclude it alongside the HUD and Notes. The direct recorder +includes hidden windows when resolving exclusions, keeping the circle excluded even when +first shown during a take. The native helper must be rebuilt after this change. The window +uses the same content-protection and Spaces policy as the HUD. + ## Capture backends | Platform | Backend | Code | Produces | diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 340df1480..94935484a 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -994,3 +994,7 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-17 | PR #1319 (`2236580`), artifact `openscreen-linux` of build run 39046512213, AppImage run directly. Reference: the `v2.1.0` AppImage from the GitHub release | Ubuntu 24.04.4, GNOME 46 Wayland, AMD Radeon 610M (RADV); `[d3d] … -> backend Hardware`, no CPU fallback | **Partial — fix slice only, no defect** | Same MCP pass and project as the rows above, on the hardware Vulkan backend. `orbit` (12 frames) and `left` (4 s, 6 s): the whole card with four rounded corners, no flat cut. `v2.1.0` reproduces the bug (top and bottom edges flat, corners cut from 1.5 s). `none` and the flat frames at 1.0 and 9.5 s are byte-identical to `v2.1.0` (equal md5). **Not covered:** the editor preview; `right`; a full MP4 export. | | 2026-10-17 | `v2.1.1-rc.1`, `release/v2.1.1` @ `6ec38bf` (= `v2.1.0` + #1319), assets of the GitHub pre-release (build run 39069024444). Windows: `Openscreen.Setup.2.1.1-rc.1-x64.exe`, sha512 matched `latest.yml`, unpacked with 7-Zip and run in place, version 2.1.1. macOS: `Openscreen-macOS-Apple-Silicon-2.1.1-rc.1.dmg`, sha256 matched the asset digest; `spctl` accepted, Notarized Developer ID, on the DMG and the app; ticket stapled to the DMG only, as on 2.1.0-rc.6; `CFBundleShortVersionString` 2.1.1-rc.1. Linux: `Openscreen-Linux-2.1.1-rc.1.AppImage`, sha512 matched `latest-linux.yml` | Windows 11 Home 26200, RTX 4070 Ti; macOS 27.0.1 (26A434), Mac mini M1 8 GB; Ubuntu 24.04.4, GNOME 46 Wayland, Radeon 610M, hardware Vulkan backend | **Partial — patch pass toward stable: no defect on any of the three** | Scoped to what the patch changes (the compositor geometry of a tilted filled card), in MCP over stdio with an isolated `--user-data-dir`, on the project of the PR rows above. On all three: `orbit` (12 frames) and `left` show the whole card with four rounded corners and no flat cut; `none` is a flat zoom in a rounded card (byte-identical to `v2.1.0` on Windows and Linux; on Windows every checked frame is byte-identical to the PR build). `exportVideo` at 1080p30: 10.00 s, 1920×1080 H.264 with an AAC track, decodes without error, and its frame at 5 s shows the tilted card with rounded corners. Before the cut, on a dev build of the PR: the four device frames, dual-frame and vertical-stack, under `orbit`, `left` and `none`, rendered byte-identical to `v2.1.0` (49 of 49 frames). **Not covered:** recording, playback and the editor preview, which the patch does not touch; the rest of the checklist (the `v2.1.0-rc.6` rows). | | 2026-10-19 | `v2.1.2-rc.1`, `release/v2.1.2` @ `c4ef8fd` (= `v2.1.1` + #1344), assets of the GitHub pre-release (build run 39224311283). Windows: `Openscreen.Setup.2.1.2-rc.1-x64.exe` unpacked with 7-Zip and run in place. macOS: `Openscreen-Mac-arm64-2.1.2-rc.1.zip`, run in place. Linux: the AppImage, run directly with `--no-sandbox` | Windows 11 Home 26200; macOS 27.0.1, Mac mini M1; Ubuntu 24.04 GNOME Wayland (Xwayland `:0`) | **Partial: fix slice only, no defect** | `openscreen mcp` over stdio, scripted, with a fresh `--user-data-dir` per run. On all three platforms it answered `initialize`, and `usage-ping.json` held today's day, which is saved only once PostHog accepted the event. With `DO_NOT_TRACK=1` no state file was written. stdout held only JSON-RPC on macOS and Linux. On Windows it starts with one `\r\n` that Electron 41 itself writes during browser-process startup, before `main.js` loads: a bare `app.exit(0)` script prints it too, `ELECTRON_RUN_AS_NODE` does not. It predates this change, and the MCP SDK clients skip it with one parse error. Not covered: everything else in this file, including recording, editing, export, the GUI ping and the tray toggle. These runs sent eight real `surface: mcp` events to PostHog on 19/10. | +| 2026-10-16 | dev `main` @ `b149c55` + uncommitted #1278 camera preview; Electron 41.2.1; capture helper compiled from this working tree using `swiftc` + macOS 26.5 SDK, SHA256 `06f142f075e4dd776b653f174379b2795a60d750557655f9bca6e2d2596c8f99`; remaining native payload copied as a complete directory from installed OpenScreen | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — camera preview slice passed** | Driven through `mcp__cua_repl` against the Electron app, with the legacy source picker and content protection disabled for UI visibility. Verified live camera frames in a circle with no card/header, HUD show/hide and circle hide without disabling camera capture, and show/hide while recording. Exercised OS dragging; final position was not independently measured. Stopping closed the preview and opened the editor; screen MP4 and separate webcam WebM were finalized. A second take started with the preview hidden, then showed it during capture using the rebuilt helper; a decoded late screen frame contains no floating circle (sample `/tmp/openscreen-camera-preview-capture.png`). This sample is not an exhaustive frame-by-frame exclusion test. Test takes: `recording-1792129915224` (borrowed helper), `recording-1792130142748` (rebuilt helper). **Skipped:** Apple's picker, full-screen Spaces, restart/cancel, disconnect/reconnect on physical hardware, export, all other checklist sections, Windows and Linux (implementation is macOS-first). SwiftPM cannot start with this machine's Command Line Tools (`BuildServerProtocol` dyld error); direct `swiftc` compilation passed, but XCTest is unavailable without full Xcode. | +| 2026-10-16 | dev `main` @ `b149c55` + #1278 audio-toggle and persistent preview-control fixes; Electron 41.2.1 | macOS 26.7, Apple Silicon | **Partial — HUD follow-up slice passed** | `mcp__cua_repl`: toggled system audio on/off and microphone off/on with camera and preview enabled; restored original audio preferences. Camera off leaves Show camera preview visible and disabled; camera on re-enables it. Regression test reproduces and fixes audio snapshots reopening the resolved default camera, asserting identical stream and no stopped tracks across both audio changes; explicit camera changes and resets still apply. **Skipped:** recording, export, all remaining checklist sections, and Windows/Linux for this follow-up. | +| 2026-10-16 | dev `main` @ `b149c55` + #1278 preview framing, placement and resizing | macOS 26.7, Apple Silicon | **Partial — preview UI slice passed** | `mcp__cua_repl`: inspected live fitted camera frames (portrait source, with black space at the sides instead of a zoomed crop), increased to 320px, decreased through 224px to 160px, verified limit buttons disable, and restored 224px. Window sizes changed visibly while remaining circular. Default placement above the HUD and resize anchoring are covered by geometry tests; their screen coordinates were not independently measured in the desktop pass. **Skipped:** recording/export, Apple’s picker, display disconnection, Windows/Linux, and all other checklist sections. | +| 2026-10-16 | dev `main` @ `b149c55` + #1278 normal camera framing follow-up | macOS 26.7, MacBook Pro Camera | **Partial — preview framing slice passed** | `mcp__cua_repl`: verified the circle fills with centred, unstretched live video and no side bars after preferring the preset’s landscape aspect ratio and removing letterboxing. The earlier webcam sidecar was 1080×1920 despite landscape width/height preferences. Temporarily hid and restored the recording bar for inspection, then left the camera and preview enabled. **Skipped:** recording/export, other cameras, Windows/Linux and all other checklist sections. | From 1e265e9f0cecb20f12f03914f077a27e2208cbf7 Mon Sep 17 00:00:00 2001 From: Satyam Raj Date: Fri, 16 Oct 2026 14:50:53 +0530 Subject: [PATCH 2/6] refactor: scope camera preview changes to macOS --- .../useScreenRecorder.cameraPrefs.test.tsx | 2 ++ ...eenRecorder.nativeMacStartWarning.test.tsx | 7 ++-- ...eScreenRecorder.nativeStopFailure.test.tsx | 7 ++-- .../useScreenRecorder.prefsRace.test.tsx | 7 ++-- .../useScreenRecorder.restartFailure.test.tsx | 7 ++-- src/hooks/useScreenRecorder.ts | 4 +++ src/hooks/webcamCaptureTarget.test.ts | 4 --- src/hooks/webcamCaptureTarget.ts | 2 -- .../architecture/recording.md | 34 ++++++------------- .../testing/manual-e2e-checklist.md | 5 +-- 10 files changed, 29 insertions(+), 50 deletions(-) diff --git a/src/hooks/useScreenRecorder.cameraPrefs.test.tsx b/src/hooks/useScreenRecorder.cameraPrefs.test.tsx index b7f4f4129..d60d1a56d 100644 --- a/src/hooks/useScreenRecorder.cameraPrefs.test.tsx +++ b/src/hooks/useScreenRecorder.cameraPrefs.test.tsx @@ -34,10 +34,12 @@ const initial: Prefs = { afterEach(() => { cleanup(); + window.history.replaceState({}, "", "/"); vi.restoreAllMocks(); }); function setup() { + window.history.replaceState({}, "", "/?windowType=hud-overlay"); let publish: ((prefs: Prefs) => void) | undefined; const stops: Array> = []; const getUserMedia = vi.fn(async () => { diff --git a/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx b/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx index 65176bcb4..817a008f9 100644 --- a/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx +++ b/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx @@ -2,10 +2,9 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => { - const translate = (key: string) => key; - return { useScopedT: () => translate }; -}); +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: () => (key: string) => key, +})); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx b/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx index caad8e673..928ac5886 100644 --- a/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx +++ b/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx @@ -2,10 +2,9 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => { - const translate = (key: string) => key; - return { useScopedT: () => translate }; -}); +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: () => (key: string) => key, +})); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.prefsRace.test.tsx b/src/hooks/useScreenRecorder.prefsRace.test.tsx index edb9c5859..ae84c2cd7 100644 --- a/src/hooks/useScreenRecorder.prefsRace.test.tsx +++ b/src/hooks/useScreenRecorder.prefsRace.test.tsx @@ -2,10 +2,9 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => { - const translate = (key: string) => key; - return { useScopedT: () => translate }; -}); +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: () => (key: string) => key, +})); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.restartFailure.test.tsx b/src/hooks/useScreenRecorder.restartFailure.test.tsx index 764338e5b..63c78d916 100644 --- a/src/hooks/useScreenRecorder.restartFailure.test.tsx +++ b/src/hooks/useScreenRecorder.restartFailure.test.tsx @@ -2,10 +2,9 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => { - const translate = (key: string) => key; - return { useScopedT: () => translate }; -}); +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: () => (key: string) => key, +})); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 37264921b..3e432353d 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -317,7 +317,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // The HUD resolves a stored default (null) to a concrete device locally. // An audio-only save republishes that same null: applying it again would // reset the resolved device, stop its stream, and visibly blink the preview. + const isMacHud = + window.electronAPI?.getPlatform?.() === "darwin" && + new URLSearchParams(window.location.search).get("windowType") === "hud-overlay"; if ( + !isMacHud || !appliedCameraSelection || appliedCameraSelection.deviceId !== deviceId || appliedCameraSelection.deviceName !== deviceName diff --git a/src/hooks/webcamCaptureTarget.test.ts b/src/hooks/webcamCaptureTarget.test.ts index 0de5da047..25f83efdd 100644 --- a/src/hooks/webcamCaptureTarget.test.ts +++ b/src/hooks/webcamCaptureTarget.test.ts @@ -16,12 +16,10 @@ describe("webcamVideoConstraints", () => { expect(webcamVideoConstraints(undefined, "2160p")).toMatchObject({ width: { ideal: 3840 }, height: { ideal: 2160 }, - aspectRatio: { ideal: 16 / 9 }, }); expect(webcamVideoConstraints(undefined, "1080p")).toMatchObject({ width: { ideal: 1920 }, height: { ideal: 1080 }, - aspectRatio: { ideal: 16 / 9 }, }); }); @@ -35,8 +33,6 @@ describe("webcamVideoConstraints", () => { expect(constraints.height).not.toHaveProperty("exact"); expect(constraints.width).not.toHaveProperty("min"); expect(constraints.height).not.toHaveProperty("min"); - expect(constraints.aspectRatio).not.toHaveProperty("exact"); - expect(constraints.aspectRatio).not.toHaveProperty("min"); }); it("still pins the chosen device and caps the frame rate", () => { diff --git a/src/hooks/webcamCaptureTarget.ts b/src/hooks/webcamCaptureTarget.ts index fa960469f..906412b19 100644 --- a/src/hooks/webcamCaptureTarget.ts +++ b/src/hooks/webcamCaptureTarget.ts @@ -74,8 +74,6 @@ export function webcamVideoConstraints( ...(deviceId ? { deviceId: { exact: deviceId } } : {}), width: { ideal: preset.width }, height: { ideal: preset.height }, - // Width/height alone can select a portrait mode on cameras offering both orientations. - aspectRatio: { ideal: preset.width / preset.height }, frameRate: { ideal: WEBCAM_TARGET_FRAME_RATE, max: WEBCAM_TARGET_FRAME_RATE }, }; } diff --git a/technical-documentation/architecture/recording.md b/technical-documentation/architecture/recording.md index 7e70f7819..41d3d6470 100644 --- a/technical-documentation/architecture/recording.md +++ b/technical-documentation/architecture/recording.md @@ -24,30 +24,16 @@ Electron applies `setContentProtection(true)` to the HUD window (`applyContentPr ## Presenter camera preview (macOS) -When the HUD camera is enabled, a draggable circular self-view floats above other windows. -It starts just above the bar’s right end, centres the camera image to fill the circle -without stretching or letterboxing, and offers hover/focus minus/plus controls for -160, 224 and 320px diameters. -Resizing retains the window id and anchors its bottom centre, clamped to its display. -Camera constraints explicitly prefer the landscape aspect ratio of the selected quality -preset; width/height preferences alone can select a portrait mode on some cameras. -The eye button on the HUD and the circle's hover/focus hide button toggle the preview without -changing camera recording. The eye button stays visible but disabled when the camera is off. -Audio preference broadcasts retain the resolved camera selection unless the stored camera -identity changes, so toggling audio does not reopen its stream. This first implementation is macOS-only; Windows and Linux keep -the existing recording controls. - -`useScreenRecorder` exposes its existing camera stream to `useFloatingCameraPreview`, which -publishes 320px JPEG frames at up to 10 fps with one IPC request in flight. The preview never -opens a second camera session or stops the recorder's tracks. The main process accepts frames -only from the owning HUD and limits the payload size. `CameraPreviewController` owns the -auxiliary window and destroys it when the camera is disabled or its HUD is destroyed. - -Hiding the circle retains the window and its capture exclusion id. Both macOS source-picker -and direct-display requests exclude it alongside the HUD and Notes. The direct recorder -includes hidden windows when resolving exclusions, keeping the circle excluded even when -first shown during a take. The native helper must be rebuilt after this change. The window -uses the same content-protection and Spaces policy as the HUD. +The HUD camera has a draggable circular self-view with show/hide and three size choices. +Its preview button stays visible but disabled when the camera is off. The preview shares +`useScreenRecorder`'s stream and stops with its owning HUD; it does not acquire another camera. +`useFloatingCameraPreview` publishes small JPEG frames to the auxiliary window, with bounded +payloads and one IPC request in flight. Audio preference broadcasts in the macOS HUD preserve +its resolved camera selection so the preview does not blink when audio is toggled. + +Hiding or resizing retains the window id. macOS capture excludes that id alongside the HUD +and Notes, including windows hidden when capture starts. This requires rebuilding the native +helper. Windows and Linux keep their existing recording controls. ## Capture backends diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 94935484a..479830b43 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -994,7 +994,4 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-17 | PR #1319 (`2236580`), artifact `openscreen-linux` of build run 39046512213, AppImage run directly. Reference: the `v2.1.0` AppImage from the GitHub release | Ubuntu 24.04.4, GNOME 46 Wayland, AMD Radeon 610M (RADV); `[d3d] … -> backend Hardware`, no CPU fallback | **Partial — fix slice only, no defect** | Same MCP pass and project as the rows above, on the hardware Vulkan backend. `orbit` (12 frames) and `left` (4 s, 6 s): the whole card with four rounded corners, no flat cut. `v2.1.0` reproduces the bug (top and bottom edges flat, corners cut from 1.5 s). `none` and the flat frames at 1.0 and 9.5 s are byte-identical to `v2.1.0` (equal md5). **Not covered:** the editor preview; `right`; a full MP4 export. | | 2026-10-17 | `v2.1.1-rc.1`, `release/v2.1.1` @ `6ec38bf` (= `v2.1.0` + #1319), assets of the GitHub pre-release (build run 39069024444). Windows: `Openscreen.Setup.2.1.1-rc.1-x64.exe`, sha512 matched `latest.yml`, unpacked with 7-Zip and run in place, version 2.1.1. macOS: `Openscreen-macOS-Apple-Silicon-2.1.1-rc.1.dmg`, sha256 matched the asset digest; `spctl` accepted, Notarized Developer ID, on the DMG and the app; ticket stapled to the DMG only, as on 2.1.0-rc.6; `CFBundleShortVersionString` 2.1.1-rc.1. Linux: `Openscreen-Linux-2.1.1-rc.1.AppImage`, sha512 matched `latest-linux.yml` | Windows 11 Home 26200, RTX 4070 Ti; macOS 27.0.1 (26A434), Mac mini M1 8 GB; Ubuntu 24.04.4, GNOME 46 Wayland, Radeon 610M, hardware Vulkan backend | **Partial — patch pass toward stable: no defect on any of the three** | Scoped to what the patch changes (the compositor geometry of a tilted filled card), in MCP over stdio with an isolated `--user-data-dir`, on the project of the PR rows above. On all three: `orbit` (12 frames) and `left` show the whole card with four rounded corners and no flat cut; `none` is a flat zoom in a rounded card (byte-identical to `v2.1.0` on Windows and Linux; on Windows every checked frame is byte-identical to the PR build). `exportVideo` at 1080p30: 10.00 s, 1920×1080 H.264 with an AAC track, decodes without error, and its frame at 5 s shows the tilted card with rounded corners. Before the cut, on a dev build of the PR: the four device frames, dual-frame and vertical-stack, under `orbit`, `left` and `none`, rendered byte-identical to `v2.1.0` (49 of 49 frames). **Not covered:** recording, playback and the editor preview, which the patch does not touch; the rest of the checklist (the `v2.1.0-rc.6` rows). | | 2026-10-19 | `v2.1.2-rc.1`, `release/v2.1.2` @ `c4ef8fd` (= `v2.1.1` + #1344), assets of the GitHub pre-release (build run 39224311283). Windows: `Openscreen.Setup.2.1.2-rc.1-x64.exe` unpacked with 7-Zip and run in place. macOS: `Openscreen-Mac-arm64-2.1.2-rc.1.zip`, run in place. Linux: the AppImage, run directly with `--no-sandbox` | Windows 11 Home 26200; macOS 27.0.1, Mac mini M1; Ubuntu 24.04 GNOME Wayland (Xwayland `:0`) | **Partial: fix slice only, no defect** | `openscreen mcp` over stdio, scripted, with a fresh `--user-data-dir` per run. On all three platforms it answered `initialize`, and `usage-ping.json` held today's day, which is saved only once PostHog accepted the event. With `DO_NOT_TRACK=1` no state file was written. stdout held only JSON-RPC on macOS and Linux. On Windows it starts with one `\r\n` that Electron 41 itself writes during browser-process startup, before `main.js` loads: a bare `app.exit(0)` script prints it too, `ELECTRON_RUN_AS_NODE` does not. It predates this change, and the MCP SDK clients skip it with one parse error. Not covered: everything else in this file, including recording, editing, export, the GUI ping and the tray toggle. These runs sent eight real `surface: mcp` events to PostHog on 19/10. | -| 2026-10-16 | dev `main` @ `b149c55` + uncommitted #1278 camera preview; Electron 41.2.1; capture helper compiled from this working tree using `swiftc` + macOS 26.5 SDK, SHA256 `06f142f075e4dd776b653f174379b2795a60d750557655f9bca6e2d2596c8f99`; remaining native payload copied as a complete directory from installed OpenScreen | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — camera preview slice passed** | Driven through `mcp__cua_repl` against the Electron app, with the legacy source picker and content protection disabled for UI visibility. Verified live camera frames in a circle with no card/header, HUD show/hide and circle hide without disabling camera capture, and show/hide while recording. Exercised OS dragging; final position was not independently measured. Stopping closed the preview and opened the editor; screen MP4 and separate webcam WebM were finalized. A second take started with the preview hidden, then showed it during capture using the rebuilt helper; a decoded late screen frame contains no floating circle (sample `/tmp/openscreen-camera-preview-capture.png`). This sample is not an exhaustive frame-by-frame exclusion test. Test takes: `recording-1792129915224` (borrowed helper), `recording-1792130142748` (rebuilt helper). **Skipped:** Apple's picker, full-screen Spaces, restart/cancel, disconnect/reconnect on physical hardware, export, all other checklist sections, Windows and Linux (implementation is macOS-first). SwiftPM cannot start with this machine's Command Line Tools (`BuildServerProtocol` dyld error); direct `swiftc` compilation passed, but XCTest is unavailable without full Xcode. | -| 2026-10-16 | dev `main` @ `b149c55` + #1278 audio-toggle and persistent preview-control fixes; Electron 41.2.1 | macOS 26.7, Apple Silicon | **Partial — HUD follow-up slice passed** | `mcp__cua_repl`: toggled system audio on/off and microphone off/on with camera and preview enabled; restored original audio preferences. Camera off leaves Show camera preview visible and disabled; camera on re-enables it. Regression test reproduces and fixes audio snapshots reopening the resolved default camera, asserting identical stream and no stopped tracks across both audio changes; explicit camera changes and resets still apply. **Skipped:** recording, export, all remaining checklist sections, and Windows/Linux for this follow-up. | -| 2026-10-16 | dev `main` @ `b149c55` + #1278 preview framing, placement and resizing | macOS 26.7, Apple Silicon | **Partial — preview UI slice passed** | `mcp__cua_repl`: inspected live fitted camera frames (portrait source, with black space at the sides instead of a zoomed crop), increased to 320px, decreased through 224px to 160px, verified limit buttons disable, and restored 224px. Window sizes changed visibly while remaining circular. Default placement above the HUD and resize anchoring are covered by geometry tests; their screen coordinates were not independently measured in the desktop pass. **Skipped:** recording/export, Apple’s picker, display disconnection, Windows/Linux, and all other checklist sections. | -| 2026-10-16 | dev `main` @ `b149c55` + #1278 normal camera framing follow-up | macOS 26.7, MacBook Pro Camera | **Partial — preview framing slice passed** | `mcp__cua_repl`: verified the circle fills with centred, unstretched live video and no side bars after preferring the preset’s landscape aspect ratio and removing letterboxing. The earlier webcam sidecar was 1080×1920 despite landscape width/height preferences. Temporarily hid and restored the recording bar for inspection, then left the camera and preview enabled. **Skipped:** recording/export, other cameras, Windows/Linux and all other checklist sections. | +| 2026-10-16 | dev `main` @ `b149c55` + #1278 macOS camera preview; Electron 41.2.1; capture helper built from the working tree with `swiftc` + macOS 26.5 SDK (SHA256 `06f142f075e4dd776b653f174379b2795a60d750557655f9bca6e2d2596c8f99`) | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — preview slice passed** | `mcp__cua_repl`: live circular self-view, show/hide during recording, three sizes and limit buttons, disabled preview button when camera is off, and recording finalization into screen MP4 + webcam WebM. Exercised OS dragging without independently measuring the final position. A take started with the preview hidden and showed it during capture; a decoded late screen frame contained no floating circle. Default placement and resize anchoring are covered by geometry tests. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras, restart/cancel, export, exhaustive frame-by-frame exclusion, Windows/Linux and the remaining checklist. SwiftPM/XCTest unavailable locally; direct helper compilation passed. | From 7763a94bec21f803d1c2d8dd754657af882d8e01 Mon Sep 17 00:00:00 2001 From: Satyam Raj Date: Fri, 16 Oct 2026 15:04:42 +0530 Subject: [PATCH 3/6] fix: clear closed camera preview windows --- electron/cameraPreview.test.ts | 44 +++++++++++++++++++++++++++++++++- electron/cameraPreview.ts | 6 +++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/electron/cameraPreview.test.ts b/electron/cameraPreview.test.ts index b393f7fce..c20e3f8a9 100644 --- a/electron/cameraPreview.test.ts +++ b/electron/cameraPreview.test.ts @@ -9,7 +9,9 @@ function setup() { webContents: { send: vi.fn() }, showInactive: vi.fn(), hide: vi.fn(), - destroy: vi.fn(), + destroy: vi.fn(() => { + overlay.emit("closed"); + }), }); const create = vi.fn(() => overlay as unknown as BrowserWindow); const resize = vi.fn(); @@ -81,6 +83,46 @@ describe("floating camera preview lifecycle", () => { expect(overlay.destroy).not.toHaveBeenCalled(); }); + it("recreates an unexpectedly closed preview and waits for the replacement to be ready", () => { + const { preview, sender, overlay, create } = setup(); + const replacement = setup().overlay; + create + .mockReturnValueOnce(overlay as unknown as BrowserWindow) + .mockReturnValueOnce(replacement as unknown as BrowserWindow); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + overlay.emit("closed"); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledTimes(2); + expect(replacement.showInactive).not.toHaveBeenCalled(); + replacement.emit("ready-to-show"); + expect(replacement.showInactive).toHaveBeenCalledOnce(); + }); + + it("ignores a stale closed event from a previously disposed preview", () => { + const { preview, sender, overlay, create } = setup(); + // Delay closure to exercise a late event after the replacement already exists. + overlay.destroy.mockImplementationOnce(() => undefined); + const replacement = setup().overlay; + create + .mockReturnValueOnce(overlay as unknown as BrowserWindow) + .mockReturnValueOnce(replacement as unknown as BrowserWindow); + preview.configure(sender, true, true); + preview.configure(sender, false, false); + preview.configure(sender, true, true); + replacement.emit("ready-to-show"); + + overlay.emit("closed"); + preview.frame(sender, jpeg); + expect(replacement.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledTimes(2); + expect(replacement.showInactive).toHaveBeenCalledTimes(2); + }); + it("destroys the auxiliary window when the HUD is destroyed", () => { const { preview, sender, owner, overlay } = setup(); preview.configure(sender, true, true); diff --git a/electron/cameraPreview.ts b/electron/cameraPreview.ts index dafa59813..abbc07e39 100644 --- a/electron/cameraPreview.ts +++ b/electron/cameraPreview.ts @@ -30,6 +30,12 @@ export class CameraPreviewController { if (!this.window) { this.window = this.createWindow(); const previewWindow = this.window; + previewWindow.once("closed", () => { + if (this.window !== previewWindow) return; + this.window = null; + this.ready = false; + this.visible = false; + }); this.window.once("ready-to-show", () => { if (this.window !== previewWindow) return; this.ready = true; From 4a9387a1d721394dbce79f459fd394594d3c4b9e Mon Sep 17 00:00:00 2001 From: Satyam Raj Date: Mon, 19 Oct 2026 20:55:34 +0530 Subject: [PATCH 4/6] fix: preserve camera framing in presenter preview --- electron/cameraPreview.test.ts | 18 +++++- electron/cameraPreview.ts | 41 ++++++++++--- electron/electron-env.d.ts | 4 +- electron/ipc/handlers.ts | 4 +- electron/preload.ts | 8 +-- electron/windows.ts | 13 +++- .../launch/FloatingCameraPreview.module.css | 11 ++-- src/components/launch/LaunchWindow.test.tsx | 22 ++++--- src/components/launch/LaunchWindow.tsx | 4 +- src/hooks/useFloatingCameraPreview.test.tsx | 1 + src/hooks/useFloatingCameraPreview.ts | 7 ++- src/lib/cameraPreview.test.ts | 11 +++- src/lib/cameraPreview.ts | 59 +++++++++++++------ .../architecture/recording.md | 5 +- .../testing/manual-e2e-checklist.md | 2 + 15 files changed, 152 insertions(+), 58 deletions(-) diff --git a/electron/cameraPreview.test.ts b/electron/cameraPreview.test.ts index c20e3f8a9..d4ba4ffe4 100644 --- a/electron/cameraPreview.test.ts +++ b/electron/cameraPreview.test.ts @@ -70,7 +70,7 @@ describe("floating camera preview lifecycle", () => { expect(create).toHaveBeenCalledOnce(); }); - it("allows only the preview to resize to a supported diameter without replacing its window", () => { + it("allows only the preview to resize to a supported width without replacing its window", () => { const { preview, sender, overlaySender, resize, create, overlay } = setup(); preview.configure(sender, true, true); preview.resize(sender, 320); @@ -78,11 +78,25 @@ describe("floating camera preview lifecycle", () => { preview.resize(overlaySender, 10000); expect(resize).not.toHaveBeenCalled(); preview.resize(overlaySender, 320); - expect(resize).toHaveBeenCalledWith(overlay, 320); + expect(resize).toHaveBeenCalledWith(overlay, 320, 16 / 9); expect(create).toHaveBeenCalledOnce(); expect(overlay.destroy).not.toHaveBeenCalled(); }); + it("sizes the same window to the delivered frame and preserves that ratio on resize", () => { + const { preview, sender, overlaySender, resize, overlay } = setup(); + preview.configure(sender, true, true); + preview.frame(sender, jpeg, 9 / 16); + expect(resize).toHaveBeenLastCalledWith(overlay, 224, 9 / 16); + preview.resize(overlaySender, 320); + expect(resize).toHaveBeenLastCalledWith(overlay, 320, 9 / 16); + const calls = resize.mock.calls.length; + for (const ratio of [0, -1, Number.NaN, Number.POSITIVE_INFINITY, 100, "16:9"]) { + preview.frame(sender, jpeg, ratio); + } + expect(resize).toHaveBeenCalledTimes(calls); + }); + it("recreates an unexpectedly closed preview and waits for the replacement to be ready", () => { const { preview, sender, overlay, create } = setup(); const replacement = setup().overlay; diff --git a/electron/cameraPreview.ts b/electron/cameraPreview.ts index abbc07e39..8567e48e2 100644 --- a/electron/cameraPreview.ts +++ b/electron/cameraPreview.ts @@ -1,17 +1,27 @@ import type { BrowserWindow, WebContents } from "electron"; -import { CAMERA_PREVIEW_SIZES } from "../src/lib/cameraPreview"; +import { + CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO, + CAMERA_PREVIEW_DEFAULT_SIZE, + CAMERA_PREVIEW_SIZES, +} from "../src/lib/cameraPreview"; /** One stable window per HUD: hiding it must not change the native capture exclusion id. */ export class CameraPreviewController { private window: BrowserWindow | null = null; private owner: WebContents | null = null; private visible = false; + private width: number = CAMERA_PREVIEW_DEFAULT_SIZE; + private aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO; private ready = false; private onOwnerDestroyed: (() => void) | null = null; constructor( private readonly createWindow: () => BrowserWindow, - private readonly resizeWindow: (window: BrowserWindow, diameter: number) => void, + private readonly resizeWindow: ( + window: BrowserWindow, + width: number, + aspectRatio: number, + ) => void, ) {} configure(owner: WebContents, enabled: boolean, visible: boolean): void { @@ -39,6 +49,7 @@ export class CameraPreviewController { this.window.once("ready-to-show", () => { if (this.window !== previewWindow) return; this.ready = true; + this.resizeWindow(previewWindow, this.width, this.aspectRatio); if (this.visible) this.window?.showInactive(); }); } @@ -54,17 +65,18 @@ export class CameraPreviewController { this.owner?.send("camera-preview:hidden"); } - resize(sender: WebContents, diameter: unknown): void { + resize(sender: WebContents, width: unknown): void { if ( sender !== this.window?.webContents || - typeof diameter !== "number" || - !CAMERA_PREVIEW_SIZES.some((size) => size === diameter) + typeof width !== "number" || + !CAMERA_PREVIEW_SIZES.some((size) => size === width) ) return; - this.resizeWindow(this.window, diameter); + this.width = width; + this.resizeWindow(this.window, this.width, this.aspectRatio); } - frame(sender: WebContents, jpeg: unknown): void { + frame(sender: WebContents, jpeg: unknown, aspectRatio?: unknown): void { if (sender !== this.owner || !this.visible || !this.window) return; // A 320px JPEG is usually under 20KB. Never relay arbitrary URLs or unbounded IPC data. if ( @@ -73,6 +85,19 @@ export class CameraPreviewController { !jpeg.startsWith("data:image/jpeg;base64,") ) return; + if (aspectRatio !== undefined) { + if ( + typeof aspectRatio !== "number" || + !Number.isFinite(aspectRatio) || + aspectRatio < 1 / 8 || + aspectRatio > 8 + ) + return; + if (this.aspectRatio !== aspectRatio) { + this.aspectRatio = aspectRatio; + this.resizeWindow(this.window, this.width, aspectRatio); + } + } this.window.webContents.send("camera-preview:frame", jpeg); } @@ -86,5 +111,7 @@ export class CameraPreviewController { this.window = null; this.owner = null; this.visible = false; + this.width = CAMERA_PREVIEW_DEFAULT_SIZE; + this.aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO; } } diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index aea81b57c..71045aa7a 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -59,8 +59,8 @@ interface Window { /** Name of the last source picked in Apple's picker, for display only: it is not selected. */ getLastPickedSource?: () => Promise; configureCameraPreview?: (enabled: boolean, visible: boolean) => Promise; - sendCameraPreviewFrame?: (jpeg: string) => Promise; - resizeCameraPreview?: (diameter: number) => Promise; + sendCameraPreviewFrame?: (jpeg: string, aspectRatio: number) => Promise; + resizeCameraPreview?: (width: number) => Promise; hideCameraPreview?: () => Promise; onCameraPreviewFrame?: (callback: (jpeg: string) => void) => () => void; onCameraPreviewHidden?: (callback: () => void) => () => void; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index b1fdd8dbe..e0a2e4f2d 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -2489,8 +2489,8 @@ export function registerIpcHandlers( return; cameraPreview.configure(event.sender, enabled === true, visible === true); }); - ipcMain.handle("camera-preview:frame", (event, jpeg: unknown) => - cameraPreview.frame(event.sender, jpeg), + ipcMain.handle("camera-preview:frame", (event, jpeg: unknown, aspectRatio: unknown) => + cameraPreview.frame(event.sender, jpeg, aspectRatio), ); ipcMain.handle("camera-preview:hide", (event) => cameraPreview.hide(event.sender)); ipcMain.handle("camera-preview:resize", (event, diameter: unknown) => diff --git a/electron/preload.ts b/electron/preload.ts index c80e50d4f..c0d6c15f4 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -171,10 +171,10 @@ contextBridge.exposeInMainWorld("electronAPI", { }, configureCameraPreview: (enabled: boolean, visible: boolean): Promise => ipcRenderer.invoke("camera-preview:configure", enabled, visible), - sendCameraPreviewFrame: (jpeg: string): Promise => - ipcRenderer.invoke("camera-preview:frame", jpeg), - resizeCameraPreview: (diameter: number): Promise => - ipcRenderer.invoke("camera-preview:resize", diameter), + sendCameraPreviewFrame: (jpeg: string, aspectRatio: number): Promise => + ipcRenderer.invoke("camera-preview:frame", jpeg, aspectRatio), + resizeCameraPreview: (width: number): Promise => + ipcRenderer.invoke("camera-preview:resize", width), hideCameraPreview: (): Promise => ipcRenderer.invoke("camera-preview:hide"), onCameraPreviewFrame: (callback: (jpeg: string) => void) => { const listener = (_event: Electron.IpcRendererEvent, jpeg: string) => callback(jpeg); diff --git a/electron/windows.ts b/electron/windows.ts index 3557ac0de..9362a4277 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -918,10 +918,19 @@ export function createCameraPreviewWindow(): BrowserWindow { return win; } -export function resizeCameraPreviewWindow(win: BrowserWindow, diameter: number): void { +export function resizeCameraPreviewWindow( + win: BrowserWindow, + width: number, + aspectRatio: number, +): void { const bounds = win.getBounds(); win.setBounds( - resizedCameraPreviewBounds(bounds, screen.getDisplayMatching(bounds).workArea, diameter), + resizedCameraPreviewBounds( + bounds, + screen.getDisplayMatching(bounds).workArea, + width, + aspectRatio, + ), ); } diff --git a/src/components/launch/FloatingCameraPreview.module.css b/src/components/launch/FloatingCameraPreview.module.css index 9239c7e43..ee65ec55e 100644 --- a/src/components/launch/FloatingCameraPreview.module.css +++ b/src/components/launch/FloatingCameraPreview.module.css @@ -7,7 +7,7 @@ height: calc(100% - 16px); margin: 8px; overflow: hidden; - border-radius: 50%; + border-radius: 12px; background: #09090b; color: #a1a1aa; font-size: 12px; @@ -19,16 +19,15 @@ inset: 0; width: 100%; height: 100%; - object-fit: cover; + object-fit: contain; object-position: center; - transform: scaleX(-1); } .hide { -webkit-app-region: no-drag; position: absolute; - top: 9%; - right: 13%; + top: 8px; + right: 8px; display: grid; place-items: center; width: 28px; @@ -53,7 +52,7 @@ .sizeControls { -webkit-app-region: no-drag; position: absolute; - bottom: 9%; + bottom: 8px; left: 50%; display: flex; gap: 4px; diff --git a/src/components/launch/LaunchWindow.test.tsx b/src/components/launch/LaunchWindow.test.tsx index 64dcdb410..44a7640c9 100644 --- a/src/components/launch/LaunchWindow.test.tsx +++ b/src/components/launch/LaunchWindow.test.tsx @@ -2394,22 +2394,26 @@ describe("macOS floating camera preview control", () => { expect(recorderState.value.setWebcamEnabled).not.toHaveBeenCalled(); }); - it("keeps the preview button visible but disabled when the camera is off", () => { + it("hides the preview control when the camera is off", () => { const view = renderLaunchWindow(); - const button = screen.getByTestId("launch-camera-preview-button"); - expect(button).toHaveAttribute("aria-disabled", "true"); - expect(button).toHaveAttribute("aria-pressed", "false"); - fireEvent.click(button); - expect(button).toHaveAttribute("aria-pressed", "false"); - expect(recorderState.value.setWebcamEnabled).not.toHaveBeenCalled(); + expect(screen.queryByTestId("launch-camera-preview-button")).not.toBeInTheDocument(); recorderState.value.webcamEnabled = true; view.rerender( , ); - expect(button).not.toHaveAttribute("aria-disabled"); - expect(button).toHaveAttribute("aria-pressed", "true"); + expect(screen.getByTestId("launch-camera-preview-button")).toHaveAttribute( + "aria-pressed", + "true", + ); + recorderState.value.webcamEnabled = false; + view.rerender( + + + , + ); + expect(screen.queryByTestId("launch-camera-preview-button")).not.toBeInTheDocument(); }); it("does not offer the preview on Windows", () => { diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index ffd3132eb..7094529af 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -1285,10 +1285,10 @@ export function LaunchWindow() { tooltip={t("tooltips.camera")} onClick={toggleWebcam} /> - {cameraPreview.supported && ( + {cameraPreview.supported && webcamEnabled && ( diff --git a/src/hooks/useFloatingCameraPreview.test.tsx b/src/hooks/useFloatingCameraPreview.test.tsx index 39c982a0b..0c0279c0d 100644 --- a/src/hooks/useFloatingCameraPreview.test.tsx +++ b/src/hooks/useFloatingCameraPreview.test.tsx @@ -51,6 +51,7 @@ describe("floating camera publisher", () => { const { unmount } = renderHook(() => useFloatingCameraPreview(stream, true)); expect(configure).toHaveBeenLastCalledWith(true, true); expect(send).toHaveBeenCalledOnce(); + expect(send).toHaveBeenCalledWith("data:image/jpeg;base64,frame", 1280 / 720); await act(() => vi.advanceTimersByTimeAsync(300)); expect(send).toHaveBeenCalledTimes(4); unmount(); diff --git a/src/hooks/useFloatingCameraPreview.ts b/src/hooks/useFloatingCameraPreview.ts index d6bb2d012..3d1f1108b 100644 --- a/src/hooks/useFloatingCameraPreview.ts +++ b/src/hooks/useFloatingCameraPreview.ts @@ -30,11 +30,14 @@ export function useFloatingCameraPreview(stream: MediaStream | null, enabled: bo let timer: ReturnType; const draw = async () => { try { - if (context && video.readyState >= 2 && video.videoWidth > 0) { + if (context && video.readyState >= 2 && video.videoWidth > 0 && video.videoHeight > 0) { canvas.height = Math.max(1, Math.round((320 * video.videoHeight) / video.videoWidth)); context.drawImage(video, 0, 0, canvas.width, canvas.height); // Await delivery: a slow renderer must not build a queue of stale frames. - await window.electronAPI?.sendCameraPreviewFrame?.(canvas.toDataURL("image/jpeg", 0.7)); + await window.electronAPI?.sendCameraPreviewFrame?.( + canvas.toDataURL("image/jpeg", 0.7), + video.videoWidth / video.videoHeight, + ); } } catch (error) { console.warn("Could not update camera preview:", error); diff --git a/src/lib/cameraPreview.test.ts b/src/lib/cameraPreview.test.ts index c98f8548f..0a2ebd6b3 100644 --- a/src/lib/cameraPreview.test.ts +++ b/src/lib/cameraPreview.test.ts @@ -15,11 +15,20 @@ describe("camera preview placement", () => { const before = cameraPreviewBounds(workArea, bar); const after = resizedCameraPreviewBounds(before, workArea, 320); expect(after.width).toBe(336); - expect(after.height).toBe(336); + expect(after.height).toBe(196); expect(after.y + after.height).toBe(before.y + before.height); expect(after.x + after.width / 2).toBe(before.x + before.width / 2); }); + it("preserves the full portrait, landscape and square frame at every preview size", () => { + for (const ratio of [9 / 16, 16 / 9, 1]) { + for (const width of [160, 224, 320]) { + const bounds = resizedCameraPreviewBounds(bar, workArea, width, ratio); + expect((bounds.width - 16) / (bounds.height - 16)).toBeCloseTo(ratio, 2); + } + } + }); + it("keeps a preview on-screen on negative-origin displays and near the screen edge", () => { const display = { x: -1280, y: -100, width: 1280, height: 720 }; const preview = resizedCameraPreviewBounds( diff --git a/src/lib/cameraPreview.ts b/src/lib/cameraPreview.ts index 613d0e410..8525b3247 100644 --- a/src/lib/cameraPreview.ts +++ b/src/lib/cameraPreview.ts @@ -1,37 +1,62 @@ export const CAMERA_PREVIEW_SIZES = [160, 224, 320] as const; export const CAMERA_PREVIEW_DEFAULT_SIZE = 224; +export const CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO = 16 / 9; const WINDOW_PADDING = 16; type Rect = { x: number; y: number; width: number; height: number }; -function clampBounds(x: number, y: number, diameter: number, workArea: Rect): Rect { - const size = Math.min(diameter + WINDOW_PADDING, workArea.width, workArea.height); +function previewSize(width: number, aspectRatio: number, workArea: Rect) { + const contentWidth = Math.min( + width, + workArea.width - WINDOW_PADDING, + (workArea.height - WINDOW_PADDING) * aspectRatio, + ); + return { + width: Math.round(contentWidth) + WINDOW_PADDING, + height: Math.round(contentWidth / aspectRatio) + WINDOW_PADDING, + }; +} + +function clampBounds( + x: number, + y: number, + size: { width: number; height: number }, + workArea: Rect, +): Rect { return { - x: Math.round(Math.max(workArea.x, Math.min(x, workArea.x + workArea.width - size))), - y: Math.round(Math.max(workArea.y, Math.min(y, workArea.y + workArea.height - size))), - width: size, - height: size, + x: Math.round(Math.max(workArea.x, Math.min(x, workArea.x + workArea.width - size.width))), + y: Math.round(Math.max(workArea.y, Math.min(y, workArea.y + workArea.height - size.height))), + ...size, }; } -/** Place the circle just above the bar's right end, including the transparent window padding. */ +/** Place the complete camera frame just above the bar's right end. */ export function cameraPreviewBounds(workArea: Rect, bar?: Rect): Rect { - const size = CAMERA_PREVIEW_DEFAULT_SIZE + WINDOW_PADDING; - return clampBounds( - bar ? bar.x + bar.width - size : workArea.x + workArea.width / 2 + 160, - (bar?.y ?? workArea.y + workArea.height - 84) - size - 8, + const size = previewSize( CAMERA_PREVIEW_DEFAULT_SIZE, + CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO, + workArea, + ); + return clampBounds( + bar ? bar.x + bar.width - size.width : workArea.x + workArea.width / 2 + 160, + (bar?.y ?? workArea.y + workArea.height - 84) - size.height - 8, + size, workArea, ); } -/** Grow upward from the bottom centre, retaining its distance from the recording bar. */ -export function resizedCameraPreviewBounds(bounds: Rect, workArea: Rect, diameter: number): Rect { - const size = diameter + WINDOW_PADDING; +/** Resize without cropping, retaining the bottom centre and distance from the bar. */ +export function resizedCameraPreviewBounds( + bounds: Rect, + workArea: Rect, + width: number, + aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO, +): Rect { + const size = previewSize(width, aspectRatio, workArea); return clampBounds( - bounds.x + (bounds.width - size) / 2, - bounds.y + bounds.height - size, - diameter, + bounds.x + (bounds.width - size.width) / 2, + bounds.y + bounds.height - size.height, + size, workArea, ); } diff --git a/technical-documentation/architecture/recording.md b/technical-documentation/architecture/recording.md index 41d3d6470..1396dd6ea 100644 --- a/technical-documentation/architecture/recording.md +++ b/technical-documentation/architecture/recording.md @@ -24,8 +24,9 @@ Electron applies `setContentProtection(true)` to the HUD window (`applyContentPr ## Presenter camera preview (macOS) -The HUD camera has a draggable circular self-view with show/hide and three size choices. -Its preview button stays visible but disabled when the camera is off. The preview shares +The HUD camera has a draggable rounded rectangular self-view with show/hide and three width choices. +It shows the complete camera frame at the shared recording stream’s aspect ratio, including when resized. +Its preview button is hidden when the camera is off. The preview shares `useScreenRecorder`'s stream and stops with its owning HUD; it does not acquire another camera. `useFloatingCameraPreview` publishes small JPEG frames to the auxiliary window, with bounded payloads and one IPC request in flight. Audio preference broadcasts in the macOS HUD preserve diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 479830b43..55b1b0a92 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -995,3 +995,5 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-17 | `v2.1.1-rc.1`, `release/v2.1.1` @ `6ec38bf` (= `v2.1.0` + #1319), assets of the GitHub pre-release (build run 39069024444). Windows: `Openscreen.Setup.2.1.1-rc.1-x64.exe`, sha512 matched `latest.yml`, unpacked with 7-Zip and run in place, version 2.1.1. macOS: `Openscreen-macOS-Apple-Silicon-2.1.1-rc.1.dmg`, sha256 matched the asset digest; `spctl` accepted, Notarized Developer ID, on the DMG and the app; ticket stapled to the DMG only, as on 2.1.0-rc.6; `CFBundleShortVersionString` 2.1.1-rc.1. Linux: `Openscreen-Linux-2.1.1-rc.1.AppImage`, sha512 matched `latest-linux.yml` | Windows 11 Home 26200, RTX 4070 Ti; macOS 27.0.1 (26A434), Mac mini M1 8 GB; Ubuntu 24.04.4, GNOME 46 Wayland, Radeon 610M, hardware Vulkan backend | **Partial — patch pass toward stable: no defect on any of the three** | Scoped to what the patch changes (the compositor geometry of a tilted filled card), in MCP over stdio with an isolated `--user-data-dir`, on the project of the PR rows above. On all three: `orbit` (12 frames) and `left` show the whole card with four rounded corners and no flat cut; `none` is a flat zoom in a rounded card (byte-identical to `v2.1.0` on Windows and Linux; on Windows every checked frame is byte-identical to the PR build). `exportVideo` at 1080p30: 10.00 s, 1920×1080 H.264 with an AAC track, decodes without error, and its frame at 5 s shows the tilted card with rounded corners. Before the cut, on a dev build of the PR: the four device frames, dual-frame and vertical-stack, under `orbit`, `left` and `none`, rendered byte-identical to `v2.1.0` (49 of 49 frames). **Not covered:** recording, playback and the editor preview, which the patch does not touch; the rest of the checklist (the `v2.1.0-rc.6` rows). | | 2026-10-19 | `v2.1.2-rc.1`, `release/v2.1.2` @ `c4ef8fd` (= `v2.1.1` + #1344), assets of the GitHub pre-release (build run 39224311283). Windows: `Openscreen.Setup.2.1.2-rc.1-x64.exe` unpacked with 7-Zip and run in place. macOS: `Openscreen-Mac-arm64-2.1.2-rc.1.zip`, run in place. Linux: the AppImage, run directly with `--no-sandbox` | Windows 11 Home 26200; macOS 27.0.1, Mac mini M1; Ubuntu 24.04 GNOME Wayland (Xwayland `:0`) | **Partial: fix slice only, no defect** | `openscreen mcp` over stdio, scripted, with a fresh `--user-data-dir` per run. On all three platforms it answered `initialize`, and `usage-ping.json` held today's day, which is saved only once PostHog accepted the event. With `DO_NOT_TRACK=1` no state file was written. stdout held only JSON-RPC on macOS and Linux. On Windows it starts with one `\r\n` that Electron 41 itself writes during browser-process startup, before `main.js` loads: a bare `app.exit(0)` script prints it too, `ELECTRON_RUN_AS_NODE` does not. It predates this change, and the MCP SDK clients skip it with one parse error. Not covered: everything else in this file, including recording, editing, export, the GUI ping and the tray toggle. These runs sent eight real `surface: mcp` events to PostHog on 19/10. | | 2026-10-16 | dev `main` @ `b149c55` + #1278 macOS camera preview; Electron 41.2.1; capture helper built from the working tree with `swiftc` + macOS 26.5 SDK (SHA256 `06f142f075e4dd776b653f174379b2795a60d750557655f9bca6e2d2596c8f99`) | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — preview slice passed** | `mcp__cua_repl`: live circular self-view, show/hide during recording, three sizes and limit buttons, disabled preview button when camera is off, and recording finalization into screen MP4 + webcam WebM. Exercised OS dragging without independently measuring the final position. A take started with the preview hidden and showed it during capture; a decoded late screen frame contained no floating circle. Default placement and resize anchoring are covered by geometry tests. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras, restart/cancel, export, exhaustive frame-by-frame exclusion, Windows/Linux and the remaining checklist. SwiftPM/XCTest unavailable locally; direct helper compilation passed. | + +| 2026-10-19 | dev `codex/macos-floating-camera-preview`, rebased on `main` @ `3620cf8`, plus #1288 review changes; Electron 41.2.1; native helper rebuilt directly with `swiftc` + macOS 26.5 SDK | macOS 26.7, Apple M5 Pro, MacBook Pro Camera | **Partial — preview review slice passed** | Computer-use verified the complete rounded rectangular camera frame, aspect-preserving resize, camera-off hiding the HUD preview control, and show/hide during a finalized camera take. In the same take, after 5 s settling per condition, 30 s process-tree CPU deltas were **17.5% hidden / 22.3% shown** (100%=one core; includes Electron children and capture helper). Native `recording-stopped` reported **0 droppedVideoFrames over the entire take**, covering both intervals; this counts screen-writer backpressure/append failures, not upstream capture or webcam drops. Screen 1 at 3024×1964, saved webcam verified 1920×1080 (30 fps requested); mic/system audio off, system cursor. One sequential pair, no statistical claim. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras/platforms, restart/cancel, export, exhaustive exclusion and remaining checklist. | From e9005e87d1bdc7a0a676085193228c08621ea333 Mon Sep 17 00:00:00 2001 From: Satyam Raj Date: Mon, 19 Oct 2026 21:13:18 +0530 Subject: [PATCH 5/6] fix: retain camera preview window across camera toggles --- electron/cameraPreview.test.ts | 36 +++++++++++++++---- electron/cameraPreview.ts | 10 ++---- electron/windows.ts | 1 - .../testing/manual-e2e-checklist.md | 2 ++ 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/electron/cameraPreview.test.ts b/electron/cameraPreview.test.ts index d4ba4ffe4..df490f1e5 100644 --- a/electron/cameraPreview.test.ts +++ b/electron/cameraPreview.test.ts @@ -125,7 +125,7 @@ describe("floating camera preview lifecycle", () => { .mockReturnValueOnce(overlay as unknown as BrowserWindow) .mockReturnValueOnce(replacement as unknown as BrowserWindow); preview.configure(sender, true, true); - preview.configure(sender, false, false); + preview.dispose(); preview.configure(sender, true, true); replacement.emit("ready-to-show"); @@ -146,14 +146,36 @@ describe("floating camera preview lifecycle", () => { expect(overlay.webContents.send).not.toHaveBeenCalled(); }); - it("releases listeners on camera-off rather than accumulating them between takes", () => { - const { preview, sender, owner, overlay } = setup(); + it("creates a hidden exclusion window even when the camera starts off", () => { + const { preview, sender, owner, overlay, create } = setup(); + preview.configure(sender, false, true); + overlay.emit("ready-to-show"); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.hide).toHaveBeenCalledOnce(); + expect(overlay.showInactive).not.toHaveBeenCalled(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + owner.emit("destroyed"); + expect(overlay.destroy).toHaveBeenCalledOnce(); + }); + + it("retains the exclusion window and one owner listener across camera toggles", () => { + const { preview, sender, owner, overlay, create } = setup(); for (let i = 0; i < 15; i++) { - preview.configure(sender, true, false); - overlay.emit("ready-to-show"); - preview.configure(sender, false, false); + preview.configure(sender, true, true); + if (i === 0) overlay.emit("ready-to-show"); + preview.configure(sender, false, true); } + expect(create).toHaveBeenCalledOnce(); + expect(overlay.hide).toHaveBeenCalledTimes(15); + expect(overlay.showInactive).toHaveBeenCalledTimes(15); + expect(overlay.destroy).not.toHaveBeenCalled(); + expect(owner.listenerCount("destroyed")).toBe(1); + owner.emit("destroyed"); expect(owner.listenerCount("destroyed")).toBe(0); - expect(overlay.destroy).toHaveBeenCalledTimes(15); + expect(overlay.destroy).toHaveBeenCalledOnce(); }); }); diff --git a/electron/cameraPreview.ts b/electron/cameraPreview.ts index 8567e48e2..c3be9b7bd 100644 --- a/electron/cameraPreview.ts +++ b/electron/cameraPreview.ts @@ -26,10 +26,6 @@ export class CameraPreviewController { configure(owner: WebContents, enabled: boolean, visible: boolean): void { if (this.owner && this.owner !== owner) return; - if (!enabled) { - this.dispose(); - return; - } if (!this.owner) { this.owner = owner; this.onOwnerDestroyed = () => { @@ -53,9 +49,9 @@ export class CameraPreviewController { if (this.visible) this.window?.showInactive(); }); } - this.visible = visible; - if (visible && this.ready) this.window.showInactive(); - else if (!visible) this.window.hide(); + this.visible = enabled && visible; + if (this.visible && this.ready) this.window.showInactive(); + else if (!this.visible) this.window.hide(); } hide(sender: WebContents): void { diff --git a/electron/windows.ts b/electron/windows.ts index 9362a4277..9a95afcf0 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -889,7 +889,6 @@ export function createCameraPreviewWindow(): BrowserWindow { resizable: false, maximizable: false, minimizable: false, - closable: false, alwaysOnTop: true, skipTaskbar: true, show: false, diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 55b1b0a92..0fa3a1cf0 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -997,3 +997,5 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-16 | dev `main` @ `b149c55` + #1278 macOS camera preview; Electron 41.2.1; capture helper built from the working tree with `swiftc` + macOS 26.5 SDK (SHA256 `06f142f075e4dd776b653f174379b2795a60d750557655f9bca6e2d2596c8f99`) | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — preview slice passed** | `mcp__cua_repl`: live circular self-view, show/hide during recording, three sizes and limit buttons, disabled preview button when camera is off, and recording finalization into screen MP4 + webcam WebM. Exercised OS dragging without independently measuring the final position. A take started with the preview hidden and showed it during capture; a decoded late screen frame contained no floating circle. Default placement and resize anchoring are covered by geometry tests. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras, restart/cancel, export, exhaustive frame-by-frame exclusion, Windows/Linux and the remaining checklist. SwiftPM/XCTest unavailable locally; direct helper compilation passed. | | 2026-10-19 | dev `codex/macos-floating-camera-preview`, rebased on `main` @ `3620cf8`, plus #1288 review changes; Electron 41.2.1; native helper rebuilt directly with `swiftc` + macOS 26.5 SDK | macOS 26.7, Apple M5 Pro, MacBook Pro Camera | **Partial — preview review slice passed** | Computer-use verified the complete rounded rectangular camera frame, aspect-preserving resize, camera-off hiding the HUD preview control, and show/hide during a finalized camera take. In the same take, after 5 s settling per condition, 30 s process-tree CPU deltas were **17.5% hidden / 22.3% shown** (100%=one core; includes Electron children and capture helper). Native `recording-stopped` reported **0 droppedVideoFrames over the entire take**, covering both intervals; this counts screen-writer backpressure/append failures, not upstream capture or webcam drops. Screen 1 at 3024×1964, saved webcam verified 1920×1080 (30 fps requested); mic/system audio off, system cursor. One sequential pair, no statistical claim. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras/platforms, restart/cancel, export, exhaustive exclusion and remaining checklist. | + +| 2026-10-19 | dev #1288 on `8c2608d9` + preview lifetime/quit fixes; Electron 41.2.1 | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — lifecycle slice passed** | Computer-use camera off/on removed/restored the HUD preview control. Normal Cmd+Q exited the OpenScreen Electron process with the preview enabled. Unit tests verify the same preview window survives initial camera-off and repeated toggles, with one owner-destruction listener and cleanup on HUD destruction. **Skipped:** recording, export, capture-exclusion measurements, other platforms and remaining checklist; no native capture changes in this slice. | From 16f65379be6a92586ca6cac68555f7620659bdfb Mon Sep 17 00:00:00 2001 From: Satyam Raj Date: Wed, 21 Oct 2026 11:35:59 +0530 Subject: [PATCH 6/6] fix: reserve camera preview before capture exclusions --- electron/cameraPreview.test.ts | 28 +++++++++++++++++++ electron/cameraPreview.ts | 15 +++++++--- electron/ipc/handlers.ts | 13 +++++++++ .../testing/manual-e2e-checklist.md | 2 ++ 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/electron/cameraPreview.test.ts b/electron/cameraPreview.test.ts index df490f1e5..2a6add0ff 100644 --- a/electron/cameraPreview.test.ts +++ b/electron/cameraPreview.test.ts @@ -1,11 +1,13 @@ import { EventEmitter } from "node:events"; import type { BrowserWindow, WebContents } from "electron"; import { describe, expect, it, vi } from "vitest"; +import { collectMacCaptureExcludedWindowIds } from "../src/lib/nativeMacRecording"; import { CameraPreviewController } from "./cameraPreview"; function setup() { const owner = Object.assign(new EventEmitter(), { send: vi.fn() }); const overlay = Object.assign(new EventEmitter(), { + getMediaSourceId: () => "window:123:0", webContents: { send: vi.fn() }, showInactive: vi.fn(), hide: vi.fn(), @@ -24,6 +26,32 @@ function setup() { const jpeg = "data:image/jpeg;base64,/9j/"; describe("floating camera preview lifecycle", () => { + it("reserves the same exclusion ID before the renderer enables the preview", () => { + const { preview, sender, overlay, create } = setup(); + const reserved = preview.ensure(sender); + const excludedIds = collectMacCaptureExcludedWindowIds([reserved!.getMediaSourceId()]); + expect(excludedIds).toContain(123); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).not.toHaveBeenCalled(); + preview.configure(sender, true, true); + expect(preview.ensure(sender)).toBe(reserved); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + expect(excludedIds).toContain(123); + }); + + it("ensuring an existing preview preserves visibility and refuses a different owner", () => { + const { preview, sender, overlay, create } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + expect(preview.ensure(sender)).toBe(overlay); + expect(preview.ensure({} as WebContents)).toBeNull(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + expect(overlay.hide).not.toHaveBeenCalled(); + expect(create).toHaveBeenCalledOnce(); + }); + it("waits for the window to paint, then shows without taking focus", () => { const { preview, sender, overlay } = setup(); preview.configure(sender, true, true); diff --git a/electron/cameraPreview.ts b/electron/cameraPreview.ts index c3be9b7bd..9f1ff2ec7 100644 --- a/electron/cameraPreview.ts +++ b/electron/cameraPreview.ts @@ -24,8 +24,9 @@ export class CameraPreviewController { ) => void, ) {} - configure(owner: WebContents, enabled: boolean, visible: boolean): void { - if (this.owner && this.owner !== owner) return; + /** Reserve the capture-exclusion window before renderer configuration arrives. */ + ensure(owner: WebContents): BrowserWindow | null { + if (this.owner && this.owner !== owner) return null; if (!this.owner) { this.owner = owner; this.onOwnerDestroyed = () => { @@ -49,9 +50,15 @@ export class CameraPreviewController { if (this.visible) this.window?.showInactive(); }); } + return this.window; + } + + configure(owner: WebContents, enabled: boolean, visible: boolean): void { + const previewWindow = this.ensure(owner); + if (!previewWindow) return; this.visible = enabled && visible; - if (this.visible && this.ready) this.window.showInactive(); - else if (!this.visible) this.window.hide(); + if (this.visible && this.ready) previewWindow.showInactive(); + else if (!this.visible) previewWindow.hide(); } hide(sender: WebContents): void { diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index e0a2e4f2d..afcd04fc0 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -2147,6 +2147,7 @@ export function registerIpcHandlers( * pick would record them unless the picker is told to leave them out. */ function macPickerExcludedWindowIds() { + ensureMacCameraPreviewWindow(); const appWindowSourceIds = [getMainWindow(), getNotesWindow(), getCameraPreviewWindow()] .filter((window): window is BrowserWindow => !!window && !window.isDestroyed()) .map((window) => window.getMediaSourceId()); @@ -2480,6 +2481,17 @@ export function registerIpcHandlers( createCameraPreviewWindow, resizeCameraPreviewWindow, ); + function ensureMacCameraPreviewWindow() { + const hud = getMainWindow(); + if ( + process.platform === "darwin" && + hud && + !hud.isDestroyed() && + new URL(hud.webContents.getURL()).searchParams.get("windowType") === "hud-overlay" + ) { + cameraPreview.ensure(hud.webContents); + } + } ipcMain.handle("camera-preview:configure", (event, enabled: unknown, visible: unknown) => { if ( process.platform !== "darwin" || @@ -3175,6 +3187,7 @@ export function registerIpcHandlers( normalizeCursorCaptureMode(request.cursor?.mode) ?? "editable-overlay"; // A source from Apple's picker records through the session that holds the pick, // and needs no Screen Recording grant -- so nothing here may go near one. + ensureMacCameraPreviewWindow(); const pickerSession = isMacPickerSourceId(request.source.sourceId) ? macPickerSession : null; // Normally caught by `get-selected-source` first; this is the last word (#965). pickerSession?.forgetSelectionUnlessExcluding(macPickerExcludedWindowIds()); diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 0fa3a1cf0..01ba21023 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -999,3 +999,5 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-19 | dev `codex/macos-floating-camera-preview`, rebased on `main` @ `3620cf8`, plus #1288 review changes; Electron 41.2.1; native helper rebuilt directly with `swiftc` + macOS 26.5 SDK | macOS 26.7, Apple M5 Pro, MacBook Pro Camera | **Partial — preview review slice passed** | Computer-use verified the complete rounded rectangular camera frame, aspect-preserving resize, camera-off hiding the HUD preview control, and show/hide during a finalized camera take. In the same take, after 5 s settling per condition, 30 s process-tree CPU deltas were **17.5% hidden / 22.3% shown** (100%=one core; includes Electron children and capture helper). Native `recording-stopped` reported **0 droppedVideoFrames over the entire take**, covering both intervals; this counts screen-writer backpressure/append failures, not upstream capture or webcam drops. Screen 1 at 3024×1964, saved webcam verified 1920×1080 (30 fps requested); mic/system audio off, system cursor. One sequential pair, no statistical claim. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras/platforms, restart/cancel, export, exhaustive exclusion and remaining checklist. | | 2026-10-19 | dev #1288 on `8c2608d9` + preview lifetime/quit fixes; Electron 41.2.1 | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — lifecycle slice passed** | Computer-use camera off/on removed/restored the HUD preview control. Normal Cmd+Q exited the OpenScreen Electron process with the preview enabled. Unit tests verify the same preview window survives initial camera-off and repeated toggles, with one owner-destruction listener and cleanup on HUD destruction. **Skipped:** recording, export, capture-exclusion measurements, other platforms and remaining checklist; no native capture changes in this slice. | + +| 2026-10-21 | dev #1288 on `20975d30` + synchronous preview reservation before capture exclusions; Electron 41.2.1; existing helper rebuilt from this branch on Oct 19 | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — capture-exclusion ordering slice passed** | Computer-use started native display recording with the preview hidden, showed it during the take and finalized into the editor. Helper diagnostics resolved both requested app window IDs and reported zero screen-writer drops; a decoded late screen frame contained no floating preview. Unit regression covers reserving the exclusion window before any renderer configuration and reusing it when enabled later without altering an existing preview’s visibility. **Skipped:** Apple’s picker (ordering path checked in code), forced renderer-effect race on desktop (covered by unit regression), other platforms, export, exhaustive frame-by-frame exclusion and remaining checklist. |