diff --git a/docs/architecture/native-bridge.md b/docs/architecture/native-bridge.md new file mode 100644 index 00000000..ef320f77 --- /dev/null +++ b/docs/architecture/native-bridge.md @@ -0,0 +1,39 @@ +# Native Bridge Architecture + +## Goal + +Provide a single, resilient source of truth for platform-native capabilities while keeping Electron transport thin and renderer APIs unified. + +## Layers + +1. Native adapters +Platform-specific providers implement stable domain interfaces such as cursor telemetry or system asset discovery. + +2. Main-process services +Services orchestrate adapters, own runtime state, and expose domain-level operations. + +3. Unified IPC transport +Renderer code talks to a single `native-bridge:invoke` channel using versioned contracts. + +4. Renderer client +React code should consume `src/native/client.ts` rather than binding directly to ad hoc Electron APIs. + +## Principles + +- Single source of truth: runtime-native state lives in the Electron main process. +- Capability-first: renderer can query support before attempting native behavior. +- Versioned contracts: requests and responses are explicit and evolve predictably. +- Resilience: every response uses a consistent result envelope with stable error codes. + +## Current rollout + +This repository now contains the initial scaffold: + +- shared contracts in `src/native/contracts.ts` +- renderer SDK in `src/native/client.ts` +- main-process state store in `electron/native-bridge/store.ts` +- cursor telemetry adapter in `electron/native-bridge/cursor/telemetryCursorAdapter.ts` +- domain services in `electron/native-bridge/services/*` +- unified handler registration in `electron/ipc/nativeBridge.ts` + +The legacy `window.electronAPI` surface still exists for backward compatibility. New native-facing features should prefer the unified bridge client. \ No newline at end of file diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 573aee8a..69837934 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -24,6 +24,9 @@ declare namespace NodeJS { // Used in Renderer process, expose in `preload.ts` interface Window { electronAPI: { + invokeNativeBridge: ( + request: import("../src/native/contracts").NativeBridgeRequest, + ) => Promise>; getSources: (opts: Electron.SourcesOptions) => Promise; switchToEditor: () => Promise; openSourceSelector: () => Promise; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 78d83448..3716cd9f 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -1,24 +1,19 @@ import fs from "node:fs/promises"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { - app, - BrowserWindow, - desktopCapturer, - dialog, - ipcMain, - screen, - shell, - systemPreferences, -} from "electron"; -import { - normalizeProjectMedia, - normalizeRecordingSession, - type RecordingSession, - type StoreRecordedSessionInput, -} from "../../src/lib/recordingSession"; -import { mainT } from "../i18n"; +import { app, BrowserWindow, desktopCapturer, dialog, ipcMain, screen, shell } from "electron"; +import type { DesktopCapturerSource } from "electron"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorAsset, + ProjectFileResult, + ProjectPathResult, +} from "../../src/native/contracts"; import { RECORDINGS_DIR } from "../main"; +import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; +import type { CursorRecordingSession } from "../native-bridge/cursor/recording/session"; +import { registerNativeBridgeHandlers } from "./nativeBridge"; const PROJECT_FILE_EXTENSION = "openscreen"; const SHORTCUTS_FILE = path.join(app.getPath("userData"), "shortcuts.json"); @@ -26,12 +21,24 @@ const RECORDING_SESSION_SUFFIX = ".session.json"; type SelectedSource = { name: string; + id?: string; + display_id?: string; [key: string]: unknown; }; let selectedSource: SelectedSource | null = null; +let selectedDesktopSource: DesktopCapturerSource | null = null; +let lastEnumeratedSources = new Map(); let currentProjectPath: string | null = null; -let currentRecordingSession: RecordingSession | null = null; + +/** + * Returns the cached DesktopCapturerSource set when the user picked a source. + * Used by setDisplayMediaRequestHandler in main.ts for cursor-free capture. + */ +export function getSelectedDesktopSource(): DesktopCapturerSource | null { + return selectedDesktopSource; +} +let currentVideoPath: string | null = null; function normalizePath(filePath: string) { return path.resolve(filePath); @@ -65,150 +72,162 @@ function isTrustedProjectPath(filePath?: string | null) { return normalizePath(filePath) === normalizePath(currentProjectPath); } -function setCurrentRecordingSessionState(session: RecordingSession | null) { - currentRecordingSession = session; -} +const CURSOR_TELEMETRY_VERSION = 2; +const CURSOR_SAMPLE_INTERVAL_MS = 33; +const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz -function getSessionManifestPathForVideo(videoPath: string) { - const parsed = path.parse(videoPath); - const baseName = parsed.name.endsWith("-webcam") - ? parsed.name.slice(0, -"-webcam".length) - : parsed.name; - return path.join(parsed.dir, `${baseName}${RECORDING_SESSION_SUFFIX}`); -} +let cursorRecordingSession: CursorRecordingSession | null = null; +let pendingCursorRecordingData: CursorRecordingData | null = null; -async function loadRecordedSessionForVideoPath( - videoPath: string, -): Promise { - const normalizedVideoPath = normalizeVideoSourcePath(videoPath); - if (!normalizedVideoPath) { +function normalizeCursorSample(sample: unknown): CursorRecordingSample | null { + if (!sample || typeof sample !== "object") { return null; } - try { - const manifestPath = getSessionManifestPathForVideo(normalizedVideoPath); - const content = await fs.readFile(manifestPath, "utf-8"); - const session = normalizeRecordingSession(JSON.parse(content)); - if (!session) { - return null; - } - - const normalizedSession: RecordingSession = { - ...session, - screenVideoPath: normalizeVideoSourcePath(session.screenVideoPath) ?? session.screenVideoPath, - ...(session.webcamVideoPath - ? { - webcamVideoPath: - normalizeVideoSourcePath(session.webcamVideoPath) ?? session.webcamVideoPath, - } - : {}), - }; - - const targetPath = normalizePath(normalizedVideoPath); - const screenMatches = normalizePath(normalizedSession.screenVideoPath) === targetPath; - const webcamMatches = normalizedSession.webcamVideoPath - ? normalizePath(normalizedSession.webcamVideoPath) === targetPath - : false; - - return screenMatches || webcamMatches ? normalizedSession : null; - } catch { - return null; - } + const point = sample as Partial; + return { + timeMs: + typeof point.timeMs === "number" && Number.isFinite(point.timeMs) + ? Math.max(0, point.timeMs) + : 0, + cx: typeof point.cx === "number" && Number.isFinite(point.cx) ? point.cx : 0.5, + cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? point.cy : 0.5, + assetId: typeof point.assetId === "string" ? point.assetId : null, + visible: typeof point.visible === "boolean" ? point.visible : true, + }; } -async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) { - const createdAt = - typeof payload.createdAt === "number" && Number.isFinite(payload.createdAt) - ? payload.createdAt - : Date.now(); - const screenVideoPath = path.join(RECORDINGS_DIR, payload.screen.fileName); - await fs.writeFile(screenVideoPath, Buffer.from(payload.screen.videoData)); - - let webcamVideoPath: string | undefined; - if (payload.webcam) { - webcamVideoPath = path.join(RECORDINGS_DIR, payload.webcam.fileName); - await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData)); +function normalizeCursorAsset(asset: unknown): NativeCursorAsset | null { + if (!asset || typeof asset !== "object") { + return null; } - const session: RecordingSession = webcamVideoPath - ? { screenVideoPath, webcamVideoPath, createdAt } - : { screenVideoPath, createdAt }; - setCurrentRecordingSessionState(session); - currentProjectPath = null; - - const telemetryPath = `${screenVideoPath}.cursor.json`; - if (pendingCursorSamples.length > 0) { - await fs.writeFile( - telemetryPath, - JSON.stringify({ version: CURSOR_TELEMETRY_VERSION, samples: pendingCursorSamples }, null, 2), - "utf-8", - ); + const candidate = asset as Partial; + if (typeof candidate.id !== "string" || typeof candidate.imageDataUrl !== "string") { + return null; } - pendingCursorSamples = []; - - const sessionManifestPath = path.join( - RECORDINGS_DIR, - `${path.parse(payload.screen.fileName).name}${RECORDING_SESSION_SUFFIX}`, - ); - await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); return { - success: true, - path: screenVideoPath, - session, - message: "Recording session stored successfully", + id: candidate.id, + platform: + candidate.platform === "win32" ? "win32" : process.platform === "darwin" ? "darwin" : "linux", + imageDataUrl: candidate.imageDataUrl, + width: + typeof candidate.width === "number" && Number.isFinite(candidate.width) + ? Math.max(1, Math.round(candidate.width)) + : 1, + height: + typeof candidate.height === "number" && Number.isFinite(candidate.height) + ? Math.max(1, Math.round(candidate.height)) + : 1, + hotspotX: + typeof candidate.hotspotX === "number" && Number.isFinite(candidate.hotspotX) + ? Math.max(0, Math.round(candidate.hotspotX)) + : 0, + hotspotY: + typeof candidate.hotspotY === "number" && Number.isFinite(candidate.hotspotY) + ? Math.max(0, Math.round(candidate.hotspotY)) + : 0, + scaleFactor: + typeof candidate.scaleFactor === "number" && Number.isFinite(candidate.scaleFactor) + ? Math.max(0.1, candidate.scaleFactor) + : undefined, }; } -const CURSOR_TELEMETRY_VERSION = 1; -const CURSOR_SAMPLE_INTERVAL_MS = 100; -const MAX_CURSOR_SAMPLES = 60 * 60 * 10; // 1 hour @ 10Hz +async function readCursorRecordingFile(targetVideoPath: string): Promise { + const telemetryPath = `${targetVideoPath}.cursor.json`; + try { + const content = await fs.readFile(telemetryPath, "utf-8"); + const parsed = JSON.parse(content); + const rawSamples = Array.isArray(parsed) + ? parsed + : Array.isArray(parsed?.samples) + ? parsed.samples + : []; + const rawAssets = Array.isArray(parsed?.assets) ? parsed.assets : []; + + const samples = rawSamples + .map((sample: unknown) => normalizeCursorSample(sample)) + .filter((sample: CursorRecordingSample | null): sample is CursorRecordingSample => + Boolean(sample), + ) + .sort((a: CursorRecordingSample, b: CursorRecordingSample) => a.timeMs - b.timeMs); + + const assets = rawAssets + .map((asset: unknown) => normalizeCursorAsset(asset)) + .filter((asset: NativeCursorAsset | null): asset is NativeCursorAsset => Boolean(asset)); + + return { + version: + typeof parsed?.version === "number" && Number.isFinite(parsed.version) ? parsed.version : 1, + provider: parsed?.provider === "native" ? "native" : "none", + samples, + assets, + }; + } catch (error) { + const nodeError = error as NodeJS.ErrnoException; + if (nodeError.code === "ENOENT") { + return { + version: CURSOR_TELEMETRY_VERSION, + provider: "none", + samples: [], + assets: [], + }; + } -interface CursorTelemetryPoint { - timeMs: number; - cx: number; - cy: number; + console.error("Failed to load cursor telemetry:", error); + throw error; + } } -let cursorCaptureInterval: NodeJS.Timeout | null = null; -let cursorCaptureStartTimeMs = 0; -let activeCursorSamples: CursorTelemetryPoint[] = []; -let pendingCursorSamples: CursorTelemetryPoint[] = []; - -function clamp(value: number, min: number, max: number) { - return Math.min(max, Math.max(min, value)); +async function readCursorTelemetryFile(targetVideoPath: string) { + try { + const recordingData = await readCursorRecordingFile(targetVideoPath); + return { + success: true, + samples: recordingData.samples.map((sample) => ({ + timeMs: sample.timeMs, + cx: sample.cx, + cy: sample.cy, + })), + }; + } catch (error) { + console.error("Failed to load cursor telemetry:", error); + return { + success: false, + message: "Failed to load cursor telemetry", + error: String(error), + samples: [], + }; + } } -function stopCursorCapture() { - if (cursorCaptureInterval) { - clearInterval(cursorCaptureInterval); - cursorCaptureInterval = null; +function resolveAssetBasePath() { + try { + if (app.isPackaged) { + const assetPath = path.join(process.resourcesPath, "assets"); + return pathToFileURL(`${assetPath}${path.sep}`).toString(); + } + const assetPath = path.join(app.getAppPath(), "public", "assets"); + return pathToFileURL(`${assetPath}${path.sep}`).toString(); + } catch (err) { + console.error("Failed to resolve asset base path:", err); + return null; } } -function sampleCursorPoint() { +function getSelectedSourceBounds() { const cursor = screen.getCursorScreenPoint(); const sourceDisplayId = Number(selectedSource?.display_id); const sourceDisplay = Number.isFinite(sourceDisplayId) ? (screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null) : null; - const display = sourceDisplay ?? screen.getDisplayNearestPoint(cursor); - const bounds = display.bounds; - const width = Math.max(1, bounds.width); - const height = Math.max(1, bounds.height); - - const cx = clamp((cursor.x - bounds.x) / width, 0, 1); - const cy = clamp((cursor.y - bounds.y) / height, 0, 1); - - activeCursorSamples.push({ - timeMs: Math.max(0, Date.now() - cursorCaptureStartTimeMs), - cx, - cy, - }); + return (sourceDisplay ?? screen.getDisplayNearestPoint(cursor)).bounds; +} - if (activeCursorSamples.length > MAX_CURSOR_SAMPLES) { - activeCursorSamples.shift(); - } +function getSelectedSourceId() { + return typeof selectedSource?.id === "string" ? selectedSource.id : null; } export function registerIpcHandlers( @@ -220,6 +239,7 @@ export function registerIpcHandlers( ) { ipcMain.handle("get-sources", async (_, opts) => { const sources = await desktopCapturer.getSources(opts); + lastEnumeratedSources = new Map(sources.map((source) => [source.id, source])); return sources.map((source) => ({ id: source.id, name: source.name, @@ -229,8 +249,26 @@ export function registerIpcHandlers( })); }); - ipcMain.handle("select-source", (_, source: SelectedSource) => { + ipcMain.handle("select-source", async (_, source: SelectedSource) => { selectedSource = source; + // Reuse the exact source object returned during enumeration to avoid + // Windows window-source id mismatches across separate getSources() calls. + selectedDesktopSource = + typeof source.id === "string" ? lastEnumeratedSources.get(source.id) ?? null : null; + + if (!selectedDesktopSource && typeof source.id === "string") { + try { + const sources = await desktopCapturer.getSources({ + types: ["screen", "window"], + thumbnailSize: { width: 0, height: 0 }, + fetchWindowIcons: true, + }); + lastEnumeratedSources = new Map(sources.map((candidate) => [candidate.id, candidate])); + selectedDesktopSource = lastEnumeratedSources.get(source.id) ?? null; + } catch { + selectedDesktopSource = null; + } + } const sourceSelectorWin = getSourceSelectorWindow(); if (sourceSelectorWin) { sourceSelectorWin.close(); @@ -293,7 +331,25 @@ export function registerIpcHandlers( ipcMain.handle("store-recorded-session", async (_, payload: StoreRecordedSessionInput) => { try { - return await storeRecordedSessionFiles(payload); + const videoPath = path.join(RECORDINGS_DIR, fileName); + await fs.writeFile(videoPath, Buffer.from(videoData)); + currentProjectPath = null; + + const telemetryPath = `${videoPath}.cursor.json`; + if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) { + await fs.writeFile( + telemetryPath, + JSON.stringify(pendingCursorRecordingData, null, 2), + "utf-8", + ); + } + pendingCursorRecordingData = null; + + return { + success: true, + path: videoPath, + message: "Video stored successfully", + }; } catch (error) { console.error("Failed to store recording session:", error); return { @@ -345,41 +401,39 @@ export function registerIpcHandlers( } }); - ipcMain.handle("read-binary-file", async (_, inputPath: string) => { - try { - const normalizedPath = normalizeVideoSourcePath(inputPath); - if (!normalizedPath) { - return { success: false, message: "Invalid file path" }; + ipcMain.handle("set-recording-state", async (_, recording: boolean) => { + if (recording) { + if (cursorRecordingSession) { + pendingCursorRecordingData = await cursorRecordingSession.stop(); + cursorRecordingSession = null; } - const data = await fs.readFile(normalizedPath); - return { - success: true, - data: data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength), - path: normalizedPath, - }; - } catch (error) { - console.error("Failed to read binary file:", error); - return { - success: false, - message: "Failed to read binary file", - error: String(error), - }; - } - }); + pendingCursorRecordingData = null; + cursorRecordingSession = createCursorRecordingSession({ + getDisplayBounds: getSelectedSourceBounds, + maxSamples: MAX_CURSOR_SAMPLES, + platform: process.platform, + sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, + sourceId: getSelectedSourceId(), + }); - ipcMain.handle("set-recording-state", (_, recording: boolean) => { - if (recording) { - stopCursorCapture(); - activeCursorSamples = []; - pendingCursorSamples = []; - cursorCaptureStartTimeMs = Date.now(); - sampleCursorPoint(); - cursorCaptureInterval = setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS); + try { + await cursorRecordingSession.start(); + } catch (error) { + console.error("Failed to start cursor recording session:", error); + cursorRecordingSession = null; + } } else { - stopCursorCapture(); - pendingCursorSamples = [...activeCursorSamples]; - activeCursorSamples = []; + if (cursorRecordingSession) { + try { + pendingCursorRecordingData = await cursorRecordingSession.stop(); + } catch (error) { + console.error("Failed to stop cursor recording session:", error); + pendingCursorRecordingData = null; + } finally { + cursorRecordingSession = null; + } + } } const source = selectedSource || { name: "Screen" }; @@ -396,51 +450,7 @@ export function registerIpcHandlers( return { success: true, samples: [] }; } - const telemetryPath = `${targetVideoPath}.cursor.json`; - try { - const content = await fs.readFile(telemetryPath, "utf-8"); - const parsed = JSON.parse(content); - const rawSamples = Array.isArray(parsed) - ? parsed - : Array.isArray(parsed?.samples) - ? parsed.samples - : []; - - const samples: CursorTelemetryPoint[] = rawSamples - .filter((sample: unknown) => Boolean(sample && typeof sample === "object")) - .map((sample: unknown) => { - const point = sample as Partial; - return { - timeMs: - typeof point.timeMs === "number" && Number.isFinite(point.timeMs) - ? Math.max(0, point.timeMs) - : 0, - cx: - typeof point.cx === "number" && Number.isFinite(point.cx) - ? clamp(point.cx, 0, 1) - : 0.5, - cy: - typeof point.cy === "number" && Number.isFinite(point.cy) - ? clamp(point.cy, 0, 1) - : 0.5, - }; - }) - .sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs); - - return { success: true, samples }; - } catch (error) { - const nodeError = error as NodeJS.ErrnoException; - if (nodeError.code === "ENOENT") { - return { success: true, samples: [] }; - } - console.error("Failed to load cursor telemetry:", error); - return { - success: false, - message: "Failed to load cursor telemetry", - error: String(error), - samples: [], - }; - } + return readCursorTelemetryFile(targetVideoPath); }); ipcMain.handle("open-external-url", async (_, url: string) => { @@ -455,17 +465,7 @@ export function registerIpcHandlers( // Return base path for assets so renderer can resolve file:// paths in production ipcMain.handle("get-asset-base-path", () => { - try { - if (app.isPackaged) { - const assetPath = path.join(process.resourcesPath, "assets"); - return pathToFileURL(`${assetPath}${path.sep}`).toString(); - } - const assetPath = path.join(app.getAppPath(), "public", "assets"); - return pathToFileURL(`${assetPath}${path.sep}`).toString(); - } catch (err) { - console.error("Failed to resolve asset base path:", err); - return null; - } + return resolveAssetBasePath(); }); ipcMain.handle("save-exported-video", async (_, videoData: ArrayBuffer, fileName: string) => { @@ -571,71 +571,80 @@ export function registerIpcHandlers( ipcMain.handle( "save-project-file", async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string) => { - try { - const trustedExistingProjectPath = isTrustedProjectPath(existingProjectPath) - ? existingProjectPath - : null; - - if (trustedExistingProjectPath) { - await fs.writeFile( - trustedExistingProjectPath, - JSON.stringify(projectData, null, 2), - "utf-8", - ); - currentProjectPath = trustedExistingProjectPath; - return { - success: true, - path: trustedExistingProjectPath, - message: "Project saved successfully", - }; - } - - const safeName = (suggestedName || `project-${Date.now()}`).replace(/[^a-zA-Z0-9-_]/g, "_"); - const defaultName = safeName.endsWith(`.${PROJECT_FILE_EXTENSION}`) - ? safeName - : `${safeName}.${PROJECT_FILE_EXTENSION}`; - - const result = await dialog.showSaveDialog({ - title: mainT("dialogs", "fileDialogs.saveProject"), - defaultPath: path.join(RECORDINGS_DIR, defaultName), - filters: [ - { - name: mainT("dialogs", "fileDialogs.openscreenProject"), - extensions: [PROJECT_FILE_EXTENSION], - }, - { name: "JSON", extensions: ["json"] }, - ], - properties: ["createDirectory", "showOverwriteConfirmation"], - }); - - if (result.canceled || !result.filePath) { - return { - success: false, - canceled: true, - message: "Save project canceled", - }; - } - - await fs.writeFile(result.filePath, JSON.stringify(projectData, null, 2), "utf-8"); - currentProjectPath = result.filePath; + return saveProjectFile(projectData, suggestedName, existingProjectPath); + }, + ); + async function saveProjectFile( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ): Promise { + try { + const trustedExistingProjectPath = isTrustedProjectPath(existingProjectPath) + ? existingProjectPath + : null; + + if (trustedExistingProjectPath) { + await fs.writeFile( + trustedExistingProjectPath, + JSON.stringify(projectData, null, 2), + "utf-8", + ); + currentProjectPath = trustedExistingProjectPath; return { success: true, - path: result.filePath, + path: trustedExistingProjectPath, message: "Project saved successfully", }; - } catch (error) { - console.error("Failed to save project file:", error); + } + + const safeName = (suggestedName || `project-${Date.now()}`).replace(/[^a-zA-Z0-9-_]/g, "_"); + const defaultName = safeName.endsWith(`.${PROJECT_FILE_EXTENSION}`) + ? safeName + : `${safeName}.${PROJECT_FILE_EXTENSION}`; + + const result = await dialog.showSaveDialog({ + title: "Save OpenScreen Project", + defaultPath: path.join(RECORDINGS_DIR, defaultName), + filters: [ + { name: "OpenScreen Project", extensions: [PROJECT_FILE_EXTENSION] }, + { name: "JSON", extensions: ["json"] }, + ], + properties: ["createDirectory", "showOverwriteConfirmation"], + }); + + if (result.canceled || !result.filePath) { return { success: false, - message: "Failed to save project file", - error: String(error), + canceled: true, + message: "Save project canceled", }; } - }, - ); + + await fs.writeFile(result.filePath, JSON.stringify(projectData, null, 2), "utf-8"); + currentProjectPath = result.filePath; + + return { + success: true, + path: result.filePath, + message: "Project saved successfully", + }; + } catch (error) { + console.error("Failed to save project file:", error); + return { + success: false, + message: "Failed to save project file", + error: String(error), + }; + } + } ipcMain.handle("load-project-file", async () => { + return loadProjectFile(); + }); + + async function loadProjectFile(): Promise { try { const result = await dialog.showOpenDialog({ title: mainT("dialogs", "fileDialogs.openProject"), @@ -685,9 +694,13 @@ export function registerIpcHandlers( error: String(error), }; } - }); + } ipcMain.handle("load-current-project-file", async () => { + return loadCurrentProjectFile(); + }); + + async function loadCurrentProjectFile(): Promise { try { if (!currentProjectPath) { return { success: false, message: "No active project" }; @@ -720,45 +733,35 @@ export function registerIpcHandlers( error: String(error), }; } - }); - ipcMain.handle("set-current-recording-session", (_, session: RecordingSession | null) => { - const normalized = normalizeRecordingSession(session); - setCurrentRecordingSessionState(normalized); - currentProjectPath = null; - return { success: true, session: normalized ?? undefined }; - }); + } - ipcMain.handle("get-current-recording-session", () => { - return currentRecordingSession - ? { success: true, session: currentRecordingSession } - : { success: false }; + ipcMain.handle("set-current-video-path", (_, path: string) => { + return setCurrentVideoPath(path); }); - ipcMain.handle("set-current-video-path", async (_, path: string) => { - const restoredSession = await loadRecordedSessionForVideoPath(path); - if (restoredSession) { - setCurrentRecordingSessionState(restoredSession); - } else { - setCurrentRecordingSessionState({ - screenVideoPath: normalizeVideoSourcePath(path) ?? path, - createdAt: Date.now(), - }); - } + function setCurrentVideoPath(path: string): ProjectPathResult { + currentVideoPath = normalizeVideoSourcePath(path) ?? path; currentProjectPath = null; return { success: true }; - }); + } ipcMain.handle("get-current-video-path", () => { - return currentRecordingSession?.screenVideoPath - ? { success: true, path: currentRecordingSession.screenVideoPath } - : { success: false }; + return getCurrentVideoPathResult(); }); + function getCurrentVideoPathResult(): ProjectPathResult { + return currentVideoPath ? { success: true, path: currentVideoPath } : { success: false }; + } + ipcMain.handle("clear-current-video-path", () => { - setCurrentRecordingSessionState(null); - return { success: true }; + return clearCurrentVideoPath(); }); + function clearCurrentVideoPath(): ProjectPathResult { + currentVideoPath = null; + return { success: true }; + } + ipcMain.handle("get-platform", () => { return process.platform; }); @@ -781,4 +784,21 @@ export function registerIpcHandlers( return { success: false, error: String(error) }; } }); + + registerNativeBridgeHandlers({ + getPlatform: () => process.platform, + getCurrentProjectPath: () => currentProjectPath, + getCurrentVideoPath: () => currentVideoPath, + saveProjectFile, + loadProjectFile, + loadCurrentProjectFile, + setCurrentVideoPath, + getCurrentVideoPathResult, + clearCurrentVideoPath, + resolveAssetBasePath, + resolveVideoPath: (videoPath?: string | null) => + normalizeVideoSourcePath(videoPath ?? currentVideoPath), + loadCursorRecordingData: readCursorRecordingFile, + loadCursorTelemetry: readCursorTelemetryFile, + }); } diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts new file mode 100644 index 00000000..ba6258a3 --- /dev/null +++ b/electron/ipc/nativeBridge.ts @@ -0,0 +1,229 @@ +import { ipcMain } from "electron"; +import { + NATIVE_BRIDGE_CHANNEL, + NATIVE_BRIDGE_VERSION, + type NativeBridgeErrorCode, + type NativeBridgeRequest, + type NativeBridgeResponse, + type NativePlatform, + type ProjectFileResult, + type ProjectPathResult, +} from "../../src/native/contracts"; +import type { CursorTelemetryLoadResult } from "../native-bridge/cursor/adapter"; +import { TelemetryCursorAdapter } from "../native-bridge/cursor/telemetryCursorAdapter"; +import { CursorService } from "../native-bridge/services/cursorService"; +import { ProjectService } from "../native-bridge/services/projectService"; +import { SystemService } from "../native-bridge/services/systemService"; +import { NativeBridgeStateStore } from "../native-bridge/store"; + +export interface NativeBridgeContext { + getPlatform: () => NodeJS.Platform; + getCurrentProjectPath: () => string | null; + getCurrentVideoPath: () => string | null; + saveProjectFile: ( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ) => Promise; + loadProjectFile: () => Promise; + loadCurrentProjectFile: () => Promise; + setCurrentVideoPath: (path: string) => ProjectPathResult; + getCurrentVideoPathResult: () => ProjectPathResult; + clearCurrentVideoPath: () => ProjectPathResult; + resolveAssetBasePath: () => string | null; + resolveVideoPath: (videoPath?: string | null) => string | null; + loadCursorRecordingData: ( + videoPath: string, + ) => Promise; + loadCursorTelemetry: (videoPath: string) => Promise; +} + +function normalizePlatform(platform: NodeJS.Platform): NativePlatform { + if (platform === "darwin" || platform === "win32") { + return platform; + } + + return "linux"; +} + +function createMeta(requestId?: string) { + return { + version: NATIVE_BRIDGE_VERSION, + requestId: requestId || `native-${Date.now()}`, + timestampMs: Date.now(), + } as const; +} + +function createSuccessResponse(requestId: string | undefined, data: TData) { + return { + ok: true, + data, + meta: createMeta(requestId), + } satisfies NativeBridgeResponse; +} + +function createErrorResponse( + requestId: string | undefined, + code: NativeBridgeErrorCode, + message: string, + retryable = false, +) { + return { + ok: false, + error: { + code, + message, + retryable, + }, + meta: createMeta(requestId), + } satisfies NativeBridgeResponse; +} + +function isBridgeRequest(value: unknown): value is NativeBridgeRequest { + if (!value || typeof value !== "object") { + return false; + } + + const candidate = value as Partial; + return typeof candidate.domain === "string" && typeof candidate.action === "string"; +} + +export function registerNativeBridgeHandlers(context: NativeBridgeContext) { + ipcMain.removeHandler(NATIVE_BRIDGE_CHANNEL); + + const platform = normalizePlatform(context.getPlatform()); + const store = new NativeBridgeStateStore(platform); + const projectService = new ProjectService({ + store, + getCurrentProjectPath: context.getCurrentProjectPath, + getCurrentVideoPath: context.getCurrentVideoPath, + saveProjectFile: context.saveProjectFile, + loadProjectFile: context.loadProjectFile, + loadCurrentProjectFile: context.loadCurrentProjectFile, + setCurrentVideoPath: context.setCurrentVideoPath, + getCurrentVideoPathResult: context.getCurrentVideoPathResult, + clearCurrentVideoPath: context.clearCurrentVideoPath, + }); + const cursorService = new CursorService({ + store, + adapter: new TelemetryCursorAdapter({ + loadRecordingData: context.loadCursorRecordingData, + resolveVideoPath: context.resolveVideoPath, + loadTelemetry: context.loadCursorTelemetry, + }), + }); + const systemService = new SystemService({ + store, + getPlatform: () => platform, + getAssetBasePath: context.resolveAssetBasePath, + getCursorCapabilities: () => cursorService.getCapabilities(), + }); + + ipcMain.handle(NATIVE_BRIDGE_CHANNEL, async (_, request: unknown) => { + if (!isBridgeRequest(request)) { + return createErrorResponse(undefined, "INVALID_REQUEST", "Invalid native bridge request."); + } + + const requestId = request.requestId; + const domain = request.domain as string; + + try { + switch (request.domain) { + case "system": { + const action = request.action as string; + switch (request.action) { + case "getPlatform": + return createSuccessResponse(requestId, systemService.getPlatform()); + case "getAssetBasePath": + return createSuccessResponse(requestId, systemService.getAssetBasePath()); + case "getCapabilities": + return createSuccessResponse(requestId, await systemService.getCapabilities()); + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported system action: ${action}`, + ); + } + } + + case "project": { + const action = request.action as string; + switch (request.action) { + case "getCurrentContext": + return createSuccessResponse(requestId, projectService.getCurrentContext()); + case "saveProjectFile": + return createSuccessResponse( + requestId, + await projectService.saveProjectFile( + request.payload.projectData, + request.payload.suggestedName, + request.payload.existingProjectPath, + ), + ); + case "loadProjectFile": + return createSuccessResponse(requestId, await projectService.loadProjectFile()); + case "loadCurrentProjectFile": + return createSuccessResponse( + requestId, + await projectService.loadCurrentProjectFile(), + ); + case "setCurrentVideoPath": + return createSuccessResponse( + requestId, + projectService.setCurrentVideoPath(request.payload.path), + ); + case "getCurrentVideoPath": + return createSuccessResponse(requestId, projectService.getCurrentVideoPath()); + case "clearCurrentVideoPath": + return createSuccessResponse(requestId, projectService.clearCurrentVideoPath()); + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported project action: ${action}`, + ); + } + } + + case "cursor": { + const action = request.action as string; + switch (request.action) { + case "getCapabilities": + return createSuccessResponse(requestId, await cursorService.getCapabilities()); + case "getTelemetry": + return createSuccessResponse( + requestId, + await cursorService.getTelemetry(request.payload?.videoPath), + ); + case "getRecordingData": + return createSuccessResponse( + requestId, + await cursorService.getRecordingData(request.payload?.videoPath), + ); + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported cursor action: ${action}`, + ); + } + } + + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported bridge domain: ${domain}`, + ); + } + } catch (error) { + return createErrorResponse( + requestId, + "INTERNAL_ERROR", + error instanceof Error ? error.message : "Unknown native bridge error.", + true, + ); + } + }); +} diff --git a/electron/main.ts b/electron/main.ts index 7e19d468..26ec301c 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -12,8 +12,7 @@ import { systemPreferences, Tray, } from "electron"; -import { mainT, setMainLocale } from "./i18n"; -import { registerIpcHandlers } from "./ipc/handlers"; +import { registerIpcHandlers, getSelectedDesktopSource } from "./ipc/handlers"; import { createEditorWindow, createHudOverlayWindow, createSourceSelectorWindow } from "./windows"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -336,6 +335,21 @@ app.on("activate", () => { // Register all IPC handlers when app is ready app.whenReady().then(async () => { + // Intercept getDisplayMedia to return the pre-selected source without the cursor. + // The source is cached synchronously at select-source time to avoid async delays here. + session.defaultSession.setDisplayMediaRequestHandler((_request, callback) => { + const source = getSelectedDesktopSource(); + if (!source) { + callback({}); + return; + } + callback({ + video: source, + // WASAPI loopback provides system audio capture on Windows. + ...(process.platform === "win32" && { audio: "loopback" as const }), + }); + }); + // Allow microphone/media permission checks session.defaultSession.setPermissionCheckHandler((_webContents, permission) => { const allowed = ["media", "audioCapture", "microphone", "videoCapture", "camera"]; diff --git a/electron/native-bridge/cursor/adapter.ts b/electron/native-bridge/cursor/adapter.ts new file mode 100644 index 00000000..cdb88e24 --- /dev/null +++ b/electron/native-bridge/cursor/adapter.ts @@ -0,0 +1,20 @@ +import type { + CursorCapabilities, + CursorProviderKind, + CursorRecordingData, + CursorTelemetryPoint, +} from "../../../src/native/contracts"; + +export interface CursorTelemetryLoadResult { + success: boolean; + samples: CursorTelemetryPoint[]; + message?: string; + error?: string; +} + +export interface CursorNativeAdapter { + readonly kind: CursorProviderKind; + getCapabilities(): Promise; + getRecordingData(videoPath?: string | null): Promise; + getTelemetry(videoPath?: string | null): Promise; +} diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts new file mode 100644 index 00000000..4e0f75c0 --- /dev/null +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -0,0 +1,31 @@ +import type { Rectangle } from "electron"; +import type { CursorRecordingSession } from "./session"; +import { TelemetryRecordingSession } from "./telemetryRecordingSession"; +import { WindowsNativeRecordingSession } from "./windowsNativeRecordingSession"; + +interface CreateCursorRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + platform: NodeJS.Platform; + sampleIntervalMs: number; + sourceId?: string | null; +} + +export function createCursorRecordingSession( + options: CreateCursorRecordingSessionOptions, +): CursorRecordingSession { + if (options.platform === "win32") { + return new WindowsNativeRecordingSession({ + getDisplayBounds: options.getDisplayBounds, + maxSamples: options.maxSamples, + sampleIntervalMs: options.sampleIntervalMs, + sourceId: options.sourceId, + }); + } + + return new TelemetryRecordingSession({ + getDisplayBounds: options.getDisplayBounds, + maxSamples: options.maxSamples, + sampleIntervalMs: options.sampleIntervalMs, + }); +} diff --git a/electron/native-bridge/cursor/recording/session.ts b/electron/native-bridge/cursor/recording/session.ts new file mode 100644 index 00000000..9cebe9f4 --- /dev/null +++ b/electron/native-bridge/cursor/recording/session.ts @@ -0,0 +1,6 @@ +import type { CursorRecordingData } from "../../../../src/native/contracts"; + +export interface CursorRecordingSession { + start(): Promise; + stop(): Promise; +} diff --git a/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts new file mode 100644 index 00000000..dd42871b --- /dev/null +++ b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts @@ -0,0 +1,62 @@ +import { type Rectangle, screen } from "electron"; +import type { CursorRecordingData, CursorRecordingSample } from "../../../../src/native/contracts"; +import type { CursorRecordingSession } from "./session"; + +interface TelemetryRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + sampleIntervalMs: number; +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +export class TelemetryRecordingSession implements CursorRecordingSession { + private samples: CursorRecordingSample[] = []; + private interval: NodeJS.Timeout | null = null; + private startTimeMs = 0; + + constructor(private readonly options: TelemetryRecordingSessionOptions) {} + + async start(): Promise { + this.samples = []; + this.startTimeMs = Date.now(); + this.captureSample(); + this.interval = setInterval(() => { + this.captureSample(); + }, this.options.sampleIntervalMs); + } + + async stop(): Promise { + if (this.interval) { + clearInterval(this.interval); + this.interval = null; + } + + return { + version: 2, + provider: "none", + samples: this.samples, + assets: [], + }; + } + + private captureSample() { + const cursor = screen.getCursorScreenPoint(); + const display = this.options.getDisplayBounds() ?? screen.getDisplayNearestPoint(cursor).bounds; + const width = Math.max(1, display.width); + const height = Math.max(1, display.height); + + this.samples.push({ + timeMs: Math.max(0, Date.now() - this.startTimeMs), + cx: clamp((cursor.x - display.x) / width, 0, 1), + cy: clamp((cursor.y - display.y) / height, 0, 1), + visible: true, + }); + + if (this.samples.length > this.options.maxSamples) { + this.samples.shift(); + } + } +} diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts new file mode 100644 index 00000000..b7a11cb4 --- /dev/null +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts @@ -0,0 +1,216 @@ +export function parseWindowHandleFromSourceId(sourceId?: string | null) { + if (!sourceId?.startsWith("window:")) { + return null; + } + + const handlePart = sourceId.split(":")[1]; + if (!handlePart || !/^\d+$/.test(handlePart)) { + return null; + } + + return handlePart; +} + +export function buildPowerShellCommand(sampleIntervalMs: number, windowHandle?: string | null) { + const script = String.raw` +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Drawing + +$targetWindowHandle = ${windowHandle ? `'${windowHandle}'` : '$null'} + +$source = @" +using System; +using System.Runtime.InteropServices; + +public static class OpenScreenCursorInterop { + [StructLayout(LayoutKind.Sequential)] + public struct POINT { + public int X; + public int Y; + } + + [StructLayout(LayoutKind.Sequential)] + public struct CURSORINFO { + public int cbSize; + public int flags; + public IntPtr hCursor; + public POINT ptScreenPos; + } + + [StructLayout(LayoutKind.Sequential)] + public struct ICONINFO { + [MarshalAs(UnmanagedType.Bool)] + public bool fIcon; + public int xHotspot; + public int yHotspot; + public IntPtr hbmMask; + public IntPtr hbmColor; + } + + [StructLayout(LayoutKind.Sequential)] + public struct RECT { + public int Left; + public int Top; + public int Right; + public int Bottom; + } + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetCursorInfo(ref CURSORINFO pci); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool IsWindow(IntPtr hWnd); + + [DllImport("user32.dll", SetLastError = true)] + public static extern IntPtr CopyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DestroyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo); + + [DllImport("gdi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DeleteObject(IntPtr hObject); +} +"@ + +Add-Type -TypeDefinition $source + +function Write-JsonLine($payload) { + [Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6)) +} + +function Get-TargetBounds() { + if ([string]::IsNullOrWhiteSpace($targetWindowHandle)) { + return $null + } + + try { + $handleValue = [int64]::Parse($targetWindowHandle) + $windowHandle = [IntPtr]::new($handleValue) + if (-not [OpenScreenCursorInterop]::IsWindow($windowHandle)) { + return $null + } + + $rect = New-Object OpenScreenCursorInterop+RECT + if (-not [OpenScreenCursorInterop]::GetWindowRect($windowHandle, [ref]$rect)) { + return $null + } + + $width = $rect.Right - $rect.Left + $height = $rect.Bottom - $rect.Top + if ($width -le 0 -or $height -le 0) { + return $null + } + + return @{ + x = $rect.Left + y = $rect.Top + width = $width + height = $height + } + } + catch { + return $null + } +} + +function Get-CursorAsset($cursorHandle, $cursorId) { + $copiedHandle = [OpenScreenCursorInterop]::CopyIcon($cursorHandle) + if ($copiedHandle -eq [IntPtr]::Zero) { + return $null + } + + $iconInfo = New-Object OpenScreenCursorInterop+ICONINFO + $hasIconInfo = [OpenScreenCursorInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo) + + try { + $icon = [System.Drawing.Icon]::FromHandle($copiedHandle) + $bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + $memoryStream = New-Object System.IO.MemoryStream + + try { + $graphics.Clear([System.Drawing.Color]::Transparent) + $graphics.DrawIcon($icon, 0, 0) + $bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png) + $base64 = [System.Convert]::ToBase64String($memoryStream.ToArray()) + + return @{ + id = $cursorId + imageDataUrl = "data:image/png;base64,$base64" + width = $bitmap.Width + height = $bitmap.Height + hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 } + hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 } + } + } + finally { + $memoryStream.Dispose() + $graphics.Dispose() + $bitmap.Dispose() + $icon.Dispose() + } + } + finally { + if ($hasIconInfo) { + if ($iconInfo.hbmMask -ne [IntPtr]::Zero) { + [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null + } + if ($iconInfo.hbmColor -ne [IntPtr]::Zero) { + [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null + } + } + [OpenScreenCursorInterop]::DestroyIcon($copiedHandle) | Out-Null + } +} + +Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() } + +$lastCursorId = $null +while ($true) { + $cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO + $cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO]) + + if (-not [OpenScreenCursorInterop]::GetCursorInfo([ref]$cursorInfo)) { + Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' } + Start-Sleep -Milliseconds ${sampleIntervalMs} + continue + } + + $visible = ($cursorInfo.flags -band 1) -ne 0 + $cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) } + $asset = $null + + if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) { + $asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId + $lastCursorId = $cursorId + } + + Write-JsonLine @{ + type = 'sample' + timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + x = $cursorInfo.ptScreenPos.X + y = $cursorInfo.ptScreenPos.Y + visible = $visible + handle = $cursorId + bounds = Get-TargetBounds + asset = $asset + } + + Start-Sleep -Milliseconds ${sampleIntervalMs} +} +`; + + return Buffer.from(script, "utf16le").toString("base64"); +} diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts new file mode 100644 index 00000000..d5e43d77 --- /dev/null +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts @@ -0,0 +1,268 @@ +import { type ChildProcessByStdio, spawn } from "node:child_process"; +import type { Readable } from "node:stream"; +import { screen } from "electron"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorAsset, +} from "../../../../src/native/contracts"; +import type { CursorRecordingSession } from "./session"; +import { buildPowerShellCommand, parseWindowHandleFromSourceId } from "./windowsNativeRecordingSession.script"; +import type { + WindowsCursorEvent, + WindowsNativeRecordingSessionOptions, +} from "./windowsNativeRecordingSession.types"; + +const READY_TIMEOUT_MS = 5_000; + +interface NormalizedSample { + sample: CursorRecordingSample; + withinBounds: boolean; +} + +export class WindowsNativeRecordingSession implements CursorRecordingSession { + private assets = new Map(); + private samples: CursorRecordingSample[] = []; + private process: ChildProcessByStdio | null = null; + private lineBuffer = ""; + private startTimeMs = 0; + private readyResolve: (() => void) | null = null; + private readyReject: ((error: Error) => void) | null = null; + private readyTimer: NodeJS.Timeout | null = null; + private sampleCount = 0; + private outOfBoundsSampleCount = 0; + + constructor(private readonly options: WindowsNativeRecordingSessionOptions) {} + + async start(): Promise { + this.assets.clear(); + this.samples = []; + this.lineBuffer = ""; + this.startTimeMs = Date.now(); + this.sampleCount = 0; + this.outOfBoundsSampleCount = 0; + + const encodedCommand = buildPowerShellCommand( + this.options.sampleIntervalMs, + parseWindowHandleFromSourceId(this.options.sourceId), + ); + const child = spawn( + "powershell.exe", + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-EncodedCommand", + encodedCommand, + ], + { + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }, + ); + + this.process = child; + this.logDiagnostic("spawn", { + pid: child.pid ?? null, + sampleIntervalMs: this.options.sampleIntervalMs, + sourceId: this.options.sourceId ?? null, + windowHandle: parseWindowHandleFromSourceId(this.options.sourceId), + }); + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + this.handleStdoutChunk(chunk); + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + const message = chunk.trim(); + if (message) { + this.logDiagnostic("stderr", { message }); + } + console.error("[cursor-native]", message); + }); + child.once("exit", (code, signal) => { + this.logDiagnostic("exit", { + code, + signal, + sampleCount: this.sampleCount, + assetCount: this.assets.size, + outOfBoundsSampleCount: this.outOfBoundsSampleCount, + }); + this.rejectReady(new Error(`Windows cursor helper exited before ready (code=${code}, signal=${signal})`)); + }); + child.once("error", (error) => { + this.logDiagnostic("process-error", { message: error.message }); + this.rejectReady(error); + }); + + await this.waitUntilReady(); + } + + async stop(): Promise { + const child = this.process; + this.process = null; + this.clearReadyState(); + + if (child && !child.killed) { + child.kill(); + } + + this.logDiagnostic("stop", { + sampleCount: this.sampleCount, + assetCount: this.assets.size, + outOfBoundsSampleCount: this.outOfBoundsSampleCount, + }); + + return { + version: 2, + provider: this.assets.size > 0 ? "native" : "none", + samples: this.samples, + assets: [...this.assets.values()], + }; + } + + private handleStdoutChunk(chunk: string) { + this.lineBuffer += chunk; + const lines = this.lineBuffer.split(/\r?\n/); + this.lineBuffer = lines.pop() ?? ""; + + for (const line of lines) { + const trimmedLine = line.trim(); + if (!trimmedLine) { + continue; + } + + try { + const payload = JSON.parse(trimmedLine) as WindowsCursorEvent; + this.handleEvent(payload); + } catch (error) { + console.error("Failed to parse Windows cursor helper output:", error, trimmedLine); + } + } + } + + private handleEvent(payload: WindowsCursorEvent) { + if (payload.type === "error") { + this.logDiagnostic("helper-error", { message: payload.message }); + console.error("Windows cursor helper error:", payload.message); + return; + } + + if (payload.type === "ready") { + this.logDiagnostic("ready", { timestampMs: payload.timestampMs }); + this.resolveReady(); + return; + } + + if (payload.asset?.id && !this.assets.has(payload.asset.id)) { + const assetDisplay = screen.getDisplayNearestPoint({ x: payload.x, y: payload.y }); + this.assets.set(payload.asset.id, { + id: payload.asset.id, + platform: "win32", + imageDataUrl: payload.asset.imageDataUrl, + width: payload.asset.width, + height: payload.asset.height, + hotspotX: payload.asset.hotspotX, + hotspotY: payload.asset.hotspotY, + scaleFactor: assetDisplay.scaleFactor, + }); + this.logDiagnostic("asset", { + id: payload.asset.id, + width: payload.asset.width, + height: payload.asset.height, + hotspotX: payload.asset.hotspotX, + hotspotY: payload.asset.hotspotY, + scaleFactor: assetDisplay.scaleFactor, + }); + } + + const normalized = this.normalizeSample(payload); + this.sampleCount += 1; + if (!normalized.withinBounds) { + this.outOfBoundsSampleCount += 1; + } + + this.samples.push(normalized.sample); + + if (this.samples.length > this.options.maxSamples) { + this.samples.shift(); + } + } + + private normalizeSample(payload: Extract): NormalizedSample { + const bounds = payload.bounds ?? this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds; + const width = Math.max(1, bounds.width); + const height = Math.max(1, bounds.height); + const normalizedX = (payload.x - bounds.x) / width; + const normalizedY = (payload.y - bounds.y) / height; + const withinBounds = normalizedX >= 0 && normalizedX <= 1 && normalizedY >= 0 && normalizedY <= 1; + + if (this.sampleCount === 0 || (!withinBounds && this.outOfBoundsSampleCount === 0)) { + this.logDiagnostic("sample", { + rawX: payload.x, + rawY: payload.y, + normalizedX, + normalizedY, + visible: payload.visible, + withinBounds, + bounds, + handle: payload.handle, + }); + } + + return { + withinBounds, + sample: { + timeMs: Math.max(0, payload.timestampMs - this.startTimeMs), + cx: normalizedX, + cy: normalizedY, + assetId: payload.handle, + visible: payload.visible && withinBounds, + }, + }; + } + + private waitUntilReady() { + return new Promise((resolve, reject) => { + this.readyResolve = resolve; + this.readyReject = reject; + this.readyTimer = setTimeout(() => { + this.rejectReady(new Error("Timed out waiting for Windows cursor helper readiness")); + }, READY_TIMEOUT_MS); + }); + } + + private resolveReady() { + const resolve = this.readyResolve; + this.clearReadyState(); + resolve?.(); + } + + private rejectReady(error: Error) { + const reject = this.readyReject; + this.clearReadyState(); + reject?.(error); + } + + private clearReadyState() { + if (this.readyTimer) { + clearTimeout(this.readyTimer); + this.readyTimer = null; + } + this.readyResolve = null; + this.readyReject = null; + } + + private logDiagnostic(event: string, data: Record) { + console.info( + "[cursor-native][win32]", + JSON.stringify({ + event, + ...data, + }), + ); + } +} diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts new file mode 100644 index 00000000..6efd59d2 --- /dev/null +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts @@ -0,0 +1,49 @@ +import type { Rectangle } from "electron"; + +export interface WindowsCursorSampleEvent { + type: "sample"; + timestampMs: number; + x: number; + y: number; + visible: boolean; + handle: string | null; + bounds?: { + x: number; + y: number; + width: number; + height: number; + } | null; + asset?: WindowsCursorAssetPayload; +} + +export interface WindowsCursorReadyEvent { + type: "ready"; + timestampMs: number; +} + +export interface WindowsCursorErrorEvent { + type: "error"; + timestampMs: number; + message: string; +} + +export interface WindowsCursorAssetPayload { + id: string; + imageDataUrl: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; +} + +export type WindowsCursorEvent = + | WindowsCursorSampleEvent + | WindowsCursorReadyEvent + | WindowsCursorErrorEvent; + +export interface WindowsNativeRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + sampleIntervalMs: number; + sourceId?: string | null; +} diff --git a/electron/native-bridge/cursor/telemetryCursorAdapter.ts b/electron/native-bridge/cursor/telemetryCursorAdapter.ts new file mode 100644 index 00000000..d0839952 --- /dev/null +++ b/electron/native-bridge/cursor/telemetryCursorAdapter.ts @@ -0,0 +1,48 @@ +import type { CursorCapabilities, CursorRecordingData } from "../../../src/native/contracts"; +import type { CursorNativeAdapter, CursorTelemetryLoadResult } from "./adapter"; + +interface TelemetryCursorAdapterOptions { + loadRecordingData: (videoPath: string) => Promise; + resolveVideoPath: (videoPath?: string | null) => string | null; + loadTelemetry: (videoPath: string) => Promise; +} + +export class TelemetryCursorAdapter implements CursorNativeAdapter { + readonly kind = "none" as const; + + constructor(private readonly options: TelemetryCursorAdapterOptions) {} + + async getCapabilities(): Promise { + return { + telemetry: true, + systemAssets: false, + provider: this.kind, + }; + } + + async getRecordingData(videoPath?: string | null): Promise { + const resolvedVideoPath = this.options.resolveVideoPath(videoPath); + if (!resolvedVideoPath) { + return { + version: 2, + provider: this.kind, + samples: [], + assets: [], + }; + } + + return this.options.loadRecordingData(resolvedVideoPath); + } + + async getTelemetry(videoPath?: string | null) { + const resolvedVideoPath = this.options.resolveVideoPath(videoPath); + if (!resolvedVideoPath) { + return { + success: true, + samples: [], + } satisfies CursorTelemetryLoadResult; + } + + return this.options.loadTelemetry(resolvedVideoPath); + } +} diff --git a/electron/native-bridge/services/cursorService.ts b/electron/native-bridge/services/cursorService.ts new file mode 100644 index 00000000..e3e9a255 --- /dev/null +++ b/electron/native-bridge/services/cursorService.ts @@ -0,0 +1,46 @@ +import type { + CursorCapabilities, + CursorRecordingData, + CursorTelemetryPoint, +} from "../../../src/native/contracts"; +import type { CursorNativeAdapter } from "../cursor/adapter"; +import type { NativeBridgeStateStore } from "../store"; + +interface CursorServiceOptions { + store: NativeBridgeStateStore; + adapter: CursorNativeAdapter; +} + +export class CursorService { + constructor(private readonly options: CursorServiceOptions) {} + + async getCapabilities(): Promise { + const capabilities = await this.options.adapter.getCapabilities(); + this.options.store.setCursorCapabilities(capabilities); + return capabilities; + } + + async getTelemetry(videoPath?: string | null): Promise { + const result = await this.options.adapter.getTelemetry(videoPath); + if (!result.success) { + throw new Error(result.message || result.error || "Failed to load cursor telemetry"); + } + + const resolvedVideoPath = videoPath ?? this.options.store.getState().project.currentVideoPath; + if (resolvedVideoPath) { + this.options.store.markCursorTelemetryLoaded(resolvedVideoPath, result.samples.length); + } + + return result.samples; + } + + async getRecordingData(videoPath?: string | null): Promise { + const data = await this.options.adapter.getRecordingData(videoPath); + const resolvedVideoPath = videoPath ?? this.options.store.getState().project.currentVideoPath; + if (resolvedVideoPath) { + this.options.store.markCursorTelemetryLoaded(resolvedVideoPath, data.samples.length); + } + + return data; + } +} diff --git a/electron/native-bridge/services/projectService.ts b/electron/native-bridge/services/projectService.ts new file mode 100644 index 00000000..e8d1cd5c --- /dev/null +++ b/electron/native-bridge/services/projectService.ts @@ -0,0 +1,80 @@ +import type { + ProjectContext, + ProjectFileResult, + ProjectPathResult, +} from "../../../src/native/contracts"; +import type { NativeBridgeStateStore } from "../store"; + +interface ProjectServiceOptions { + store: NativeBridgeStateStore; + getCurrentProjectPath: () => string | null; + getCurrentVideoPath: () => string | null; + saveProjectFile: ( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ) => Promise; + loadProjectFile: () => Promise; + loadCurrentProjectFile: () => Promise; + setCurrentVideoPath: (path: string) => ProjectPathResult; + getCurrentVideoPathResult: () => ProjectPathResult; + clearCurrentVideoPath: () => ProjectPathResult; +} + +export class ProjectService { + constructor(private readonly options: ProjectServiceOptions) {} + + getCurrentContext(): ProjectContext { + const context = { + currentProjectPath: this.options.getCurrentProjectPath(), + currentVideoPath: this.options.getCurrentVideoPath(), + }; + + this.options.store.setProjectContext(context); + return context; + } + + async saveProjectFile( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ) { + const result = await this.options.saveProjectFile( + projectData, + suggestedName, + existingProjectPath, + ); + this.getCurrentContext(); + return result; + } + + async loadProjectFile() { + const result = await this.options.loadProjectFile(); + this.getCurrentContext(); + return result; + } + + async loadCurrentProjectFile() { + const result = await this.options.loadCurrentProjectFile(); + this.getCurrentContext(); + return result; + } + + setCurrentVideoPath(path: string) { + const result = this.options.setCurrentVideoPath(path); + this.getCurrentContext(); + return result; + } + + getCurrentVideoPath() { + const result = this.options.getCurrentVideoPathResult(); + this.getCurrentContext(); + return result; + } + + clearCurrentVideoPath() { + const result = this.options.clearCurrentVideoPath(); + this.getCurrentContext(); + return result; + } +} diff --git a/electron/native-bridge/services/systemService.ts b/electron/native-bridge/services/systemService.ts new file mode 100644 index 00000000..50eff283 --- /dev/null +++ b/electron/native-bridge/services/systemService.ts @@ -0,0 +1,43 @@ +import type { + CursorCapabilities, + NativePlatform, + SystemCapabilities, +} from "../../../src/native/contracts"; +import { NATIVE_BRIDGE_VERSION } from "../../../src/native/contracts"; +import type { NativeBridgeStateStore } from "../store"; + +interface SystemServiceOptions { + store: NativeBridgeStateStore; + getPlatform: () => NativePlatform; + getAssetBasePath: () => string | null; + getCursorCapabilities: () => Promise; +} + +export class SystemService { + constructor(private readonly options: SystemServiceOptions) {} + + getPlatform() { + return this.options.getPlatform(); + } + + getAssetBasePath() { + return this.options.getAssetBasePath(); + } + + async getCapabilities(): Promise { + const platform = this.getPlatform(); + const cursorCapabilities = await this.options.getCursorCapabilities(); + + const capabilities: SystemCapabilities = { + bridgeVersion: NATIVE_BRIDGE_VERSION, + platform, + cursor: cursorCapabilities, + project: { + currentContext: true, + }, + }; + + this.options.store.setSystemCapabilities(capabilities); + return capabilities; + } +} diff --git a/electron/native-bridge/store.ts b/electron/native-bridge/store.ts new file mode 100644 index 00000000..dcdbed15 --- /dev/null +++ b/electron/native-bridge/store.ts @@ -0,0 +1,88 @@ +import type { + CursorCapabilities, + NativePlatform, + ProjectContext, + SystemCapabilities, +} from "../../src/native/contracts"; + +export interface NativeBridgeState { + system: { + platform: NativePlatform; + capabilities: SystemCapabilities | null; + }; + project: ProjectContext; + cursor: { + capabilities: CursorCapabilities | null; + lastTelemetryLoad: { + videoPath: string; + sampleCount: number; + loadedAt: number; + } | null; + }; +} + +export class NativeBridgeStateStore { + private state: NativeBridgeState; + + constructor(platform: NativePlatform) { + this.state = { + system: { + platform, + capabilities: null, + }, + project: { + currentProjectPath: null, + currentVideoPath: null, + }, + cursor: { + capabilities: null, + lastTelemetryLoad: null, + }, + }; + } + + getState() { + return this.state; + } + + setProjectContext(project: ProjectContext) { + this.state = { + ...this.state, + project, + }; + } + + setSystemCapabilities(capabilities: SystemCapabilities) { + this.state = { + ...this.state, + system: { + ...this.state.system, + capabilities, + }, + }; + } + + setCursorCapabilities(capabilities: CursorCapabilities) { + this.state = { + ...this.state, + cursor: { + ...this.state.cursor, + capabilities, + }, + }; + } + + markCursorTelemetryLoaded(videoPath: string, sampleCount: number) { + this.state = { + ...this.state, + cursor: { + ...this.state.cursor, + lastTelemetryLoad: { + videoPath, + sampleCount, + loadedAt: Date.now(), + }, + }, + }; + } +} diff --git a/electron/preload.ts b/electron/preload.ts index 8f1836bd..c6f4f4e5 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,7 +1,10 @@ import { contextBridge, ipcRenderer } from "electron"; -import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession"; +import { NATIVE_BRIDGE_CHANNEL, type NativeBridgeRequest } from "../src/native/contracts"; contextBridge.exposeInMainWorld("electronAPI", { + invokeNativeBridge: (request: NativeBridgeRequest) => { + return ipcRenderer.invoke(NATIVE_BRIDGE_CHANNEL, request) as Promise; + }, hudOverlayHide: () => { ipcRenderer.send("hud-overlay-hide"); }, diff --git a/src/assets/cursors/Cursor=Beachball.svg b/src/assets/cursors/Cursor=Beachball.svg new file mode 100644 index 00000000..30bdbe50 --- /dev/null +++ b/src/assets/cursors/Cursor=Beachball.svg @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/assets/cursors/Cursor=Cross.svg b/src/assets/cursors/Cursor=Cross.svg new file mode 100644 index 00000000..b404553d --- /dev/null +++ b/src/assets/cursors/Cursor=Cross.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Default.svg b/src/assets/cursors/Cursor=Default.svg new file mode 100644 index 00000000..f76f31fd --- /dev/null +++ b/src/assets/cursors/Cursor=Default.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Hand-(Grabbing).svg b/src/assets/cursors/Cursor=Hand-(Grabbing).svg new file mode 100644 index 00000000..08278675 --- /dev/null +++ b/src/assets/cursors/Cursor=Hand-(Grabbing).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Hand-(Open).svg b/src/assets/cursors/Cursor=Hand-(Open).svg new file mode 100644 index 00000000..4ceafb0f --- /dev/null +++ b/src/assets/cursors/Cursor=Hand-(Open).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Hand-(Pointing).svg b/src/assets/cursors/Cursor=Hand-(Pointing).svg new file mode 100644 index 00000000..19a70a67 --- /dev/null +++ b/src/assets/cursors/Cursor=Hand-(Pointing).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Menu.svg b/src/assets/cursors/Cursor=Menu.svg new file mode 100644 index 00000000..3489257b --- /dev/null +++ b/src/assets/cursors/Cursor=Menu.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/src/assets/cursors/Cursor=Move.svg b/src/assets/cursors/Cursor=Move.svg new file mode 100644 index 00000000..50e56b76 --- /dev/null +++ b/src/assets/cursors/Cursor=Move.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Down).svg b/src/assets/cursors/Cursor=Resize-(Down).svg new file mode 100644 index 00000000..fba36729 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Down).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Left).svg b/src/assets/cursors/Cursor=Resize-(Left).svg new file mode 100644 index 00000000..6e21fb77 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Left).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Left-Right).svg b/src/assets/cursors/Cursor=Resize-(Left-Right).svg new file mode 100644 index 00000000..7021d229 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Left-Right).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Right).svg b/src/assets/cursors/Cursor=Resize-(Right).svg new file mode 100644 index 00000000..1ce801ce --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Right).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Up).svg b/src/assets/cursors/Cursor=Resize-(Up).svg new file mode 100644 index 00000000..9c4ac0f0 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Up).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Up-Down).svg b/src/assets/cursors/Cursor=Resize-(Up-Down).svg new file mode 100644 index 00000000..b01a40e3 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Up-Down).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-North-East-South-West.svg b/src/assets/cursors/Cursor=Resize-North-East-South-West.svg new file mode 100644 index 00000000..1185c1ff --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-North-East-South-West.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-North-South.svg b/src/assets/cursors/Cursor=Resize-North-South.svg new file mode 100644 index 00000000..57eaa056 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-North-South.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-North-West-South-East.svg b/src/assets/cursors/Cursor=Resize-North-West-South-East.svg new file mode 100644 index 00000000..f00fc879 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-North-West-South-East.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-West-East.svg b/src/assets/cursors/Cursor=Resize-West-East.svg new file mode 100644 index 00000000..ef1929fb --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-West-East.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Text-Cursor.svg b/src/assets/cursors/Cursor=Text-Cursor.svg new file mode 100644 index 00000000..1bfd0809 --- /dev/null +++ b/src/assets/cursors/Cursor=Text-Cursor.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Zoom-In.svg b/src/assets/cursors/Cursor=Zoom-In.svg new file mode 100644 index 00000000..8ec9b3ce --- /dev/null +++ b/src/assets/cursors/Cursor=Zoom-In.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/assets/cursors/Cursor=Zoom-Out.svg b/src/assets/cursors/Cursor=Zoom-Out.svg new file mode 100644 index 00000000..810878ba --- /dev/null +++ b/src/assets/cursors/Cursor=Zoom-Out.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index 252f4d76..0e5ebf59 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -16,10 +16,7 @@ import { MdVolumeUp, } from "react-icons/md"; import { RxDragHandleDots2 } from "react-icons/rx"; -import { useI18n, useScopedT } from "@/contexts/I18nContext"; -import { type Locale, SUPPORTED_LOCALES } from "@/i18n/config"; -import { getLocaleName } from "@/i18n/loader"; -import { isMac as getIsMac } from "@/utils/platformUtils"; +import { nativeBridgeClient } from "@/native"; import { useAudioLevelMeter } from "../../hooks/useAudioLevelMeter"; import { useMicrophoneDevices } from "../../hooks/useMicrophoneDevices"; import { useScreenRecorder } from "../../hooks/useScreenRecorder"; @@ -135,6 +132,8 @@ export function LaunchWindow() { const [selectedSource, setSelectedSource] = useState("Screen"); const [hasSelectedSource, setHasSelectedSource] = useState(false); + const [hudPointerDownCount, setHudPointerDownCount] = useState(0); + const [recordPointerDownCount, setRecordPointerDownCount] = useState(0); useEffect(() => { const checkSelectedSource = async () => { @@ -170,13 +169,13 @@ export function LaunchWindow() { } if (result.success && result.path) { - await window.electronAPI.setCurrentVideoPath(result.path); + await nativeBridgeClient.project.setCurrentVideoPath(result.path); await window.electronAPI.switchToEditor(); } }; const openProjectFile = async () => { - const result = await window.electronAPI.loadProjectFile(); + const result = await nativeBridgeClient.project.loadProjectFile(); if (result.canceled || !result.success) return; await window.electronAPI.switchToEditor(); }; @@ -199,27 +198,24 @@ export function LaunchWindow() { }; return ( -
- {/* Language switcher — top-left, beside traffic lights */} -
- - -
- -
+ HUD TEST BUILD 2026-03-23 16:20 +
+
+ HUD PTR {hudPointerDownCount} | REC PTR {recordPointerDownCount} | SRC{" "} + {hasSelectedSource ? "YES" : "NO"} +
{/* Mic controls panel */} {showMicControls && (
{ + setRecordPointerDownCount((count) => count + 1); + }} onClick={hasSelectedSource ? toggleRecording : openSourceSelector} - disabled={!hasSelectedSource && !recording} style={{ flex: "0 0 auto" }} > {recording ? ( diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index f5afe35a..24d4106a 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -140,9 +140,18 @@ interface SettingsPanelProps { selectedSpeedValue?: PlaybackSpeed | null; onSpeedChange?: (speed: PlaybackSpeed) => void; onSpeedDelete?: (id: string) => void; - hasWebcam?: boolean; - webcamLayoutPreset?: WebcamLayoutPreset; - onWebcamLayoutPresetChange?: (preset: WebcamLayoutPreset) => void; + // Cursor settings + showCursor?: boolean; + onShowCursorChange?: (show: boolean) => void; + cursorSize?: number; + onCursorSizeChange?: (size: number) => void; + cursorSmoothing?: number; + onCursorSmoothingChange?: (smoothing: number) => void; + cursorMotionBlur?: number; + onCursorMotionBlurChange?: (blur: number) => void; + cursorClickBounce?: number; + onCursorClickBounceChange?: (bounce: number) => void; + hasCursorData?: boolean; } export default SettingsPanel; @@ -208,9 +217,17 @@ export function SettingsPanel({ selectedSpeedValue, onSpeedChange, onSpeedDelete, - hasWebcam = false, - webcamLayoutPreset = "picture-in-picture", - onWebcamLayoutPresetChange, + showCursor = true, + onShowCursorChange, + cursorSize = 3.0, + onCursorSizeChange, + cursorSmoothing = 0.67, + onCursorSmoothingChange, + cursorMotionBlur = 0.35, + onCursorMotionBlurChange, + cursorClickBounce = 2.5, + onCursorClickBounceChange, + hasCursorData = false, }: SettingsPanelProps) { const t = useScopedT("settings"); const [wallpaperPaths, setWallpaperPaths] = useState([]); @@ -252,8 +269,6 @@ export function SettingsPanel({ const [selectedColor, setSelectedColor] = useState("#ADADAD"); const [gradient, setGradient] = useState(GRADIENTS[0]); - const [showCropModal, setShowCropModal] = useState(false); - const cropSnapshotRef = useRef(null); const [cropAspectLocked, setCropAspectLocked] = useState(false); const [cropAspectRatio, setCropAspectRatio] = useState(""); @@ -353,6 +368,7 @@ export function SettingsPanel({ }, [cropRegion, videoWidth, videoHeight], ); + const [showCropDropdown, setShowCropDropdown] = useState(false); const zoomEnabled = Boolean(selectedZoomDepth); const trimEnabled = Boolean(selectedTrimId); @@ -416,20 +432,6 @@ export function SettingsPanel({ } }; - const handleCropToggle = () => { - if (!showCropModal && cropRegion) { - cropSnapshotRef.current = { ...cropRegion }; - } - setShowCropModal(!showCropModal); - }; - - const handleCropCancel = () => { - if (cropSnapshotRef.current && onCropChange) { - onCropChange(cropSnapshotRef.current); - } - setShowCropModal(false); - }; - // Find selected annotation const selectedAnnotation = selectedAnnotationId ? annotationRegions.find((a) => a.id === selectedAnnotationId) @@ -729,7 +731,7 @@ export function SettingsPanel({
- {showCropModal && cropRegion && onCropChange && ( + {showCropDropdown && cropRegion && onCropChange && ( <>
setShowCropDropdown(false)} />
@@ -920,7 +1014,7 @@ export function SettingsPanel({
@@ -1531,6 +1509,7 @@ export default function VideoEditor() {
{/* Right section: settings panel */} -
- pushState({ wallpaper: w })} - selectedZoomDepth={ - selectedZoomId ? zoomRegions.find((z) => z.id === selectedZoomId)?.depth : null - } - onZoomDepthChange={(depth) => selectedZoomId && handleZoomDepthChange(depth)} - selectedZoomId={selectedZoomId} - onZoomDelete={handleZoomDelete} - selectedTrimId={selectedTrimId} - onTrimDelete={handleTrimDelete} - shadowIntensity={shadowIntensity} - onShadowChange={(v) => updateState({ shadowIntensity: v })} - onShadowCommit={commitState} - showBlur={showBlur} - onBlurChange={(v) => pushState({ showBlur: v })} - motionBlurAmount={motionBlurAmount} - onMotionBlurChange={(v) => updateState({ motionBlurAmount: v })} - onMotionBlurCommit={commitState} - borderRadius={borderRadius} - onBorderRadiusChange={(v) => updateState({ borderRadius: v })} - onBorderRadiusCommit={commitState} - padding={padding} - onPaddingChange={(v) => updateState({ padding: v })} - onPaddingCommit={commitState} - cropRegion={cropRegion} - onCropChange={(r) => pushState({ cropRegion: r })} - aspectRatio={aspectRatio} - hasWebcam={Boolean(webcamVideoPath)} - webcamLayoutPreset={webcamLayoutPreset} - onWebcamLayoutPresetChange={(preset) => - pushState({ - webcamLayoutPreset: preset, - webcamPosition: preset === "vertical-stack" ? null : webcamPosition, - }) - } - videoElement={videoPlaybackRef.current?.video || null} - exportQuality={exportQuality} - onExportQualityChange={setExportQuality} - exportFormat={exportFormat} - onExportFormatChange={setExportFormat} - gifFrameRate={gifFrameRate} - onGifFrameRateChange={setGifFrameRate} - gifLoop={gifLoop} - onGifLoopChange={setGifLoop} - gifSizePreset={gifSizePreset} - onGifSizePresetChange={setGifSizePreset} - gifOutputDimensions={calculateOutputDimensions( - videoPlaybackRef.current?.video?.videoWidth || 1920, - videoPlaybackRef.current?.video?.videoHeight || 1080, - gifSizePreset, - GIF_SIZE_PRESETS, - aspectRatio === "native" - ? getNativeAspectRatioValue( - videoPlaybackRef.current?.video?.videoWidth || 1920, - videoPlaybackRef.current?.video?.videoHeight || 1080, - cropRegion, - ) - : getAspectRatioValue(aspectRatio), - )} - onExport={handleOpenExportDialog} - selectedAnnotationId={selectedAnnotationId} - annotationRegions={annotationRegions} - onAnnotationContentChange={handleAnnotationContentChange} - onAnnotationTypeChange={handleAnnotationTypeChange} - onAnnotationStyleChange={handleAnnotationStyleChange} - onAnnotationFigureDataChange={handleAnnotationFigureDataChange} - onAnnotationDelete={handleAnnotationDelete} - selectedSpeedId={selectedSpeedId} - selectedSpeedValue={ - selectedSpeedId - ? (speedRegions.find((r) => r.id === selectedSpeedId)?.speed ?? null) - : null - } - onSpeedChange={handleSpeedChange} - onSpeedDelete={handleSpeedDelete} - unsavedExport={unsavedExport} - onSaveUnsavedExport={handleSaveUnsavedExport} - /> -
+ pushState({ wallpaper: w })} + selectedZoomDepth={ + selectedZoomId ? zoomRegions.find((z) => z.id === selectedZoomId)?.depth : null + } + onZoomDepthChange={(depth) => selectedZoomId && handleZoomDepthChange(depth)} + selectedZoomId={selectedZoomId} + onZoomDelete={handleZoomDelete} + selectedTrimId={selectedTrimId} + onTrimDelete={handleTrimDelete} + shadowIntensity={shadowIntensity} + onShadowChange={(v) => updateState({ shadowIntensity: v })} + onShadowCommit={commitState} + showBlur={showBlur} + onBlurChange={(v) => pushState({ showBlur: v })} + motionBlurEnabled={motionBlurEnabled} + onMotionBlurChange={(v) => pushState({ motionBlurEnabled: v })} + borderRadius={borderRadius} + onBorderRadiusChange={(v) => updateState({ borderRadius: v })} + onBorderRadiusCommit={commitState} + padding={padding} + onPaddingChange={(v) => updateState({ padding: v })} + onPaddingCommit={commitState} + cropRegion={cropRegion} + onCropChange={(r) => pushState({ cropRegion: r })} + aspectRatio={aspectRatio} + videoElement={videoPlaybackRef.current?.video || null} + exportQuality={exportQuality} + onExportQualityChange={setExportQuality} + exportFormat={exportFormat} + onExportFormatChange={setExportFormat} + gifFrameRate={gifFrameRate} + onGifFrameRateChange={setGifFrameRate} + gifLoop={gifLoop} + onGifLoopChange={setGifLoop} + gifSizePreset={gifSizePreset} + onGifSizePresetChange={setGifSizePreset} + gifOutputDimensions={calculateOutputDimensions( + videoPlaybackRef.current?.video?.videoWidth || 1920, + videoPlaybackRef.current?.video?.videoHeight || 1080, + gifSizePreset, + GIF_SIZE_PRESETS, + )} + onExport={handleOpenExportDialog} + selectedAnnotationId={selectedAnnotationId} + annotationRegions={annotationRegions} + onAnnotationContentChange={handleAnnotationContentChange} + onAnnotationTypeChange={handleAnnotationTypeChange} + onAnnotationStyleChange={handleAnnotationStyleChange} + onAnnotationFigureDataChange={handleAnnotationFigureDataChange} + onAnnotationDelete={handleAnnotationDelete} + onSaveProject={handleSaveProject} + onLoadProject={handleLoadProject} + selectedSpeedId={selectedSpeedId} + selectedSpeedValue={ + selectedSpeedId + ? (speedRegions.find((r) => r.id === selectedSpeedId)?.speed ?? null) + : null + } + onSpeedChange={handleSpeedChange} + onSpeedDelete={handleSpeedDelete} + showCursor={showCursor} + onShowCursorChange={setShowCursor} + cursorSize={cursorSize} + onCursorSizeChange={setCursorSize} + cursorSmoothing={cursorSmoothing} + onCursorSmoothingChange={setCursorSmoothing} + cursorMotionBlur={cursorMotionBlur} + onCursorMotionBlurChange={setCursorMotionBlur} + cursorClickBounce={cursorClickBounce} + onCursorClickBounceChange={setCursorClickBounce} + hasCursorData={cursorTelemetry.length > 0} + />
void; onAnnotationPositionChange?: (id: string, position: { x: number; y: number }) => void; onAnnotationSizeChange?: (id: string, size: { width: number; height: number }) => void; + cursorTelemetry?: CursorTelemetryPoint[]; + showCursor?: boolean; + cursorSize?: number; + cursorSmoothing?: number; + cursorMotionBlur?: number; + cursorClickBounce?: number; } export interface VideoPlaybackRef { @@ -105,6 +123,22 @@ export interface VideoPlaybackRef { pause: () => void; } +function getResolvedVideoDuration(video: HTMLVideoElement): number | null { + if (Number.isFinite(video.duration) && video.duration > 0) { + return video.duration; + } + + if (video.seekable.length > 0) { + const lastRangeIndex = video.seekable.length - 1; + const seekableEnd = video.seekable.end(lastRangeIndex); + if (Number.isFinite(seekableEnd) && seekableEnd > 0) { + return seekableEnd; + } + } + + return null; +} + const VideoPlayback = forwardRef( ( { @@ -136,11 +170,18 @@ const VideoPlayback = forwardRef( trimRegions = [], speedRegions = [], aspectRatio, + cursorRecordingData, annotationRegions = [], selectedAnnotationId, onSelectAnnotation, onAnnotationPositionChange, onAnnotationSizeChange, + cursorTelemetry = [], + showCursor = false, + cursorSize = DEFAULT_CURSOR_SIZE, + cursorSmoothing = DEFAULT_CURSOR_SMOOTHING, + cursorMotionBlur = DEFAULT_CURSOR_MOTION_BLUR, + cursorClickBounce = DEFAULT_CURSOR_CLICK_BOUNCE, }, ref, ) => { @@ -151,6 +192,7 @@ const VideoPlayback = forwardRef( const videoSpriteRef = useRef(null); const videoContainerRef = useRef(null); const cameraContainerRef = useRef(null); + const cursorContainerRef = useRef(null); const timeUpdateAnimationRef = useRef(null); const [pixiReady, setPixiReady] = useState(false); const [videoReady, setVideoReady] = useState(false); @@ -189,11 +231,133 @@ const VideoPlayback = forwardRef( const layoutVideoContentRef = useRef<(() => void) | null>(null); const trimRegionsRef = useRef([]); const speedRegionsRef = useRef([]); - const motionBlurAmountRef = useRef(motionBlurAmount); + const motionBlurEnabledRef = useRef(motionBlurEnabled); + const cursorOverlayRef = useRef(null); + const cursorTelemetryRef = useRef([]); + const showCursorRef = useRef(showCursor); + const cursorSizeRef = useRef(cursorSize); + const cursorSmoothingRef = useRef(cursorSmoothing); + const cursorMotionBlurRef = useRef(cursorMotionBlur); + const cursorClickBounceRef = useRef(cursorClickBounce); const motionBlurStateRef = useRef(createMotionBlurState()); const onTimeUpdateRef = useRef(onTimeUpdate); const onPlayStateChangeRef = useRef(onPlayStateChange); const videoReadyRafRef = useRef(null); + const durationResolutionTimeoutRef = useRef(null); + const lastResolvedDurationRef = useRef(null); + const isResolvingDurationRef = useRef(false); + const hasNativeCursorRecordingRef = useRef(false); + const cursorRecordingDataRef = useRef(cursorRecordingData); + const cropRegionRef = useRef(cropRegion); + const nativeCursorImgRef = useRef(null); + + const hasNativeCursorRecording = useMemo( + () => hasNativeCursorRecordingData(cursorRecordingData), + [cursorRecordingData], + ); + + const syncResolvedDuration = useCallback( + (video: HTMLVideoElement) => { + const resolvedDuration = getResolvedVideoDuration(video); + if (!resolvedDuration) { + return false; + } + + const normalizedDuration = Math.round(resolvedDuration * 1000) / 1000; + if (lastResolvedDurationRef.current !== normalizedDuration) { + lastResolvedDurationRef.current = normalizedDuration; + onDurationChange(normalizedDuration); + } + + return true; + }, + [onDurationChange], + ); + + const forceResolveDuration = useCallback( + (video: HTMLVideoElement) => { + if (isResolvingDurationRef.current) { + return; + } + + if (video.readyState < HTMLMediaElement.HAVE_METADATA) { + return; + } + + isResolvingDurationRef.current = true; + const previousMuted = video.muted; + + const finalize = () => { + video.removeEventListener("durationchange", handleProgress); + video.removeEventListener("timeupdate", handleProgress); + video.removeEventListener("loadeddata", handleProgress); + video.removeEventListener("ended", handleProgress); + if (durationResolutionTimeoutRef.current) { + clearTimeout(durationResolutionTimeoutRef.current); + durationResolutionTimeoutRef.current = null; + } + video.muted = previousMuted; + isResolvingDurationRef.current = false; + }; + + const resolveCurrentDuration = () => { + if (syncResolvedDuration(video)) { + return true; + } + + if (Number.isFinite(video.currentTime) && video.currentTime > 0) { + lastResolvedDurationRef.current = null; + onDurationChange(Math.round(video.currentTime * 1000) / 1000); + return true; + } + + return false; + }; + + const handleProgress = () => { + if (!resolveCurrentDuration()) { + return; + } + + try { + video.pause(); + video.currentTime = 0; + } catch { + // no-op + } + currentTimeRef.current = 0; + finalize(); + }; + + video.addEventListener("durationchange", handleProgress); + video.addEventListener("timeupdate", handleProgress); + video.addEventListener("loadeddata", handleProgress); + video.addEventListener("ended", handleProgress); + durationResolutionTimeoutRef.current = window.setTimeout(() => { + handleProgress(); + finalize(); + }, 1500); + video.muted = true; + + const playAttempt = video.play(); + if (playAttempt && typeof playAttempt.catch === "function") { + playAttempt.catch(() => { + try { + video.currentTime = Math.max(video.currentTime, 0.1); + } catch { + finalize(); + } + }); + } + + try { + video.currentTime = Math.max(video.currentTime, 0.1); + } catch { + finalize(); + } + }, + [onDurationChange, syncResolvedDuration], + ); const clampFocusToStage = useCallback((focus: ZoomFocus, depth: ZoomDepth) => { return clampFocusToStageUtil(focus, depth, stageSizeRef.current); @@ -482,6 +646,53 @@ const VideoPlayback = forwardRef( motionBlurAmountRef.current = motionBlurAmount; }, [motionBlurAmount]); + useEffect(() => { + cursorTelemetryRef.current = cursorTelemetry; + }, [cursorTelemetry]); + + useEffect(() => { + showCursorRef.current = showCursor; + }, [showCursor]); + + useEffect(() => { + hasNativeCursorRecordingRef.current = hasNativeCursorRecording; + }, [hasNativeCursorRecording]); + + useEffect(() => { + cursorRecordingDataRef.current = cursorRecordingData; + }, [cursorRecordingData]); + + useEffect(() => { + cropRegionRef.current = cropRegion; + }, [cropRegion]); + + useEffect(() => { + cursorSizeRef.current = cursorSize; + }, [cursorSize]); + + useEffect(() => { + cursorSmoothingRef.current = cursorSmoothing; + }, [cursorSmoothing]); + + useEffect(() => { + cursorMotionBlurRef.current = cursorMotionBlur; + }, [cursorMotionBlur]); + + useEffect(() => { + cursorClickBounceRef.current = cursorClickBounce; + }, [cursorClickBounce]); + + // Sync cursor overlay config when settings change + useEffect(() => { + const overlay = cursorOverlayRef.current; + if (!overlay) return; + overlay.setDotRadius(DEFAULT_CURSOR_CONFIG.dotRadius * cursorSize); + overlay.setSmoothingFactor(cursorSmoothing); + overlay.setMotionBlur(cursorMotionBlur); + overlay.setClickBounce(cursorClickBounce); + overlay.reset(); + }, [cursorSize, cursorSmoothing, cursorMotionBlur, cursorClickBounce]); + useEffect(() => { onTimeUpdateRef.current = onTimeUpdate; }, [onTimeUpdate]); @@ -519,6 +730,9 @@ const VideoPlayback = forwardRef( appliedScale: 1, }; + // Reset cursor overlay smoothing on layout change + cursorOverlayRef.current?.reset(); + // Reset motion blur state for clean transitions motionBlurStateRef.current = createMotionBlurState(); @@ -547,14 +761,18 @@ const VideoPlayback = forwardRef( applyZoomTransform({ cameraContainer: container, blurFilter: blurFilterRef.current, + motionBlurFilter: motionBlurFilterRef.current, stageSize: stageSizeRef.current, baseMask: baseMaskRef.current, zoomScale: 1, + zoomProgress: 0, focusX: DEFAULT_FOCUS.cx, focusY: DEFAULT_FOCUS.cy, motionIntensity: 0, isPlaying: false, - motionBlurAmount: motionBlurAmountRef.current, + motionBlurAmount: motionBlurEnabledRef.current ? ZOOM_MOTION_BLUR_AMOUNT : 0, + motionBlurState: motionBlurStateRef.current, + frameTimeMs: performance.now(), }); requestAnimationFrame(() => { @@ -615,6 +833,13 @@ const VideoPlayback = forwardRef( let app: Application | null = null; (async () => { + let cursorOverlayEnabled = true; + try { + await preloadCursorAssets(); + } catch { + cursorOverlayEnabled = false; + } + app = new Application(); await app.init({ @@ -646,24 +871,48 @@ const VideoPlayback = forwardRef( videoContainerRef.current = videoContainer; cameraContainer.addChild(videoContainer); + // Cursor container - rendered above video + const cursorContainer = new Container(); + cursorContainerRef.current = cursorContainer; + cameraContainer.addChild(cursorContainer); + + // Cursor overlay - rendered above the masked video + if (cursorOverlayEnabled) { + const cursorOverlay = new PixiCursorOverlay({ + dotRadius: DEFAULT_CURSOR_CONFIG.dotRadius * cursorSizeRef.current, + smoothingFactor: cursorSmoothingRef.current, + motionBlur: cursorMotionBlurRef.current, + clickBounce: cursorClickBounceRef.current, + }); + cursorOverlayRef.current = cursorOverlay; + cursorContainer.addChild(cursorOverlay.container); + } + setPixiReady(true); })(); return () => { mounted = false; setPixiReady(false); + if (cursorOverlayRef.current) { + cursorOverlayRef.current.destroy(); + cursorOverlayRef.current = null; + } if (app && app.renderer) { app.destroy(true, { children: true, texture: true, textureSource: true }); } appRef.current = null; cameraContainerRef.current = null; videoContainerRef.current = null; + cursorContainerRef.current = null; videoSpriteRef.current = null; }; }, []); useEffect(() => { if (!videoPath) { + lastResolvedDurationRef.current = null; + isResolvingDurationRef.current = false; setVideoReady(false); return; } @@ -674,11 +923,18 @@ const VideoPlayback = forwardRef( video.currentTime = 0; allowPlaybackRef.current = false; lockedVideoDimensionsRef.current = null; + lastResolvedDurationRef.current = null; + isResolvingDurationRef.current = false; + if (durationResolutionTimeoutRef.current) { + clearTimeout(durationResolutionTimeoutRef.current); + durationResolutionTimeoutRef.current = null; + } setVideoReady(false); if (videoReadyRafRef.current) { cancelAnimationFrame(videoReadyRafRef.current); videoReadyRafRef.current = null; } + video.load(); }, [videoPath]); useEffect(() => { @@ -687,8 +943,9 @@ const VideoPlayback = forwardRef( const video = videoRef.current; const app = appRef.current; const videoContainer = videoContainerRef.current; + const cursorContainer = cursorContainerRef.current; - if (!video || !app || !videoContainer) return; + if (!video || !app || !videoContainer || !cursorContainer) return; if (video.videoWidth === 0 || video.videoHeight === 0) return; const source = VideoSource.from(video); @@ -708,6 +965,9 @@ const VideoPlayback = forwardRef( videoContainer.addChild(maskGraphics); videoContainer.mask = maskGraphics; maskGraphicsRef.current = maskGraphics; + if (cursorOverlayRef.current) { + cursorContainer.addChild(cursorOverlayRef.current.container); + } animationStateRef.current = { scale: 1, @@ -942,6 +1202,72 @@ const VideoPlayback = forwardRef( motionIntensity, motionVector, ); + + // Update cursor overlay + const cursorOverlay = cursorOverlayRef.current; + if (cursorOverlay) { + const timeMs = currentTimeRef.current; // already in ms + cursorOverlay.update( + cursorTelemetryRef.current, + timeMs, + baseMaskRef.current, + showCursorRef.current && !hasNativeCursorRecordingRef.current, + !isPlayingRef.current || isSeekingRef.current, + ); + } + + // Update native cursor image position at ticker rate (60fps) + const nativeCursorImg = nativeCursorImgRef.current; + if (nativeCursorImg) { + const cameraContainerRc = cameraContainerRef.current; + const videoContainerRc = videoContainerRef.current; + if ( + hasNativeCursorRecordingRef.current && + showCursorRef.current && + cameraContainerRc && + videoContainerRc + ) { + const timeMs = currentTimeRef.current; // already in ms + const frame = resolveInterpolatedNativeCursorFrame( + cursorRecordingDataRef.current, + timeMs, + ); + if (frame) { + const projectedPoint = projectNativeCursorToStage({ + cameraContainer: cameraContainerRc, + cropRegion: cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 }, + maskRect: baseMaskRef.current, + videoContainerPosition: { + x: videoContainerRc.x, + y: videoContainerRc.y, + }, + sample: frame.sample, + }); + if (projectedPoint) { + const metrics = getNativeCursorDisplayMetrics( + frame.asset, + window.devicePixelRatio || 1, + ); + const scale = Math.max(0, cursorSizeRef.current); + if (nativeCursorImg.dataset.cursorId !== frame.asset.id) { + nativeCursorImg.src = frame.asset.imageDataUrl; + nativeCursorImg.dataset.cursorId = frame.asset.id; + } + nativeCursorImg.style.left = `${projectedPoint.x - metrics.hotspotX * scale}px`; + nativeCursorImg.style.top = `${projectedPoint.y - metrics.hotspotY * scale}px`; + nativeCursorImg.style.width = `${metrics.width * scale}px`; + nativeCursorImg.style.height = `${metrics.height * scale}px`; + nativeCursorImg.style.display = "block"; + } else { + nativeCursorImg.style.display = "none"; + } + } else { + nativeCursorImg.style.display = "none"; + } + } else { + nativeCursorImg.style.display = "none"; + } + } }; app.ticker.add(ticker); @@ -954,8 +1280,12 @@ const VideoPlayback = forwardRef( const handleLoadedMetadata = (e: React.SyntheticEvent) => { const video = e.currentTarget; - onDurationChange(video.duration); - video.currentTime = 0; + const hasResolvedDuration = syncResolvedDuration(video); + if (!hasResolvedDuration) { + forceResolveDuration(video); + } else { + video.currentTime = 0; + } video.pause(); allowPlaybackRef.current = false; currentTimeRef.current = 0; @@ -968,6 +1298,9 @@ const VideoPlayback = forwardRef( const waitForRenderableFrame = () => { const hasDimensions = video.videoWidth > 0 && video.videoHeight > 0; const hasData = video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA; + if (!syncResolvedDuration(video)) { + forceResolveDuration(video); + } if (hasDimensions && hasData) { videoReadyRafRef.current = null; setVideoReady(true); @@ -1105,6 +1438,10 @@ const VideoPlayback = forwardRef( cancelAnimationFrame(videoReadyRafRef.current); videoReadyRafRef.current = null; } + if (durationResolutionTimeoutRef.current) { + clearTimeout(durationResolutionTimeoutRef.current); + durationResolutionTimeoutRef.current = null; + } }; }, []); @@ -1195,6 +1532,19 @@ const VideoPlayback = forwardRef( className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]" style={{ display: "none", pointerEvents: "none" }} /> + {hasNativeCursorRecording ? ( + + ) : null} {(() => { const filtered = (annotationRegions || []).filter((annotation) => { if (typeof annotation.startMs !== "number" || typeof annotation.endMs !== "number") @@ -1246,11 +1596,24 @@ const VideoPlayback = forwardRef( ref={videoRef} src={videoPath} className="hidden" - preload="metadata" + preload="auto" + muted playsInline onLoadedMetadata={handleLoadedMetadata} onDurationChange={(e) => { - onDurationChange(e.currentTarget.duration); + if (!syncResolvedDuration(e.currentTarget)) { + forceResolveDuration(e.currentTarget); + } + }} + onLoadedData={(e) => { + if (!syncResolvedDuration(e.currentTarget)) { + forceResolveDuration(e.currentTarget); + } + }} + onCanPlay={(e) => { + if (!syncResolvedDuration(e.currentTarget)) { + forceResolveDuration(e.currentTarget); + } }} onError={() => onError("Failed to load video")} /> diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 99f1bbaa..60786ca7 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -67,50 +67,21 @@ function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); } -function isFileUrl(value: string): boolean { - return /^file:\/\//i.test(value); -} - -function encodePathSegments(pathname: string, keepWindowsDrive = false): string { - return pathname - .split("/") - .map((segment, index) => { - if (!segment) return ""; - if (keepWindowsDrive && index === 1 && /^[a-zA-Z]:$/.test(segment)) { - return segment; - } - return encodeURIComponent(segment); - }) - .join("/"); -} - export function toFileUrl(filePath: string): string { const normalized = filePath.replace(/\\/g, "/"); - - // Windows drive path: C:/Users/... - if (/^[a-zA-Z]:\//.test(normalized)) { - return `file://${encodePathSegments(`/${normalized}`, true)}`; + if (normalized.match(/^[a-zA-Z]:/)) { + return `file:///${encodeURI(normalized)}`; } - - // UNC path: //server/share/... - if (normalized.startsWith("//")) { - const [host, ...pathParts] = normalized.replace(/^\/+/, "").split("/"); - const encodedPath = pathParts.map((part) => encodeURIComponent(part)).join("/"); - return encodedPath ? `file://${host}/${encodedPath}` : `file://${host}/`; - } - - const absolutePath = normalized.startsWith("/") ? normalized : `/${normalized}`; - return `file://${encodePathSegments(absolutePath)}`; + return `file://${encodeURI(normalized)}`; } export function fromFileUrl(fileUrl: string): string { - const value = fileUrl.trim(); - if (!isFileUrl(value)) { + if (!fileUrl.startsWith("file://")) { return fileUrl; } try { - const url = new URL(value); + const url = new URL(fileUrl); const pathname = decodeURIComponent(url.pathname); if (url.host && url.host !== "localhost") { @@ -123,13 +94,7 @@ export function fromFileUrl(fileUrl: string): string { return pathname; } catch { - const rawFallbackPath = value.replace(/^file:\/\//i, ""); - let fallbackPath = rawFallbackPath; - try { - fallbackPath = decodeURIComponent(rawFallbackPath); - } catch { - // Keep raw best-effort path if percent decoding fails. - } + const fallbackPath = decodeURIComponent(fileUrl.replace(/^file:\/\//, "")); return fallbackPath.replace(/^\/([a-zA-Z]:)/, "$1"); } } diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index b64aad0b..1c5c7091 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -51,6 +51,7 @@ const SUGGESTION_SPACING_MS = 1800; interface TimelineEditorProps { videoDuration: number; + hasVideoSource?: boolean; currentTime: number; onSeek?: (time: number) => void; cursorTelemetry?: CursorTelemetryPoint[]; @@ -733,6 +734,7 @@ function Timeline({ export default function TimelineEditor({ videoDuration, + hasVideoSource = false, currentTime, onSeek, cursorTelemetry = [], @@ -1356,8 +1358,14 @@ export default function TimelineEditor({
-

{t("emptyState.noVideo")}

-

{t("emptyState.dragAndDrop")}

+

+ {hasVideoSource ? "Loading Timeline" : "No Video Loaded"} +

+

+ {hasVideoSource + ? "Video opened, waiting for duration metadata" + : "Drag and drop a video to start editing"} +

); diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index ce49f8e4..841bb89a 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -29,8 +29,32 @@ export interface CursorTelemetryPoint { timeMs: number; cx: number; cy: number; + interactionType?: "move" | "click" | "double-click" | "right-click" | "middle-click" | "mouseup"; + cursorType?: + | "arrow" + | "text" + | "pointer" + | "crosshair" + | "open-hand" + | "closed-hand" + | "resize-ew" + | "resize-ns" + | "not-allowed"; } +export interface CursorVisualSettings { + size: number; + smoothing: number; + motionBlur: number; + clickBounce: number; +} + +export const DEFAULT_CURSOR_SIZE = 3.0; +export const DEFAULT_CURSOR_SMOOTHING = 0.67; +export const DEFAULT_CURSOR_MOTION_BLUR = 0.35; +export const DEFAULT_CURSOR_CLICK_BOUNCE = 2.5; +export const DEFAULT_ZOOM_MOTION_BLUR = 0.35; + export interface TrimRegion { id: string; startMs: number; diff --git a/src/components/video-editor/videoPlayback/cursorRenderer.ts b/src/components/video-editor/videoPlayback/cursorRenderer.ts new file mode 100644 index 00000000..7b912c54 --- /dev/null +++ b/src/components/video-editor/videoPlayback/cursorRenderer.ts @@ -0,0 +1,766 @@ +import { Assets, BlurFilter, Container, Graphics, Sprite, Texture } from "pixi.js"; +import { MotionBlurFilter } from "pixi-filters/motion-blur"; +import type { CursorTelemetryPoint } from "../types"; +import { + createSpringState, + getCursorSpringConfig, + resetSpringState, + stepSpringValue, +} from "./motionSmoothing"; +import { UPLOADED_CURSOR_SAMPLE_SIZE, uploadedCursorAssets } from "./uploadedCursorAssets"; + +type CursorAssetKey = NonNullable; + +/** System cursor asset from native helper (macOS only). */ +type SystemCursorAsset = { + dataUrl: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; +}; + +type LoadedCursorAsset = { + texture: Texture; + image: HTMLImageElement; + aspectRatio: number; + anchorX: number; + anchorY: number; +}; + +export interface CursorViewportRect { + x: number; + y: number; + width: number; + height: number; +} + +/** + * Configuration for cursor rendering. + */ +export interface CursorRenderConfig { + /** Base cursor height in pixels (at reference width of 1920px) */ + dotRadius: number; + /** Cursor fill color (hex number for PixiJS) */ + dotColor: number; + /** Cursor opacity (0–1) */ + dotAlpha: number; + /** Unused, kept for interface compatibility */ + trailLength: number; + /** Smoothing factor for cursor interpolation (0–1, lower = smoother/slower) */ + smoothingFactor: number; + /** Directional cursor motion blur amount. */ + motionBlur: number; + /** Click bounce multiplier. */ + clickBounce: number; +} + +export const DEFAULT_CURSOR_CONFIG: CursorRenderConfig = { + dotRadius: 28, + dotColor: 0xffffff, + dotAlpha: 0.95, + trailLength: 0, + smoothingFactor: 0.18, + motionBlur: 0, + clickBounce: 1, +}; + +const REFERENCE_WIDTH = 1920; +const MIN_CURSOR_VIEWPORT_SCALE = 0.55; +const CLICK_ANIMATION_MS = 140; +const CLICK_RING_FADE_MS = 240; +const CURSOR_MOTION_BLUR_BASE_MULTIPLIER = 0.08; +const CURSOR_TIME_DISCONTINUITY_MS = 100; +const CURSOR_SVG_DROP_SHADOW_FILTER = "drop-shadow(0px 2px 3px rgba(0, 0, 0, 0.35))"; +const CURSOR_SHADOW_COLOR = 0x000000; +const CURSOR_SHADOW_ALPHA = 0.35; +const CURSOR_SHADOW_OFFSET_X = 0; +const CURSOR_SHADOW_OFFSET_Y = 2; +const CURSOR_SHADOW_BLUR = 3; +const CURSOR_SHADOW_PADDING = 12; + +let cursorAssetsPromise: Promise | null = null; +let loadedCursorAssets: Partial> = {}; +const SUPPORTED_CURSOR_KEYS: CursorAssetKey[] = [ + "arrow", + "text", + "pointer", + "crosshair", + "open-hand", + "closed-hand", + "resize-ew", + "resize-ns", + "not-allowed", +]; + +function loadImage(dataUrl: string) { + return new Promise((resolve, reject) => { + const image = new Image(); + image.onload = () => resolve(image); + image.onerror = () => + reject(new Error(`Failed to load cursor image: ${dataUrl.slice(0, 128)}`)); + image.src = dataUrl; + }); +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +function getNormalizedAnchor( + systemAsset: SystemCursorAsset | undefined, + fallbackAnchor: { x: number; y: number }, +) { + if (!systemAsset || systemAsset.width <= 0 || systemAsset.height <= 0) { + return fallbackAnchor; + } + + return { + x: clamp(systemAsset.hotspotX / systemAsset.width, 0, 1), + y: clamp(systemAsset.hotspotY / systemAsset.height, 0, 1), + }; +} + +/** + * Loads an SVG at `sampleSize × sampleSize`, crops the trim region out of it, + * and returns a PNG data-URL of the cropped result. This is required because + * SVG files have their own natural pixel size (e.g. 32×32) which does not + * match the 1024-sample coordinate space used by the trim measurements. + */ +async function rasterizeAndCropSvg( + url: string, + sampleSize: number, + trimX: number, + trimY: number, + trimWidth: number, + trimHeight: number, +): Promise<{ dataUrl: string; width: number; height: number }> { + const img = await loadImage(url); + + // Draw at full sample size + const srcCanvas = document.createElement("canvas"); + srcCanvas.width = sampleSize; + srcCanvas.height = sampleSize; + const srcCtx = srcCanvas.getContext("2d")!; + srcCtx.drawImage(img, 0, 0, sampleSize, sampleSize); + + // Crop to trim bounds + const dstCanvas = document.createElement("canvas"); + dstCanvas.width = trimWidth; + dstCanvas.height = trimHeight; + const dstCtx = dstCanvas.getContext("2d")!; + dstCtx.drawImage(srcCanvas, trimX, trimY, trimWidth, trimHeight, 0, 0, trimWidth, trimHeight); + + return { + dataUrl: dstCanvas.toDataURL("image/png"), + width: dstCanvas.width, + height: dstCanvas.height, + }; +} + +function getCursorAsset(key: CursorAssetKey): LoadedCursorAsset { + const asset = loadedCursorAssets[key]; + if (!asset) { + throw new Error(`Missing cursor asset for ${key}`); + } + + return asset; +} + +function getAvailableCursorKeys(): CursorAssetKey[] { + const loadedKeys = Object.keys(loadedCursorAssets) as CursorAssetKey[]; + return loadedKeys.length > 0 ? loadedKeys : ["arrow"]; +} + +export async function preloadCursorAssets() { + if (!cursorAssetsPromise) { + cursorAssetsPromise = (async () => { + let systemCursors: Record = {}; + + try { + const api = window.electronAPI as Record; + if (typeof api.getSystemCursorAssets === "function") { + const result = await ( + api.getSystemCursorAssets as () => Promise<{ + success: boolean; + cursors?: Record; + }> + )(); + if (result.success && result.cursors) { + systemCursors = result.cursors; + } + } + } catch (error) { + console.warn("[CursorRenderer] Failed to fetch system cursor assets:", error); + } + + const entries = await Promise.all( + SUPPORTED_CURSOR_KEYS.map(async (key) => { + const systemAsset = systemCursors[key]; + const uploadedAsset = uploadedCursorAssets[key]; + const assetUrl = uploadedAsset?.url ?? systemAsset?.dataUrl; + + if (!assetUrl) { + console.warn(`[CursorRenderer] No cursor image for: ${key}`); + return null; + } + + try { + let finalUrl: string; + let width: number; + let height: number; + let normalizedAnchor: { x: number; y: number }; + + if (uploadedAsset) { + const { trim, fallbackAnchor } = uploadedAsset; + const rasterized = await rasterizeAndCropSvg( + assetUrl, + UPLOADED_CURSOR_SAMPLE_SIZE, + trim.x, + trim.y, + trim.width, + trim.height, + ); + finalUrl = rasterized.dataUrl; + width = rasterized.width; + height = rasterized.height; + normalizedAnchor = { + x: clamp((fallbackAnchor.x * trim.width) / width, 0, 1), + y: clamp((fallbackAnchor.y * trim.height) / height, 0, 1), + }; + } else { + finalUrl = assetUrl; + const img = await loadImage(finalUrl); + width = img.naturalWidth; + height = img.naturalHeight; + normalizedAnchor = getNormalizedAnchor(systemAsset, { x: 0, y: 0 }); + } + + await Assets.load(finalUrl); + const image = await loadImage(finalUrl); + const texture = Texture.from(finalUrl); + + return [ + key, + { + texture, + image, + aspectRatio: height > 0 ? width / height : 1, + anchorX: normalizedAnchor.x, + anchorY: normalizedAnchor.y, + } satisfies LoadedCursorAsset, + ] as const; + } catch (error) { + console.warn(`[CursorRenderer] Failed to load cursor image for: ${key}`, error); + return null; + } + }), + ); + + loadedCursorAssets = Object.fromEntries( + entries.filter(Boolean).map((entry) => entry!), + ) as Partial>; + + if (!loadedCursorAssets.arrow) { + throw new Error("Failed to initialize the fallback arrow cursor asset"); + } + })(); + } + + return cursorAssetsPromise; +} + +/** + * Interpolates cursor position from telemetry samples at a given time. + * Uses linear interpolation between the two nearest samples. + */ +export function interpolateCursorPosition( + samples: CursorTelemetryPoint[], + timeMs: number, +): { cx: number; cy: number } | null { + if (!samples || samples.length === 0) return null; + + if (timeMs <= samples[0].timeMs) { + return { cx: samples[0].cx, cy: samples[0].cy }; + } + + if (timeMs >= samples[samples.length - 1].timeMs) { + return { cx: samples[samples.length - 1].cx, cy: samples[samples.length - 1].cy }; + } + + let lo = 0; + let hi = samples.length - 1; + while (lo < hi - 1) { + const mid = (lo + hi) >> 1; + if (samples[mid].timeMs <= timeMs) { + lo = mid; + } else { + hi = mid; + } + } + + const a = samples[lo]; + const b = samples[hi]; + const span = b.timeMs - a.timeMs; + if (span <= 0) return { cx: a.cx, cy: a.cy }; + + const t = (timeMs - a.timeMs) / span; + return { + cx: a.cx + (b.cx - a.cx) * t, + cy: a.cy + (b.cy - a.cy) * t, + }; +} + +function findLatestSample(samples: CursorTelemetryPoint[], timeMs: number) { + if (samples.length === 0) return null; + + let lo = 0; + let hi = samples.length - 1; + while (lo < hi) { + const mid = Math.ceil((lo + hi) / 2); + if (samples[mid].timeMs <= timeMs) { + lo = mid; + } else { + hi = mid - 1; + } + } + + return samples[lo]?.timeMs <= timeMs ? samples[lo] : null; +} + +function findLatestInteractionSample(samples: CursorTelemetryPoint[], timeMs: number) { + for (let index = samples.length - 1; index >= 0; index -= 1) { + const sample = samples[index]; + if (sample.timeMs > timeMs) { + continue; + } + + if ( + sample.interactionType === "click" || + sample.interactionType === "double-click" || + sample.interactionType === "right-click" || + sample.interactionType === "middle-click" + ) { + return sample; + } + } + + return null; +} + +function findLatestStableCursorType(samples: CursorTelemetryPoint[], timeMs: number) { + // Binary search to find position at timeMs, then scan backwards + let lo = 0; + let hi = samples.length - 1; + while (lo < hi) { + const mid = Math.ceil((lo + hi) / 2); + if (samples[mid].timeMs <= timeMs) { + lo = mid; + } else { + hi = mid - 1; + } + } + + // Scan backwards from the position to find a sample with cursorType + // Skip click events only (not mouseup) to avoid transient re-type during clicks + for (let index = lo; index >= 0; index -= 1) { + const sample = samples[index]; + if (sample.timeMs > timeMs) { + continue; + } + + if (!sample.cursorType) { + continue; + } + + if ( + sample.interactionType === "click" || + sample.interactionType === "double-click" || + sample.interactionType === "right-click" || + sample.interactionType === "middle-click" + ) { + continue; + } + + return sample.cursorType; + } + + return findLatestSample(samples, timeMs)?.cursorType ?? "arrow"; +} + +function getCursorViewportScale(viewport: CursorViewportRect) { + return Math.max(MIN_CURSOR_VIEWPORT_SCALE, viewport.width / REFERENCE_WIDTH); +} + +function getCursorVisualState(samples: CursorTelemetryPoint[], timeMs: number) { + const latestClick = findLatestInteractionSample(samples, timeMs); + const interactionType = latestClick?.interactionType; + const ageMs = latestClick ? Math.max(0, timeMs - latestClick.timeMs) : Number.POSITIVE_INFINITY; + const isClickEvent = + interactionType === "click" || + interactionType === "double-click" || + interactionType === "right-click" || + interactionType === "middle-click"; + const clickBounceProgress = + latestClick && isClickEvent && ageMs <= CLICK_ANIMATION_MS ? 1 - ageMs / CLICK_ANIMATION_MS : 0; + + return { + cursorType: findLatestStableCursorType(samples, timeMs), + clickBounceProgress, + clickProgress: + latestClick && isClickEvent && ageMs <= CLICK_RING_FADE_MS + ? 1 - ageMs / CLICK_RING_FADE_MS + : 0, + }; +} + +/** + * Manages a smoothed cursor state that chases the interpolated target. + */ +export class SmoothedCursorState { + public x = 0.5; + public y = 0.5; + public trail: Array<{ x: number; y: number }> = []; + private smoothingFactor: number; + private trailLength: number; + private initialized = false; + private lastTimeMs: number | null = null; + private xSpring = createSpringState(0.5); + private ySpring = createSpringState(0.5); + + constructor(config: Pick) { + this.smoothingFactor = config.smoothingFactor; + this.trailLength = config.trailLength; + } + + update(targetX: number, targetY: number, timeMs: number): void { + if (!this.initialized) { + this.x = targetX; + this.y = targetY; + this.initialized = true; + this.lastTimeMs = timeMs; + this.xSpring.value = targetX; + this.ySpring.value = targetY; + this.xSpring.velocity = 0; + this.ySpring.velocity = 0; + this.xSpring.initialized = true; + this.ySpring.initialized = true; + this.trail = []; + return; + } + + if (this.smoothingFactor <= 0 || (this.lastTimeMs !== null && timeMs < this.lastTimeMs)) { + this.snapTo(targetX, targetY, timeMs); + return; + } + + this.trail.unshift({ x: this.x, y: this.y }); + if (this.trail.length > this.trailLength) { + this.trail.length = this.trailLength; + } + + const deltaMs = this.lastTimeMs === null ? 1000 / 60 : Math.max(1, timeMs - this.lastTimeMs); + this.lastTimeMs = timeMs; + + const springConfig = getCursorSpringConfig(this.smoothingFactor); + this.x = stepSpringValue(this.xSpring, targetX, deltaMs, springConfig); + this.y = stepSpringValue(this.ySpring, targetY, deltaMs, springConfig); + } + + setSmoothingFactor(smoothingFactor: number): void { + this.smoothingFactor = smoothingFactor; + } + + snapTo(targetX: number, targetY: number, timeMs: number): void { + this.x = targetX; + this.y = targetY; + this.initialized = true; + this.lastTimeMs = timeMs; + this.xSpring.value = targetX; + this.ySpring.value = targetY; + this.xSpring.velocity = 0; + this.ySpring.velocity = 0; + this.xSpring.initialized = true; + this.ySpring.initialized = true; + this.trail = []; + } + + reset(): void { + this.initialized = false; + this.lastTimeMs = null; + this.trail = []; + resetSpringState(this.xSpring, this.x); + resetSpringState(this.ySpring, this.y); + } +} + +function drawClickRing(graphics: Graphics, px: number, py: number, h: number, progress: number) { + void graphics; + void px; + void py; + void h; + void progress; +} + +export class PixiCursorOverlay { + public readonly container: Container; + private clickRingGraphics: Graphics; + private cursorShadowSprites: Partial>; + private cursorShadowFilters: Partial>; + private cursorSprites: Partial>; + private cursorMotionBlurFilter: MotionBlurFilter; + private state: SmoothedCursorState; + private config: CursorRenderConfig; + private lastRenderedPoint: { px: number; py: number } | null = null; + private lastRenderedTimeMs: number | null = null; + + constructor(config: Partial = {}) { + this.config = { ...DEFAULT_CURSOR_CONFIG, ...config }; + this.state = new SmoothedCursorState(this.config); + + this.container = new Container(); + this.container.label = "cursor-overlay"; + + this.clickRingGraphics = new Graphics(); + this.cursorShadowSprites = {}; + this.cursorShadowFilters = {}; + this.cursorSprites = {}; + for (const key of getAvailableCursorKeys()) { + const asset = getCursorAsset(key); + const shadowSprite = new Sprite(asset.texture); + shadowSprite.anchor.set(asset.anchorX, asset.anchorY); + shadowSprite.visible = false; + shadowSprite.tint = CURSOR_SHADOW_COLOR; + shadowSprite.alpha = CURSOR_SHADOW_ALPHA; + const shadowFilter = new BlurFilter(); + shadowFilter.blur = CURSOR_SHADOW_BLUR; + shadowFilter.quality = 4; + shadowFilter.padding = CURSOR_SHADOW_PADDING; + shadowSprite.filters = [shadowFilter]; + this.cursorShadowSprites[key] = shadowSprite; + this.cursorShadowFilters[key] = shadowFilter; + + const sprite = new Sprite(asset.texture); + sprite.anchor.set(asset.anchorX, asset.anchorY); + sprite.visible = false; + this.cursorSprites[key] = sprite; + } + + this.cursorMotionBlurFilter = new MotionBlurFilter([0, 0], 5, 0); + this.container.filters = null; + + this.container.addChild( + this.clickRingGraphics, + ...Object.values(this.cursorShadowSprites), + ...Object.values(this.cursorSprites), + ); + this.setMotionBlur(this.config.motionBlur); + } + + setDotRadius(dotRadius: number) { + this.config.dotRadius = dotRadius; + } + + setSmoothingFactor(smoothingFactor: number) { + this.config.smoothingFactor = smoothingFactor; + this.state.setSmoothingFactor(smoothingFactor); + } + + setMotionBlur(motionBlur: number) { + this.config.motionBlur = Math.max(0, motionBlur); + this.container.filters = this.config.motionBlur > 0 ? [this.cursorMotionBlurFilter] : null; + if (this.config.motionBlur <= 0) { + this.cursorMotionBlurFilter.velocity = { x: 0, y: 0 }; + this.cursorMotionBlurFilter.kernelSize = 5; + this.cursorMotionBlurFilter.offset = 0; + } + } + + setClickBounce(clickBounce: number) { + this.config.clickBounce = Math.max(0, clickBounce); + } + + update( + samples: CursorTelemetryPoint[], + timeMs: number, + viewport: CursorViewportRect, + visible: boolean, + freeze = false, + ): void { + if (!visible || samples.length === 0 || viewport.width <= 0 || viewport.height <= 0) { + this.container.visible = false; + this.lastRenderedPoint = null; + this.lastRenderedTimeMs = null; + this.cursorMotionBlurFilter.velocity = { x: 0, y: 0 }; + return; + } + + const target = interpolateCursorPosition(samples, timeMs); + if (!target) { + this.container.visible = false; + return; + } + + const sameFrameTime = + this.lastRenderedTimeMs !== null && Math.abs(this.lastRenderedTimeMs - timeMs) < 0.0001; + const hasTimeDiscontinuity = + this.lastRenderedTimeMs !== null && + Math.abs(timeMs - this.lastRenderedTimeMs) > CURSOR_TIME_DISCONTINUITY_MS; + + if (freeze || hasTimeDiscontinuity) { + if (!sameFrameTime || !this.lastRenderedPoint) { + this.state.snapTo(target.cx, target.cy, timeMs); + } + } else { + this.state.update(target.cx, target.cy, timeMs); + } + this.container.visible = true; + + const px = viewport.x + this.state.x * viewport.width; + const py = viewport.y + this.state.y * viewport.height; + const h = this.config.dotRadius * getCursorViewportScale(viewport); + const { cursorType, clickBounceProgress, clickProgress } = getCursorVisualState( + samples, + timeMs, + ); + const spriteKey = (cursorType in this.cursorSprites ? cursorType : "arrow") as CursorAssetKey; + const asset = getCursorAsset(spriteKey); + const shadowSprite = this.cursorShadowSprites[spriteKey] ?? this.cursorShadowSprites.arrow!; + const sprite = this.cursorSprites[spriteKey] ?? this.cursorSprites.arrow!; + const bounceScale = Math.max( + 0.72, + 1 - Math.sin(clickBounceProgress * Math.PI) * (0.08 * this.config.clickBounce), + ); + const scaledH = h; + + this.clickRingGraphics.clear(); + drawClickRing(this.clickRingGraphics, px, py, h, clickProgress); + + for (const [key, currentShadowSprite] of Object.entries(this.cursorShadowSprites) as Array< + [CursorAssetKey, Sprite] + >) { + currentShadowSprite.visible = key === spriteKey; + } + + for (const [key, currentSprite] of Object.entries(this.cursorSprites) as Array< + [CursorAssetKey, Sprite] + >) { + currentSprite.visible = key === spriteKey; + } + + if (shadowSprite) { + shadowSprite.height = scaledH * bounceScale; + shadowSprite.width = scaledH * bounceScale * asset.aspectRatio; + shadowSprite.position.set(px + CURSOR_SHADOW_OFFSET_X, py + CURSOR_SHADOW_OFFSET_Y); + } + + if (sprite) { + sprite.alpha = this.config.dotAlpha; + sprite.height = scaledH * bounceScale; + sprite.width = scaledH * bounceScale * asset.aspectRatio; + sprite.position.set(px, py); + } + + this.applyCursorMotionBlur(px, py, timeMs, freeze); + this.lastRenderedPoint = { px, py }; + this.lastRenderedTimeMs = timeMs; + } + + private applyCursorMotionBlur(px: number, py: number, timeMs: number, freeze: boolean) { + if ( + freeze || + this.config.motionBlur <= 0 || + !this.lastRenderedPoint || + this.lastRenderedTimeMs === null + ) { + this.cursorMotionBlurFilter.velocity = { x: 0, y: 0 }; + this.cursorMotionBlurFilter.kernelSize = 5; + this.cursorMotionBlurFilter.offset = 0; + return; + } + + const deltaMs = Math.max(1, timeMs - this.lastRenderedTimeMs); + const dx = px - this.lastRenderedPoint.px; + const dy = py - this.lastRenderedPoint.py; + const velocityScale = + (1000 / deltaMs) * this.config.motionBlur * CURSOR_MOTION_BLUR_BASE_MULTIPLIER; + const velocity = { + x: dx * velocityScale, + y: dy * velocityScale, + }; + const magnitude = Math.hypot(velocity.x, velocity.y); + + this.cursorMotionBlurFilter.velocity = magnitude > 0.05 ? velocity : { x: 0, y: 0 }; + this.cursorMotionBlurFilter.kernelSize = magnitude > 3 ? 9 : magnitude > 1 ? 7 : 5; + this.cursorMotionBlurFilter.offset = magnitude > 0.5 ? -0.25 : 0; + } + + reset(): void { + this.state.reset(); + this.clickRingGraphics.clear(); + for (const shadowSprite of Object.values(this.cursorShadowSprites)) { + shadowSprite.visible = false; + shadowSprite.scale.set(1); + } + for (const sprite of Object.values(this.cursorSprites)) { + sprite.visible = false; + sprite.scale.set(1); + } + this.container.visible = false; + this.lastRenderedPoint = null; + this.lastRenderedTimeMs = null; + this.cursorMotionBlurFilter.velocity = { x: 0, y: 0 }; + this.cursorMotionBlurFilter.kernelSize = 5; + this.cursorMotionBlurFilter.offset = 0; + } + + destroy(): void { + this.clickRingGraphics.destroy(); + for (const shadowFilter of Object.values(this.cursorShadowFilters)) { + shadowFilter.destroy(); + } + this.cursorMotionBlurFilter.destroy(); + this.container.destroy({ children: true }); + } +} + +export function drawCursorOnCanvas( + ctx: CanvasRenderingContext2D, + samples: CursorTelemetryPoint[], + timeMs: number, + viewport: CursorViewportRect, + smoothedState: SmoothedCursorState, + config: CursorRenderConfig = DEFAULT_CURSOR_CONFIG, +): void { + if (samples.length === 0 || viewport.width <= 0 || viewport.height <= 0) return; + + const target = interpolateCursorPosition(samples, timeMs); + if (!target) return; + + smoothedState.update(target.cx, target.cy, timeMs); + + const px = viewport.x + smoothedState.x * viewport.width; + const py = viewport.y + smoothedState.y * viewport.height; + const h = config.dotRadius * getCursorViewportScale(viewport); + const { cursorType, clickBounceProgress } = getCursorVisualState(samples, timeMs); + const spriteKey = ( + cursorType && loadedCursorAssets[cursorType] ? cursorType : "arrow" + ) as CursorAssetKey; + const asset = getCursorAsset(spriteKey); + const bounceScale = Math.max( + 0.72, + 1 - Math.sin(clickBounceProgress * Math.PI) * (0.08 * config.clickBounce), + ); + + ctx.save(); + ctx.filter = CURSOR_SVG_DROP_SHADOW_FILTER; + + const drawHeight = h * bounceScale; + const drawWidth = drawHeight * asset.aspectRatio; + const hotspotX = asset.anchorX * drawWidth; + const hotspotY = asset.anchorY * drawHeight; + ctx.globalAlpha = config.dotAlpha; + ctx.drawImage(asset.image, px - hotspotX, py - hotspotY, drawWidth, drawHeight); + + ctx.restore(); +} diff --git a/src/components/video-editor/videoPlayback/motionSmoothing.ts b/src/components/video-editor/videoPlayback/motionSmoothing.ts new file mode 100644 index 00000000..e2cec6fd --- /dev/null +++ b/src/components/video-editor/videoPlayback/motionSmoothing.ts @@ -0,0 +1,149 @@ +import { spring } from "motion"; + +export interface SpringState { + value: number; + velocity: number; + initialized: boolean; +} + +export interface SpringConfig { + stiffness: number; + damping: number; + mass: number; + restDelta?: number; + restSpeed?: number; +} + +const CURSOR_SMOOTHING_MIN = 0; +const CURSOR_SMOOTHING_MAX = 2; +const CURSOR_SMOOTHING_LEGACY_MAX = 0.5; + +export function createSpringState(initialValue = 0): SpringState { + return { + value: initialValue, + velocity: 0, + initialized: false, + }; +} + +export function resetSpringState(state: SpringState, initialValue?: number) { + if (typeof initialValue === "number") { + state.value = initialValue; + } + + state.velocity = 0; + state.initialized = false; +} + +export function clampDeltaMs(deltaMs: number, fallbackMs = 1000 / 60) { + if (!Number.isFinite(deltaMs) || deltaMs <= 0) { + return fallbackMs; + } + + return Math.min(80, Math.max(1, deltaMs)); +} + +export function stepSpringValue( + state: SpringState, + target: number, + deltaMs: number, + config: SpringConfig, +) { + const safeDeltaMs = clampDeltaMs(deltaMs); + + if (!state.initialized || !Number.isFinite(state.value)) { + state.value = target; + state.velocity = 0; + state.initialized = true; + return state.value; + } + + const restDelta = config.restDelta ?? 0.0005; + const restSpeed = config.restSpeed ?? 0.02; + + if (Math.abs(target - state.value) <= restDelta && Math.abs(state.velocity) <= restSpeed) { + state.value = target; + state.velocity = 0; + return state.value; + } + + const previousValue = state.value; + const generator = spring({ + keyframes: [state.value, target], + velocity: state.velocity, + stiffness: config.stiffness, + damping: config.damping, + mass: config.mass, + restDelta, + restSpeed, + }); + + const result = generator.next(safeDeltaMs); + state.value = result.done ? target : result.value; + state.velocity = ((state.value - previousValue) / safeDeltaMs) * 1000; + + if (result.done) { + state.velocity = 0; + } + + return state.value; +} + +export function getCursorSpringConfig(smoothingFactor: number): SpringConfig { + const clamped = Math.min(CURSOR_SMOOTHING_MAX, Math.max(CURSOR_SMOOTHING_MIN, smoothingFactor)); + + if (clamped <= 0) { + return { + stiffness: 1000, + damping: 100, + mass: 1, + restDelta: 0.0001, + restSpeed: 0.001, + }; + } + + if (clamped <= CURSOR_SMOOTHING_LEGACY_MAX) { + const legacyNormalized = Math.min( + 1, + Math.max( + 0, + (clamped - CURSOR_SMOOTHING_MIN) / (CURSOR_SMOOTHING_LEGACY_MAX - CURSOR_SMOOTHING_MIN), + ), + ); + + return { + stiffness: 760 - legacyNormalized * 420, + damping: 34 + legacyNormalized * 24, + mass: 0.55 + legacyNormalized * 0.45, + restDelta: 0.0002, + restSpeed: 0.01, + }; + } + + const extendedNormalized = Math.min( + 1, + Math.max( + 0, + (clamped - CURSOR_SMOOTHING_LEGACY_MAX) / + (CURSOR_SMOOTHING_MAX - CURSOR_SMOOTHING_LEGACY_MAX), + ), + ); + + return { + stiffness: 340 - extendedNormalized * 180, + damping: 58 + extendedNormalized * 22, + mass: 1 + extendedNormalized * 0.35, + restDelta: 0.0002, + restSpeed: 0.01, + }; +} + +export function getZoomSpringConfig(): SpringConfig { + return { + stiffness: 320, + damping: 40, + mass: 0.92, + restDelta: 0.0005, + restSpeed: 0.015, + }; +} diff --git a/src/components/video-editor/videoPlayback/uploadedCursorAssets.ts b/src/components/video-editor/videoPlayback/uploadedCursorAssets.ts new file mode 100644 index 00000000..4a0cd29b --- /dev/null +++ b/src/components/video-editor/videoPlayback/uploadedCursorAssets.ts @@ -0,0 +1,70 @@ +import crosshairUrl from "../../../assets/cursors/Cursor=Cross.svg"; +import arrowUrl from "../../../assets/cursors/Cursor=Default.svg"; +import closedHandUrl from "../../../assets/cursors/Cursor=Hand-(Grabbing).svg"; +import openHandUrl from "../../../assets/cursors/Cursor=Hand-(Open).svg"; +import pointerUrl from "../../../assets/cursors/Cursor=Hand-(Pointing).svg"; +import resizeNsUrl from "../../../assets/cursors/Cursor=Resize-North-South.svg"; +import resizeEwUrl from "../../../assets/cursors/Cursor=Resize-West-East.svg"; +import textUrl from "../../../assets/cursors/Cursor=Text-Cursor.svg"; +import type { CursorTelemetryPoint } from "../types"; + +type CursorAssetKey = NonNullable; + +export type UploadedCursorAsset = { + url: string; + trim: { + x: number; + y: number; + width: number; + height: number; + }; + fallbackAnchor: { + x: number; + y: number; + }; +}; + +export const UPLOADED_CURSOR_SAMPLE_SIZE = 1024; + +export const uploadedCursorAssets: Partial> = { + arrow: { + url: arrowUrl, + trim: { x: 480, y: 435, width: 333, height: 553 }, + fallbackAnchor: { x: 0.18, y: 0.1 }, + }, + text: { + url: textUrl, + trim: { x: 404, y: 192, width: 247, height: 596 }, + fallbackAnchor: { x: 0.5, y: 0.5 }, + }, + pointer: { + url: pointerUrl, + trim: { x: 352, y: 441, width: 466, height: 583 }, + fallbackAnchor: { x: 0.37, y: 0.08 }, + }, + crosshair: { + url: crosshairUrl, + trim: { x: 288, y: 288, width: 480, height: 480 }, + fallbackAnchor: { x: 0.5, y: 0.5 }, + }, + "open-hand": { + url: openHandUrl, + trim: { x: 288, y: 188, width: 512, height: 580 }, + fallbackAnchor: { x: 0.5, y: 0.28 }, + }, + "closed-hand": { + url: closedHandUrl, + trim: { x: 344, y: 365, width: 432, height: 403 }, + fallbackAnchor: { x: 0.5, y: 0.28 }, + }, + "resize-ew": { + url: resizeEwUrl, + trim: { x: 187, y: 384, width: 669, height: 270 }, + fallbackAnchor: { x: 0.5, y: 0.5 }, + }, + "resize-ns": { + url: resizeNsUrl, + trim: { x: 376, y: 178, width: 271, height: 669 }, + fallbackAnchor: { x: 0.5, y: 0.5 }, + }, +}; diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 0e1e9ba6..1346f979 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -1,8 +1,7 @@ import { fixWebmDuration } from "@fix-webm-duration/fix"; import { useCallback, useEffect, useRef, useState } from "react"; import { toast } from "sonner"; -import { useScopedT } from "@/contexts/I18nContext"; -import { requestCameraAccess } from "@/lib/requestCameraAccess"; +import { nativeBridgeClient } from "@/native"; const TARGET_FRAME_RATE = 60; const MIN_FRAME_RATE = 30; @@ -26,7 +25,6 @@ const CODEC_ALIGNMENT = 2; const RECORDER_TIMESLICE_MS = 1000; const BITS_PER_MEGABIT = 1_000_000; -const CHROME_MEDIA_SOURCE = "desktop"; const RECORDING_FILE_PREFIX = "recording-"; const VIDEO_FILE_EXTENSION = ".webm"; const WEBCAM_FILE_SUFFIX = "-webcam"; @@ -343,42 +341,19 @@ export function useScreenRecorder(): UseScreenRecorderReturn { let screenMediaStream: MediaStream; - const videoConstraints = { - mandatory: { - chromeMediaSource: CHROME_MEDIA_SOURCE, - chromeMediaSourceId: selectedSource.id, - maxWidth: TARGET_WIDTH, - maxHeight: TARGET_HEIGHT, - maxFrameRate: TARGET_FRAME_RATE, - minFrameRate: MIN_FRAME_RATE, - }, - }; - - if (systemAudioEnabled) { - try { - screenMediaStream = await navigator.mediaDevices.getUserMedia({ - audio: { - mandatory: { - chromeMediaSource: CHROME_MEDIA_SOURCE, - chromeMediaSourceId: selectedSource.id, - }, - }, - video: videoConstraints, - } as unknown as MediaStreamConstraints); - } catch (audioErr) { - console.warn("System audio capture failed, falling back to video-only:", audioErr); - toast.error(t("recording.systemAudioUnavailable")); - screenMediaStream = await navigator.mediaDevices.getUserMedia({ - audio: false, - video: videoConstraints, - } as unknown as MediaStreamConstraints); - } - } else { - screenMediaStream = await navigator.mediaDevices.getUserMedia({ - audio: false, - video: videoConstraints, - } as unknown as MediaStreamConstraints); - } + // getDisplayMedia + setDisplayMediaRequestHandler (main.ts) supplies the + // pre-selected source and honors cursor:"never" to exclude the system cursor + // from every captured frame. System audio is provided via WASAPI loopback + // on Windows when the user has enabled it. + screenMediaStream = await navigator.mediaDevices.getDisplayMedia({ + video: { + cursor: "never", + width: { max: TARGET_WIDTH }, + height: { max: TARGET_HEIGHT }, + frameRate: { ideal: TARGET_FRAME_RATE, min: MIN_FRAME_RATE }, + } as MediaTrackConstraints, + audio: systemAudioEnabled, + } as DisplayMediaStreamOptions); screenStream.current = screenMediaStream; if (microphoneEnabled) { @@ -507,9 +482,18 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }); } - recordingId.current = Date.now(); - startTime.current = recordingId.current; - allowAutoFinalize.current = true; + if (videoResult.path) { + await nativeBridgeClient.project.setCurrentVideoPath(videoResult.path); + } + + await window.electronAPI.switchToEditor(); + } catch (error) { + console.error("Error saving recording:", error); + } + }; + recorder.onerror = () => setRecording(false); + recorder.start(RECORDER_TIMESLICE_MS); + startTime.current = Date.now(); setRecording(true); window.electronAPI?.setRecordingState(true); diff --git a/src/lib/assetPath.ts b/src/lib/assetPath.ts index 8188de51..912f04a0 100644 --- a/src/lib/assetPath.ts +++ b/src/lib/assetPath.ts @@ -1,3 +1,5 @@ +import { nativeBridgeClient } from "@/native"; + function encodeRelativeAssetPath(relativePath: string): string { return relativePath .replace(/^\/+/, "") @@ -25,11 +27,9 @@ export async function getAssetPath(relativePath: string): Promise { return `/${encodedRelativePath}`; } - if (window.electronAPI && typeof window.electronAPI.getAssetBasePath === "function") { - const base = await window.electronAPI.getAssetBasePath(); - if (base) { - return new URL(encodedRelativePath, ensureTrailingSlash(base)).toString(); - } + const base = await nativeBridgeClient.system.getAssetBasePath(); + if (base) { + return new URL(encodedRelativePath, ensureTrailingSlash(base)).toString(); } } } catch { diff --git a/src/lib/cursor/nativeCursor.ts b/src/lib/cursor/nativeCursor.ts new file mode 100644 index 00000000..b32bd9e0 --- /dev/null +++ b/src/lib/cursor/nativeCursor.ts @@ -0,0 +1,171 @@ +import { type Container, Point } from "pixi.js"; +import type { CropRegion } from "@/components/video-editor/types"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorAsset, +} from "@/native/contracts"; + +export interface ActiveNativeCursorFrame { + asset: NativeCursorAsset; + sample: CursorRecordingSample; +} + +interface ProjectNativeCursorOptions { + cameraContainer: Container; + cropRegion: CropRegion; + maskRect: { x: number; y: number; width: number; height: number }; + videoContainerPosition: { x: number; y: number }; + sample: CursorRecordingSample; +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +export function hasNativeCursorRecordingData( + recordingData: CursorRecordingData | null | undefined, +): recordingData is CursorRecordingData { + return Boolean( + recordingData && + recordingData.provider === "native" && + recordingData.samples.length > 0 && + recordingData.assets.length > 0, + ); +} + +function getCroppedCursorPosition(sample: CursorRecordingSample, cropRegion: CropRegion) { + if (cropRegion.width <= 0 || cropRegion.height <= 0) { + return null; + } + + const croppedCx = (sample.cx - cropRegion.x) / cropRegion.width; + const croppedCy = (sample.cy - cropRegion.y) / cropRegion.height; + + if (croppedCx < 0 || croppedCx > 1 || croppedCy < 0 || croppedCy > 1) { + return null; + } + + return { + cx: clamp(croppedCx, 0, 1), + cy: clamp(croppedCy, 0, 1), + }; +} + +export function resolveActiveNativeCursorFrame( + recordingData: CursorRecordingData | null | undefined, + timeMs: number, +): ActiveNativeCursorFrame | null { + if (!hasNativeCursorRecordingData(recordingData)) { + return null; + } + + for (let index = recordingData.samples.length - 1; index >= 0; index -= 1) { + const sample = recordingData.samples[index]; + if (sample.timeMs > timeMs) { + continue; + } + + if (sample.visible === false || !sample.assetId) { + return null; + } + + const asset = recordingData.assets.find((candidate) => candidate.id === sample.assetId); + if (!asset) { + return null; + } + + return { sample, asset }; + } + + return null; +} + +export function resolveInterpolatedNativeCursorFrame( + recordingData: CursorRecordingData | null | undefined, + timeMs: number, +): ActiveNativeCursorFrame | null { + if (!hasNativeCursorRecordingData(recordingData)) { + return null; + } + + const samples = recordingData.samples; + let activeIndex = -1; + + for (let index = samples.length - 1; index >= 0; index -= 1) { + if (samples[index].timeMs <= timeMs) { + activeIndex = index; + break; + } + } + + if (activeIndex < 0) { + return null; + } + + const activeSample = samples[activeIndex]; + if (activeSample.visible === false || !activeSample.assetId) { + return null; + } + + const asset = recordingData.assets.find((candidate) => candidate.id === activeSample.assetId); + if (!asset) { + return null; + } + + const nextSample = samples[activeIndex + 1]; + if ( + !nextSample || + nextSample.timeMs <= activeSample.timeMs || + nextSample.visible === false || + nextSample.assetId !== activeSample.assetId || + timeMs <= activeSample.timeMs + ) { + return { asset, sample: activeSample }; + } + + const interpolation = clamp( + (timeMs - activeSample.timeMs) / (nextSample.timeMs - activeSample.timeMs), + 0, + 1, + ); + + return { + asset, + sample: { + ...activeSample, + cx: activeSample.cx + (nextSample.cx - activeSample.cx) * interpolation, + cy: activeSample.cy + (nextSample.cy - activeSample.cy) * interpolation, + }, + }; +} + +export function projectNativeCursorToStage({ + cameraContainer, + cropRegion, + maskRect, + videoContainerPosition, + sample, +}: ProjectNativeCursorOptions) { + const croppedPosition = getCroppedCursorPosition(sample, cropRegion); + if (!croppedPosition) { + return null; + } + + const localPoint = new Point( + videoContainerPosition.x + maskRect.x + croppedPosition.cx * maskRect.width, + videoContainerPosition.y + maskRect.y + croppedPosition.cy * maskRect.height, + ); + + return cameraContainer.toGlobal(localPoint); +} + +export function getNativeCursorDisplayMetrics(asset: NativeCursorAsset, deviceScaleFactor: number) { + const scaleFactor = asset.scaleFactor ?? deviceScaleFactor ?? 1; + return { + width: asset.width / scaleFactor, + height: asset.height / scaleFactor, + hotspotX: asset.hotspotX / scaleFactor, + hotspotY: asset.hotspotY / scaleFactor, + }; +} diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index 4a9b2bdc..f6e27718 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -32,11 +32,11 @@ import { type MotionBlurState, } from "@/components/video-editor/videoPlayback/zoomTransform"; import { - computeCompositeLayout, - getWebcamLayoutPresetDefinition, - type Size, - type StyledRenderRect, -} from "@/lib/compositeLayout"; + getNativeCursorDisplayMetrics, + projectNativeCursorToStage, + resolveInterpolatedNativeCursorFrame, +} from "@/lib/cursor/nativeCursor"; +import type { CursorRecordingData, NativeCursorAsset } from "@/native/contracts"; import { renderAnnotations } from "./annotationRenderer"; import { getLinearGradientPoints, @@ -57,6 +57,8 @@ interface FrameRenderConfig { borderRadius?: number; padding?: number; cropRegion: CropRegion; + cursorRecordingData?: CursorRecordingData | null; + cursorScale?: number; videoWidth: number; videoHeight: number; webcamSize?: Size | null; @@ -102,6 +104,7 @@ export class FrameRenderer { private shadowCtx: CanvasRenderingContext2D | null = null; private compositeCanvas: HTMLCanvasElement | null = null; private compositeCtx: CanvasRenderingContext2D | null = null; + private cursorImageCache = new Map(); private config: FrameRenderConfig; private animationState: AnimationState; private layoutCache: LayoutCache | null = null; @@ -386,6 +389,8 @@ export class FrameRenderer { // Composite with shadows to final output canvas this.compositeWithShadows(webcamFrame); + await this.drawNativeCursor(timeMs); + // Render annotations on top if present if ( this.config.annotationRegions && @@ -410,7 +415,67 @@ export class FrameRenderer { } } - private updateLayout(webcamFrame?: VideoFrame | null): void { + private async drawNativeCursor(timeMs: number) { + if (!this.compositeCtx || !this.cameraContainer || !this.videoContainer || !this.layoutCache) { + return; + } + + if ((this.config.cursorScale ?? 1) <= 0) { + return; + } + + const activeNativeCursor = resolveInterpolatedNativeCursorFrame( + this.config.cursorRecordingData, + timeMs, + ); + if (!activeNativeCursor) { + return; + } + + const projectedPoint = projectNativeCursorToStage({ + cameraContainer: this.cameraContainer, + cropRegion: this.config.cropRegion, + maskRect: this.layoutCache.maskRect, + videoContainerPosition: { + x: this.videoContainer.x, + y: this.videoContainer.y, + }, + sample: activeNativeCursor.sample, + }); + if (!projectedPoint) { + return; + } + + const image = await this.getCursorImage(activeNativeCursor.asset); + const metrics = getNativeCursorDisplayMetrics(activeNativeCursor.asset, 1); + const scale = Math.max(0, this.config.cursorScale ?? 1); + this.compositeCtx.drawImage( + image, + projectedPoint.x - metrics.hotspotX * scale, + projectedPoint.y - metrics.hotspotY * scale, + metrics.width * scale, + metrics.height * scale, + ); + } + + private async getCursorImage(asset: NativeCursorAsset) { + const cachedImage = this.cursorImageCache.get(asset.id); + if (cachedImage) { + return cachedImage; + } + + const image = new Image(); + await new Promise((resolve, reject) => { + image.onload = () => resolve(); + image.onerror = () => reject(new Error(`Failed to load cursor asset ${asset.id}`)); + image.src = asset.imageDataUrl; + }); + + this.cursorImageCache.set(asset.id, image); + return image; + } + + private updateLayout(): void { if (!this.app || !this.videoSprite || !this.maskGraphics || !this.videoContainer) return; const { width, height } = this.config; @@ -728,5 +793,6 @@ export class FrameRenderer { this.shadowCtx = null; this.compositeCanvas = null; this.compositeCtx = null; + this.cursorImageCache.clear(); } } diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts index 8aac0b56..84b68205 100644 --- a/src/lib/exporter/gifExporter.ts +++ b/src/lib/exporter/gifExporter.ts @@ -7,7 +7,7 @@ import type { WebcamLayoutPreset, ZoomRegion, } from "@/components/video-editor/types"; -import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue"; +import type { CursorRecordingData } from "@/native/contracts"; import { FrameRenderer } from "./frameRenderer"; import { StreamingVideoDecoder } from "./streamingDecoder"; import type { @@ -40,8 +40,8 @@ interface GifExporterConfig { padding?: number; videoPadding?: number; cropRegion: CropRegion; - webcamLayoutPreset?: WebcamLayoutPreset; - webcamPosition?: { cx: number; cy: number } | null; + cursorRecordingData?: CursorRecordingData | null; + cursorScale?: number; annotationRegions?: AnnotationRegion[]; previewWidth?: number; previewHeight?: number; @@ -137,6 +137,8 @@ export class GifExporter { borderRadius: this.config.borderRadius, padding: this.config.padding, cropRegion: this.config.cropRegion, + cursorRecordingData: this.config.cursorRecordingData, + cursorScale: this.config.cursorScale, videoWidth: videoInfo.width, videoHeight: videoInfo.height, webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null, diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 1b75a2e8..4c0196c6 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -6,7 +6,7 @@ import type { WebcamLayoutPreset, ZoomRegion, } from "@/components/video-editor/types"; -import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue"; +import type { CursorRecordingData } from "@/native/contracts"; import { AudioProcessor } from "./audioEncoder"; import { FrameRenderer } from "./frameRenderer"; import { VideoMuxer } from "./muxer"; @@ -31,8 +31,8 @@ interface VideoExporterConfig extends ExportConfig { padding?: number; videoPadding?: number; cropRegion: CropRegion; - webcamLayoutPreset?: WebcamLayoutPreset; - webcamPosition?: { cx: number; cy: number } | null; + cursorRecordingData?: CursorRecordingData | null; + cursorScale?: number; annotationRegions?: AnnotationRegion[]; previewWidth?: number; previewHeight?: number; @@ -130,6 +130,8 @@ export class VideoExporter { borderRadius: this.config.borderRadius, padding: this.config.padding, cropRegion: this.config.cropRegion, + cursorRecordingData: this.config.cursorRecordingData, + cursorScale: this.config.cursorScale, videoWidth: videoInfo.width, videoHeight: videoInfo.height, webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null, diff --git a/src/native/client.ts b/src/native/client.ts new file mode 100644 index 00000000..3f53ce48 --- /dev/null +++ b/src/native/client.ts @@ -0,0 +1,133 @@ +import { + type CursorCapabilities, + type CursorRecordingData, + type CursorTelemetryPoint, + NATIVE_BRIDGE_CHANNEL, + type NativeBridgeRequest, + type NativeBridgeResponse, + type NativePlatform, + type ProjectContext, + type ProjectFileResult, + type ProjectPathResult, + type SystemCapabilities, +} from "./contracts"; + +function createRequestId() { + if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") { + return crypto.randomUUID(); + } + + return `req-${Date.now()}-${Math.random().toString(16).slice(2)}`; +} + +function getElectronBridge() { + if (!window.electronAPI?.invokeNativeBridge) { + throw new Error( + `Native bridge unavailable. Expected ${NATIVE_BRIDGE_CHANNEL} transport in preload.`, + ); + } + + return window.electronAPI.invokeNativeBridge; +} + +export async function invokeNativeBridge( + request: NativeBridgeRequest, +): Promise> { + const invoke = getElectronBridge(); + return invoke({ + ...request, + requestId: request.requestId ?? createRequestId(), + }); +} + +export async function requireNativeBridgeData(request: NativeBridgeRequest): Promise { + const response = await invokeNativeBridge(request); + if (!response.ok) { + throw new Error(response.error.message); + } + + return response.data; +} + +export const nativeBridgeClient = { + rawInvoke: invokeNativeBridge, + system: { + getPlatform: () => + requireNativeBridgeData({ + domain: "system", + action: "getPlatform", + }), + getAssetBasePath: () => + requireNativeBridgeData({ + domain: "system", + action: "getAssetBasePath", + }), + getCapabilities: () => + requireNativeBridgeData({ + domain: "system", + action: "getCapabilities", + }), + }, + project: { + getCurrentContext: () => + requireNativeBridgeData({ + domain: "project", + action: "getCurrentContext", + }), + saveProjectFile: (projectData: unknown, suggestedName?: string, existingProjectPath?: string) => + requireNativeBridgeData({ + domain: "project", + action: "saveProjectFile", + payload: { + projectData, + suggestedName, + existingProjectPath, + }, + }), + loadProjectFile: () => + requireNativeBridgeData({ + domain: "project", + action: "loadProjectFile", + }), + loadCurrentProjectFile: () => + requireNativeBridgeData({ + domain: "project", + action: "loadCurrentProjectFile", + }), + setCurrentVideoPath: (path: string) => + requireNativeBridgeData({ + domain: "project", + action: "setCurrentVideoPath", + payload: { path }, + }), + getCurrentVideoPath: () => + requireNativeBridgeData({ + domain: "project", + action: "getCurrentVideoPath", + }), + clearCurrentVideoPath: () => + requireNativeBridgeData({ + domain: "project", + action: "clearCurrentVideoPath", + }), + }, + cursor: { + getCapabilities: () => + requireNativeBridgeData({ + domain: "cursor", + action: "getCapabilities", + }), + getRecordingData: (videoPath?: string) => + requireNativeBridgeData({ + domain: "cursor", + action: "getRecordingData", + payload: videoPath ? { videoPath } : {}, + }), + getTelemetry: (videoPath?: string) => + requireNativeBridgeData({ + domain: "cursor", + action: "getTelemetry", + payload: videoPath ? { videoPath } : {}, + }), + }, +}; diff --git a/src/native/contracts.ts b/src/native/contracts.ts new file mode 100644 index 00000000..73d53dbb --- /dev/null +++ b/src/native/contracts.ts @@ -0,0 +1,209 @@ +export const NATIVE_BRIDGE_CHANNEL = "native-bridge:invoke"; +export const NATIVE_BRIDGE_VERSION = 1; + +export type NativePlatform = "darwin" | "win32" | "linux"; +export type CursorProviderKind = "native" | "none"; + +export interface CursorTelemetryPoint { + timeMs: number; + cx: number; + cy: number; +} + +export interface CursorRecordingSample extends CursorTelemetryPoint { + assetId?: string | null; + visible?: boolean; +} + +export interface NativeCursorAsset { + id: string; + platform: NativePlatform; + imageDataUrl: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; + scaleFactor?: number; +} + +export interface CursorRecordingData { + version: number; + provider: CursorProviderKind; + samples: CursorRecordingSample[]; + assets: NativeCursorAsset[]; +} + +export interface CursorCapabilities { + telemetry: boolean; + systemAssets: boolean; + provider: CursorProviderKind; +} + +export interface SystemCapabilities { + bridgeVersion: typeof NATIVE_BRIDGE_VERSION; + platform: NativePlatform; + cursor: CursorCapabilities; + project: { + currentContext: boolean; + }; +} + +export interface ProjectContext { + currentProjectPath: string | null; + currentVideoPath: string | null; +} + +export interface ProjectPathResult { + success: boolean; + path?: string; + message?: string; + canceled?: boolean; + error?: string; +} + +export interface ProjectFileResult { + success: boolean; + path?: string; + project?: unknown; + message?: string; + canceled?: boolean; + error?: string; +} + +export type NativeBridgeErrorCode = + | "INVALID_REQUEST" + | "UNSUPPORTED_ACTION" + | "NOT_FOUND" + | "UNAVAILABLE" + | "INTERNAL_ERROR"; + +export interface NativeBridgeError { + code: NativeBridgeErrorCode; + message: string; + retryable: boolean; +} + +export interface NativeBridgeMeta { + version: typeof NATIVE_BRIDGE_VERSION; + requestId: string; + timestampMs: number; +} + +export interface NativeBridgeSuccess { + ok: true; + data: TData; + meta: NativeBridgeMeta; +} + +export interface NativeBridgeFailure { + ok: false; + error: NativeBridgeError; + meta: NativeBridgeMeta; +} + +export type NativeBridgeResponse = + | NativeBridgeSuccess + | NativeBridgeFailure; + +type EmptyPayload = Record; + +export type NativeBridgeRequest = + | { + domain: "system"; + action: "getPlatform"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "system"; + action: "getAssetBasePath"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "system"; + action: "getCapabilities"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "project"; + action: "getCurrentContext"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "project"; + action: "saveProjectFile"; + payload: { + projectData: unknown; + suggestedName?: string; + existingProjectPath?: string; + }; + requestId?: string; + } + | { + domain: "project"; + action: "loadProjectFile"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "project"; + action: "loadCurrentProjectFile"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "project"; + action: "setCurrentVideoPath"; + payload: { + path: string; + }; + requestId?: string; + } + | { + domain: "project"; + action: "getCurrentVideoPath"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "project"; + action: "clearCurrentVideoPath"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "cursor"; + action: "getCapabilities"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "cursor"; + action: "getTelemetry"; + payload?: { + videoPath?: string; + }; + requestId?: string; + } + | { + domain: "cursor"; + action: "getRecordingData"; + payload?: { + videoPath?: string; + }; + requestId?: string; + }; + +export type NativeBridgeEventName = + | "project.contextChanged" + | "cursor.providerChanged" + | "cursor.telemetryLoaded"; + +export interface NativeBridgeEvent { + name: NativeBridgeEventName; + payload: TPayload; + meta: NativeBridgeMeta; +} diff --git a/src/native/hooks/useCursorRecordingData.ts b/src/native/hooks/useCursorRecordingData.ts new file mode 100644 index 00000000..6b3451a8 --- /dev/null +++ b/src/native/hooks/useCursorRecordingData.ts @@ -0,0 +1,61 @@ +import { useEffect, useState } from "react"; +import type { CursorRecordingData } from "@/native/contracts"; +import { nativeBridgeClient } from "../client"; + +interface UseCursorRecordingDataResult { + data: CursorRecordingData | null; + loading: boolean; + error: string | null; +} + +export function useCursorRecordingData(videoPath: string | null): UseCursorRecordingDataResult { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + + async function loadCursorRecordingData() { + if (!videoPath) { + setData(null); + setLoading(false); + setError(null); + return; + } + + setLoading(true); + setError(null); + + try { + const nextData = await nativeBridgeClient.cursor.getRecordingData(videoPath); + if (!cancelled) { + setData(nextData); + } + } catch (nextError) { + if (!cancelled) { + setData(null); + setError( + nextError instanceof Error ? nextError.message : "Failed to load cursor recording data", + ); + } + } finally { + if (!cancelled) { + setLoading(false); + } + } + } + + loadCursorRecordingData(); + + return () => { + cancelled = true; + }; + }, [videoPath]); + + return { + data, + loading, + error, + }; +} diff --git a/src/native/hooks/useCursorTelemetry.ts b/src/native/hooks/useCursorTelemetry.ts new file mode 100644 index 00000000..16176805 --- /dev/null +++ b/src/native/hooks/useCursorTelemetry.ts @@ -0,0 +1,61 @@ +import { useEffect, useState } from "react"; +import type { CursorTelemetryPoint } from "@/components/video-editor/types"; +import { nativeBridgeClient } from "../client"; + +interface UseCursorTelemetryResult { + samples: CursorTelemetryPoint[]; + loading: boolean; + error: string | null; +} + +export function useCursorTelemetry(videoPath: string | null): UseCursorTelemetryResult { + const [samples, setSamples] = useState([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + + async function loadCursorTelemetry() { + if (!videoPath) { + setSamples([]); + setLoading(false); + setError(null); + return; + } + + setLoading(true); + setError(null); + + try { + const nextSamples = await nativeBridgeClient.cursor.getTelemetry(videoPath); + if (!cancelled) { + setSamples(nextSamples); + } + } catch (nextError) { + if (!cancelled) { + setSamples([]); + setError( + nextError instanceof Error ? nextError.message : "Failed to load cursor telemetry", + ); + } + } finally { + if (!cancelled) { + setLoading(false); + } + } + } + + loadCursorTelemetry(); + + return () => { + cancelled = true; + }; + }, [videoPath]); + + return { + samples, + loading, + error, + }; +} diff --git a/src/native/index.ts b/src/native/index.ts new file mode 100644 index 00000000..817d1cf1 --- /dev/null +++ b/src/native/index.ts @@ -0,0 +1,4 @@ +export * from "./client"; +export * from "./contracts"; +export * from "./hooks/useCursorRecordingData"; +export * from "./hooks/useCursorTelemetry"; diff --git a/src/vite-env.d.ts b/src/vite-env.d.ts index 4e668f3c..feda744b 100644 --- a/src/vite-env.d.ts +++ b/src/vite-env.d.ts @@ -1,6 +1,8 @@ /// /// +import type { NativeBridgeRequest, NativeBridgeResponse } from "./native/contracts"; + interface ProcessedDesktopSource { id: string; name: string; @@ -15,110 +17,90 @@ interface CursorTelemetryPoint { cy: number; } -interface Window { - electronAPI: { - getSources: (opts: Electron.SourcesOptions) => Promise; - switchToEditor: () => Promise; - openSourceSelector: () => Promise; - selectSource: (source: ProcessedDesktopSource) => Promise; - getSelectedSource: () => Promise; - requestCameraAccess: () => Promise<{ - success: boolean; - granted: boolean; - status: string; - error?: string; - }>; - storeRecordedVideo: ( - videoData: ArrayBuffer, - fileName: string, - ) => Promise<{ - success: boolean; - path?: string; - session?: import("./lib/recordingSession").RecordingSession; - message?: string; - error?: string; - }>; - storeRecordedSession: ( - payload: import("./lib/recordingSession").StoreRecordedSessionInput, - ) => Promise<{ - success: boolean; - path?: string; - session?: import("./lib/recordingSession").RecordingSession; - message?: string; - error?: string; - }>; - getRecordedVideoPath: () => Promise<{ - success: boolean; - path?: string; - message?: string; - error?: string; - }>; - getAssetBasePath: () => Promise; - setRecordingState: (recording: boolean) => Promise; - getCursorTelemetry: (videoPath?: string) => Promise<{ - success: boolean; - samples: CursorTelemetryPoint[]; - message?: string; - error?: string; - }>; - onStopRecordingFromTray: (callback: () => void) => () => void; - openExternalUrl: (url: string) => Promise<{ success: boolean; error?: string }>; - saveExportedVideo: ( - videoData: ArrayBuffer, - fileName: string, - ) => Promise<{ - success: boolean; - path?: string; - message?: string; - canceled?: boolean; - }>; - openVideoFilePicker: () => Promise<{ success: boolean; path?: string; canceled?: boolean }>; - setCurrentVideoPath: (path: string) => Promise<{ success: boolean }>; - setCurrentRecordingSession: ( - session: import("./lib/recordingSession").RecordingSession | null, - ) => Promise<{ - success: boolean; - session?: import("./lib/recordingSession").RecordingSession; - }>; - getCurrentVideoPath: () => Promise<{ success: boolean; path?: string }>; - getCurrentRecordingSession: () => Promise<{ - success: boolean; - session?: import("./lib/recordingSession").RecordingSession; - }>; - clearCurrentVideoPath: () => Promise<{ success: boolean }>; - saveProjectFile: ( - projectData: unknown, - suggestedName?: string, - existingProjectPath?: string, - ) => Promise<{ - success: boolean; - path?: string; - message?: string; - canceled?: boolean; - error?: string; - }>; - loadProjectFile: () => Promise<{ - success: boolean; - path?: string; - project?: unknown; - message?: string; - canceled?: boolean; - error?: string; - }>; - loadCurrentProjectFile: () => Promise<{ - success: boolean; - path?: string; - project?: unknown; - message?: string; - canceled?: boolean; - error?: string; - }>; - onMenuLoadProject: (callback: () => void) => () => void; - onMenuSaveProject: (callback: () => void) => () => void; - onMenuSaveProjectAs: (callback: () => void) => () => void; - setMicrophoneExpanded: (expanded: boolean) => void; - setHasUnsavedChanges: (hasChanges: boolean) => void; - onRequestSaveBeforeClose: (callback: () => Promise | boolean) => () => void; - setLocale: (locale: string) => Promise; - }; +declare global { + interface Window { + electronAPI: { + invokeNativeBridge: ( + request: NativeBridgeRequest, + ) => Promise>; + getSources: (opts: Electron.SourcesOptions) => Promise; + switchToEditor: () => Promise; + openSourceSelector: () => Promise; + selectSource: (source: ProcessedDesktopSource) => Promise; + getSelectedSource: () => Promise; + storeRecordedVideo: ( + videoData: ArrayBuffer, + fileName: string, + ) => Promise<{ + success: boolean; + path?: string; + message: string; + error?: string; + }>; + getRecordedVideoPath: () => Promise<{ + success: boolean; + path?: string; + message?: string; + error?: string; + }>; + getAssetBasePath: () => Promise; + setRecordingState: (recording: boolean) => Promise; + getCursorTelemetry: (videoPath?: string) => Promise<{ + success: boolean; + samples: CursorTelemetryPoint[]; + message?: string; + error?: string; + }>; + onStopRecordingFromTray: (callback: () => void) => () => void; + openExternalUrl: (url: string) => Promise<{ success: boolean; error?: string }>; + saveExportedVideo: ( + videoData: ArrayBuffer, + fileName: string, + ) => Promise<{ + success: boolean; + path?: string; + message?: string; + canceled?: boolean; + }>; + openVideoFilePicker: () => Promise<{ success: boolean; path?: string; canceled?: boolean }>; + setCurrentVideoPath: (path: string) => Promise<{ success: boolean }>; + getCurrentVideoPath: () => Promise<{ success: boolean; path?: string }>; + clearCurrentVideoPath: () => Promise<{ success: boolean }>; + saveProjectFile: ( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ) => Promise<{ + success: boolean; + path?: string; + message?: string; + canceled?: boolean; + error?: string; + }>; + loadProjectFile: () => Promise<{ + success: boolean; + path?: string; + project?: unknown; + message?: string; + canceled?: boolean; + error?: string; + }>; + loadCurrentProjectFile: () => Promise<{ + success: boolean; + path?: string; + project?: unknown; + message?: string; + canceled?: boolean; + error?: string; + }>; + onMenuLoadProject: (callback: () => void) => () => void; + onMenuSaveProject: (callback: () => void) => () => void; + onMenuSaveProjectAs: (callback: () => void) => () => void; + setMicrophoneExpanded: (expanded: boolean) => void; + setHasUnsavedChanges: (hasChanges: boolean) => void; + onRequestSaveBeforeClose: (callback: () => Promise | boolean) => () => void; + }; + } } + +export {};