diff --git a/.changeset/bright-chapters-search.md b/.changeset/bright-chapters-search.md new file mode 100644 index 0000000000..43065833e7 --- /dev/null +++ b/.changeset/bright-chapters-search.md @@ -0,0 +1,6 @@ +--- +"@agent-native/core": patch +"@agent-native/skills": patch +--- + +Add r5 semantic chapter metadata and provenance-aware local Screen Memory search, publish a reusable Rewind skill whose installer also repairs the local Screen Memory MCP connection, and keep the standalone skills installer aligned. diff --git a/packages/core/src/cli/mcp-config-writers.spec.ts b/packages/core/src/cli/mcp-config-writers.spec.ts index f1d44294d1..e87416ecd7 100644 --- a/packages/core/src/cli/mcp-config-writers.spec.ts +++ b/packages/core/src/cli/mcp-config-writers.spec.ts @@ -330,6 +330,20 @@ describe("writeHttpEntryForClient", () => { }); describe("buildLocalMcpEntryForClient", () => { + it("can configure a resolvable npx command for a self-installing MCP", () => { + expect( + buildLocalMcpEntryForClient( + "claude-code", + ["-y", "@agent-native/core@latest", "mcp", "screen-memory"], + {}, + "npx", + ), + ).toEqual({ + command: "npx", + args: ["-y", "@agent-native/core@latest", "mcp", "screen-memory"], + }); + }); + it("uses the OpenCode local command-array shape", () => { expect( buildLocalMcpEntryForClient("opencode", ["mcp", "serve"], { diff --git a/packages/core/src/cli/mcp-config-writers.ts b/packages/core/src/cli/mcp-config-writers.ts index d0a72d90b4..41ac468344 100644 --- a/packages/core/src/cli/mcp-config-writers.ts +++ b/packages/core/src/cli/mcp-config-writers.ts @@ -123,12 +123,13 @@ export function buildLocalMcpEntryForClient( client: ClientId, args: string[], env?: Record, + command = "agent-native", ): Record { const cleanEnv = env ? Object.fromEntries(Object.entries(env)) : {}; if (client === "opencode") { return { type: "local", - command: ["agent-native", ...args], + command: [command, ...args], enabled: true, ...(Object.keys(cleanEnv).length ? { environment: cleanEnv } : {}), }; @@ -136,13 +137,13 @@ export function buildLocalMcpEntryForClient( if (client === "github-copilot") { return { type: "stdio", - command: "agent-native", + command, args, ...(Object.keys(cleanEnv).length ? { env: cleanEnv } : {}), }; } return { - command: "agent-native", + command, args, ...(Object.keys(cleanEnv).length ? { env: cleanEnv } : {}), }; @@ -543,9 +544,10 @@ export function buildCodexLocalBlock( name: string, args: string[], env?: Record, + command = "agent-native", ): string { const lines: string[] = [codexMcpHeader(name)]; - lines.push(`command = "agent-native"`); + lines.push(`command = ${tomlQuote(command)}`); lines.push(`args = [${args.map(tomlQuote).join(", ")}]`); const cleanEnv = env ? Object.fromEntries(Object.entries(env)) : {}; if (Object.keys(cleanEnv).length) { diff --git a/packages/core/src/cli/mcp-screen-memory-store.spec.ts b/packages/core/src/cli/mcp-screen-memory-store.spec.ts new file mode 100644 index 0000000000..e6632353aa --- /dev/null +++ b/packages/core/src/cli/mcp-screen-memory-store.spec.ts @@ -0,0 +1,63 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { resolveScreenMemoryStoreDir } from "./mcp.js"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +function home(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "clips-store-resolve-")); + roots.push(root); + return root; +} + +describe("resolveScreenMemoryStoreDir", () => { + it("prefers an explicit directory, then the environment override", () => { + expect( + resolveScreenMemoryStoreDir({ + explicitDir: "/tmp/explicit-clips-store", + env: { CLIPS_SCREEN_MEMORY_DIR: "/tmp/env-clips-store" }, + }), + ).toBe("/tmp/explicit-clips-store"); + expect( + resolveScreenMemoryStoreDir({ + env: { AGENT_NATIVE_SCREEN_MEMORY_DIR: "/tmp/env-clips-store" }, + }), + ).toBe("/tmp/env-clips-store"); + }); + + it("discovers whichever Clips store was most recently active", () => { + const root = home(); + const appData = path.join(root, "Library", "Application Support"); + const stable = path.join(appData, "com.clips.tray", "screen-memory"); + const alpha = path.join(appData, "com.clips.tray.alpha", "screen-memory"); + fs.mkdirSync(stable, { recursive: true }); + fs.mkdirSync(alpha, { recursive: true }); + const now = new Date(); + fs.utimesSync( + stable, + new Date(now.getTime() - 60_000), + new Date(now.getTime() - 60_000), + ); + fs.utimesSync(alpha, now, now); + + expect( + resolveScreenMemoryStoreDir({ platform: "darwin", homeDir: root }), + ).toBe(alpha); + }); + + it("returns undefined when Rewind has not created a store", () => { + expect( + resolveScreenMemoryStoreDir({ platform: "darwin", homeDir: home() }), + ).toBeUndefined(); + }); +}); diff --git a/packages/core/src/cli/mcp.ts b/packages/core/src/cli/mcp.ts index 6070c061c8..e23ffd2db9 100644 --- a/packages/core/src/cli/mcp.ts +++ b/packages/core/src/cli/mcp.ts @@ -15,6 +15,7 @@ import crypto from "node:crypto"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { MCP_PUBLIC_ROUTE_PREFIX } from "../mcp/route-paths.js"; @@ -42,6 +43,7 @@ import { } from "./mcp-config-writers.js"; const SERVER_NAME_PREFIX = "agent-native"; +const SCREEN_MEMORY_SERVER_NAME = "clips-screen-memory"; interface ParsedArgs { _: string[]; @@ -83,6 +85,58 @@ function logOut(msg: string): void { process.stdout.write(`${msg}\n`); } +export interface ScreenMemoryStoreResolutionOptions { + explicitDir?: string; + env?: NodeJS.ProcessEnv; + platform?: NodeJS.Platform; + homeDir?: string; +} + +/** + * Resolve the active Clips store without asking people to find an app-data + * path. Environment overrides remain the unambiguous escape hatch; otherwise + * the most recently touched installed Clips/Clips Alpha store wins. + */ +export function resolveScreenMemoryStoreDir( + options: ScreenMemoryStoreResolutionOptions = {}, +): string | undefined { + if (options.explicitDir) return path.resolve(options.explicitDir); + const env = options.env ?? process.env; + const override = + env.CLIPS_SCREEN_MEMORY_DIR || env.AGENT_NATIVE_SCREEN_MEMORY_DIR; + if (override) return path.resolve(override); + + const platform = options.platform ?? process.platform; + const home = options.homeDir ?? os.homedir(); + const appDataRoot = + platform === "darwin" + ? path.join(home, "Library", "Application Support") + : platform === "win32" + ? env.APPDATA || path.join(home, "AppData", "Roaming") + : env.XDG_DATA_HOME || path.join(home, ".local", "share"); + return ["com.clips.tray", "com.clips.tray.alpha"] + .map((bundleId) => path.join(appDataRoot, bundleId, "screen-memory")) + .filter((candidate) => fs.existsSync(candidate)) + .map((candidate) => ({ + candidate, + modifiedAt: Math.max( + fs.statSync(candidate).mtimeMs, + ...["feature-config.json", "chapters.json"] + .map((name) => + name === "feature-config.json" + ? path.join(path.dirname(candidate), name) + : path.join(candidate, name), + ) + .filter((file) => fs.existsSync(file)) + .map((file) => fs.statSync(file).mtimeMs), + ), + })) + .sort( + (a, b) => + b.modifiedAt - a.modifiedAt || a.candidate.localeCompare(b.candidate), + )[0]?.candidate; +} + // --------------------------------------------------------------------------- // .env token provisioning (local dev) — hand-rolled idempotent upsert // --------------------------------------------------------------------------- @@ -343,6 +397,38 @@ function installForClient( return file; } +export function installScreenMemoryForClient( + client: ClientId, + storeDir: string, + cwd: string, + scope: string | undefined, +): string { + const file = configPathFor(client, cwd, scope); + const args = [ + "-y", + "@agent-native/core@latest", + "mcp", + "screen-memory", + "--dir", + path.resolve(storeDir), + ]; + if (client === "codex") { + writeCodexBlock( + file, + SCREEN_MEMORY_SERVER_NAME, + buildCodexLocalBlock(SCREEN_MEMORY_SERVER_NAME, args, {}, "npx"), + ); + } else { + writeJsonMcpEntryForClient( + client, + file, + SCREEN_MEMORY_SERVER_NAME, + buildLocalMcpEntryForClient(client, args, {}, "npx"), + ); + } + return file; +} + function uninstallForClient( client: ClientId, appId: string, @@ -454,6 +540,36 @@ async function cmdInstall(p: ParsedArgs): Promise { logOut(` Restart ${client} to pick up the new MCP server.`); } +function cmdInstallScreenMemory(p: ParsedArgs): void { + const client = normalizeClientId(p.client); + if (!client) { + logErr( + `Usage: npx @agent-native/core@latest mcp install-screen-memory --client ${SELECTABLE_CLIENTS.join("|")} [--dir ] [--scope user|project]`, + ); + process.exit(1); + } + const screenMemoryDir = resolveScreenMemoryStoreDir({ + explicitDir: p.screenMemoryDir, + }); + if (!screenMemoryDir) { + logErr( + "No local Clips Screen Memory store was found. Turn Rewind on in Clips, or pass --dir .", + ); + process.exit(1); + } + const file = installScreenMemoryForClient( + client, + screenMemoryDir, + process.cwd(), + p.scope, + ); + logOut(`Installed \"${SCREEN_MEMORY_SERVER_NAME}\" for ${client} → ${file}`); + logOut(" Store: current local Clips Rewind memory"); + logOut( + ` Restart ${client} to pick up the repaired Screen Memory MCP server.`, + ); +} + function cmdUninstall(p: ParsedArgs): void { const client = normalizeClientId(p.client); if (!client) { @@ -544,6 +660,11 @@ Usage: Run the local Clips Screen Memory stdio server. Defaults to the Clips app-data screen-memory folder. + npx @agent-native/core@latest mcp install-screen-memory --client [--dir ] [--scope user|project] + Install or repair the dedicated local Screen Memory MCP. The active Clips + or Clips Alpha store is discovered automatically; --dir is an override. + Clients: claude-code, codex, cowork, cursor, opencode, github-copilot + npx @agent-native/core@latest mcp install --client [--app ] [--scope user|project] Provision a token and write the client's MCP config (idempotent). Clients: claude-code, codex, cowork, cursor, opencode, github-copilot @@ -571,6 +692,9 @@ export async function runMcp(args: string[]): Promise { case "install": await cmdInstall(p); return; + case "install-screen-memory": + cmdInstallScreenMemory(p); + return; case "uninstall": cmdUninstall(p); return; diff --git a/packages/core/src/cli/skills-content/help.ts b/packages/core/src/cli/skills-content/help.ts index 4d6df0e2f6..50c7724f5a 100644 --- a/packages/core/src/cli/skills-content/help.ts +++ b/packages/core/src/cli/skills-content/help.ts @@ -2,13 +2,14 @@ export const HELP = `npx @agent-native/core@latest skills Usage: npx @agent-native/core@latest skills list - npx @agent-native/core@latest skills status [assets|content|design-exploration|visual-edit|visual-plan|visual-recap|visualize-repo|context-xray|scaffold] [--client codex|claude-code|pi|all] [--scope user|project] [--json] - npx @agent-native/core@latest skills update [assets|content|design-exploration|visual-edit|visual-plan|visual-recap|visualize-repo|context-xray|scaffold] [--client codex|claude-code|pi|all] [--scope user|project] [--dry-run] [--json] - npx @agent-native/core@latest skills add assets|content|design-exploration|visual-edit|visual-plan|visual-recap|visualize-repo|context-xray [--client codex|claude-code|cowork|cursor|opencode|github-copilot|all] [--scope user|project] [--mode hosted|local-files|self-hosted] [--mcp-url ] [--no-connect] [--with-github-action] [--yes] [--dry-run] [--json] + npx @agent-native/core@latest skills status [assets|content|rewind|design-exploration|visual-edit|visual-plan|visual-recap|visualize-repo|context-xray|scaffold] [--client codex|claude-code|pi|all] [--scope user|project] [--json] + npx @agent-native/core@latest skills update [assets|content|rewind|design-exploration|visual-edit|visual-plan|visual-recap|visualize-repo|context-xray|scaffold] [--client codex|claude-code|pi|all] [--scope user|project] [--dry-run] [--json] + npx @agent-native/core@latest skills add assets|content|rewind|design-exploration|visual-edit|visual-plan|visual-recap|visualize-repo|context-xray [--client codex|claude-code|cowork|cursor|opencode|github-copilot|all] [--scope user|project] [--mode hosted|local-files|self-hosted] [--mcp-url ] [--no-connect] [--with-github-action] [--yes] [--dry-run] [--json] npx @agent-native/core@latest skills add [--skill ] [--client ...] [--yes] Examples: npx @agent-native/core@latest skills add assets + npx @agent-native/core@latest skills add rewind --client codex --scope user --yes npx @agent-native/core@latest skills add content --mode local-files npx @agent-native/core@latest skills add design-exploration npx @agent-native/core@latest skills add visual-edit diff --git a/packages/core/src/cli/skills-content/index.ts b/packages/core/src/cli/skills-content/index.ts index ed199c779b..3547d6fa95 100644 --- a/packages/core/src/cli/skills-content/index.ts +++ b/packages/core/src/cli/skills-content/index.ts @@ -8,6 +8,7 @@ export * from "./help.js"; export * from "./assets-skill.js"; export * from "./content-skill.js"; +export * from "./rewind-skill.js"; export * from "./design-exploration-skill.js"; export * from "./design-visual-edit-skill.js"; export * from "./plan-setup-auth.js"; diff --git a/packages/core/src/cli/skills-content/rewind-skill.ts b/packages/core/src/cli/skills-content/rewind-skill.ts new file mode 100644 index 0000000000..3bde26f64c --- /dev/null +++ b/packages/core/src/cli/skills-content/rewind-skill.ts @@ -0,0 +1,52 @@ +export const REWIND_SKILL_MD = `--- +name: rewind +description: >- + Retrieve recent local Clips Rewind context when the user says "Look at + Rewind," asks what just happened, or refers to something they recently said + or saw. +metadata: + visibility: exported +--- + +# Rewind + +Use Clips Rewind as local screen memory. Start broad enough to find the right +moment, then read only the smallest relevant range. + +## Retrieval Flow + +1. Call \`screen_memory_status\` first. If the newest segment is still open, + wait for it to finalize rather than substituting an older segment. +2. Search \`screen_memory_search_chapters\` before requesting raw recent + context. Use the user's words, the visible app or project, and the likely + time range as clues. +3. If several chapters plausibly match, show the candidates and ask which one + the user means. Do not blend separate workstreams together. +4. Read the smallest useful range with \`screen_memory_recent_context\`. + Restate what you recovered before acting, and flag transcription or coverage + uncertainty. +5. Use \`screen_memory_frame_at\` for one exact visual moment or + \`screen_memory_contact_sheet\` to scan a bounded range. Prefer local frames + before escalating to cloud processing. +6. Request the smallest relevant timestamp range through Clips' bounded private + Clip handoff only when local text and frames are insufficient, such as + garbled speech, important motion, dense analysis, or a Clip the user wants + to keep and query later. + +## Boundaries + +- Rewind recordings, screenshots, audio, transcripts, OCR, and indexes remain + local unless the user explicitly asks for a bounded Clip handoff. +- Do not reveal archive filesystem paths, crawl Clips' app-data folders, or + bypass the Screen Memory MCP broker. +- Do not upload frames returned by local Screen Memory tools. +- Treat foreground apps and chapter labels as evidence, not proof of intent. +- If the Screen Memory MCP is missing, explain that the one-time setup needs to + be repaired with: + + \`npx -y @agent-native/core@latest skills add rewind --client --scope user --yes\` + + Replace \`\` with the current compatible host: \`codex\`, + \`claude-code\`, \`cursor\`, \`opencode\`, \`github-copilot\`, or \`cowork\`. + Ask the user to restart the host if it cannot reload MCP servers in place. +`; diff --git a/packages/core/src/cli/skills.spec.ts b/packages/core/src/cli/skills.spec.ts index be2f2c92a8..f5514a416d 100644 --- a/packages/core/src/cli/skills.spec.ts +++ b/packages/core/src/cli/skills.spec.ts @@ -130,6 +130,183 @@ describe("agent-native skills", () => { }); }); + it("installs Rewind instructions and repairs the local Screen Memory MCP together", async () => { + const root = tmpDir(); + const home = path.join(root, "home"); + const codexHome = path.join(root, "codex-home"); + const store = path.join(root, "screen-memory"); + fs.mkdirSync(home, { recursive: true }); + fs.mkdirSync(codexHome, { recursive: true }); + fs.mkdirSync(store, { recursive: true }); + const previousHome = process.env.HOME; + const previousCodexHome = process.env.CODEX_HOME; + const previousStore = process.env.CLIPS_SCREEN_MEMORY_DIR; + process.env.HOME = home; + process.env.CODEX_HOME = codexHome; + process.env.CLIPS_SCREEN_MEMORY_DIR = store; + + try { + const result = await addAgentNativeSkill( + parseSkillsArgs([ + "add", + "rewind", + "--client", + "codex", + "--scope", + "user", + "--yes", + ]), + { baseDir: root }, + ); + + expect(result).toMatchObject({ + id: "rewind", + skillNames: ["rewind"], + mcpUrl: "", + mcpClients: ["codex"], + }); + expect( + fs.readFileSync( + path.join(codexHome, "skills", "rewind", "SKILL.md"), + "utf-8", + ), + ).toContain("screen_memory_search_chapters"); + const config = fs.readFileSync( + path.join(codexHome, "config.toml"), + "utf-8", + ); + expect(config).toContain("clips-screen-memory"); + expect(config).toContain(path.resolve(store)); + expect(result.commands).toContain( + "npx @agent-native/core@latest mcp install-screen-memory --client codex --scope user", + ); + } finally { + if (previousHome === undefined) delete process.env.HOME; + else process.env.HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + if (previousStore === undefined) + delete process.env.CLIPS_SCREEN_MEMORY_DIR; + else process.env.CLIPS_SCREEN_MEMORY_DIR = previousStore; + } + }); + + it("can install only the Rewind instructions without touching MCP config", async () => { + const root = tmpDir(); + const codexHome = path.join(root, "codex-home"); + fs.mkdirSync(codexHome, { recursive: true }); + const previousCodexHome = process.env.CODEX_HOME; + process.env.CODEX_HOME = codexHome; + + try { + const result = await addAgentNativeSkill( + parseSkillsArgs([ + "add", + "rewind", + "--client", + "codex", + "--scope", + "user", + "--no-mcp", + "--yes", + ]), + { baseDir: root }, + ); + + expect(result.mcpClients).toEqual([]); + expect( + fs.existsSync(path.join(codexHome, "skills", "rewind", "SKILL.md")), + ).toBe(true); + expect(fs.existsSync(path.join(codexHome, "config.toml"))).toBe(false); + } finally { + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + } + }); + + it("installs Rewind into the shared user skill directory for Cursor", async () => { + const root = tmpDir(); + const home = path.join(root, "home"); + const store = path.join(root, "screen-memory"); + fs.mkdirSync(home, { recursive: true }); + fs.mkdirSync(store, { recursive: true }); + const previousHome = process.env.HOME; + const previousStore = process.env.CLIPS_SCREEN_MEMORY_DIR; + process.env.HOME = home; + process.env.CLIPS_SCREEN_MEMORY_DIR = store; + + try { + const result = await addAgentNativeSkill( + parseSkillsArgs([ + "add", + "rewind", + "--client", + "cursor", + "--scope", + "user", + "--yes", + ]), + { baseDir: root }, + ); + + expect(result.skillsAgents).toEqual(["cursor"]); + expect( + fs.existsSync( + path.join(home, ".agents", "skills", "rewind", "SKILL.md"), + ), + ).toBe(true); + expect( + fs.readFileSync(path.join(home, ".cursor", "mcp.json"), "utf-8"), + ).toContain("clips-screen-memory"); + } finally { + if (previousHome === undefined) delete process.env.HOME; + else process.env.HOME = previousHome; + if (previousStore === undefined) + delete process.env.CLIPS_SCREEN_MEMORY_DIR; + else process.env.CLIPS_SCREEN_MEMORY_DIR = previousStore; + } + }); + + it("dry-runs Rewind setup without requiring an active local store", async () => { + const result = await addAgentNativeSkill( + parseSkillsArgs([ + "add", + "rewind", + "--client", + "claude-code", + "--scope", + "user", + "--dry-run", + "--yes", + ]), + { baseDir: tmpDir() }, + ); + + expect(result).toMatchObject({ + id: "rewind", + dryRun: true, + mcpUrl: "", + mcpClients: ["claude-code"], + }); + }); + + it("rejects hosted MCP overrides for local Rewind memory", async () => { + await expect( + addAgentNativeSkill( + parseSkillsArgs([ + "add", + "rewind", + "--client", + "codex", + "--mcp-url", + "https://example.com/mcp", + "--yes", + ]), + { baseDir: tmpDir() }, + ), + ).rejects.toThrow("uses the local Clips Screen Memory MCP"); + }); + it("tracks when --client is explicit", () => { expect( parseSkillsArgs(["add", "assets", "--client", "claude-code"]), @@ -1505,6 +1682,7 @@ describe("agent-native skills", () => { "visualize-repo", "assets", "content", + "rewind", "design-exploration", "visual-edit", "context-xray", @@ -1591,6 +1769,7 @@ describe("agent-native skills", () => { "visualize-repo", "assets", "content", + "rewind", "design-exploration", "visual-edit", "context-xray", diff --git a/packages/core/src/cli/skills.ts b/packages/core/src/cli/skills.ts index 4edd006b83..70542ac177 100644 --- a/packages/core/src/cli/skills.ts +++ b/packages/core/src/cli/skills.ts @@ -33,6 +33,10 @@ import { installLocalContextXray, } from "./context-xray-local.js"; import { CLIENTS, type ClientId } from "./mcp-config-writers.js"; +import { + installScreenMemoryForClient, + resolveScreenMemoryStoreDir, +} from "./mcp.js"; import { PR_VISUAL_RECAP_SETUP, writePrVisualRecapWorkflow } from "./recap.js"; import { setupAgentSymlinks } from "./setup-agents.js"; import { @@ -46,6 +50,7 @@ import { EXEMPLAR_REFERENCE_MD, HELP, LOCAL_FILES_REFERENCE_MD, + REWIND_SKILL_MD, VISUAL_PLANS_SKILL_MD, VISUAL_RECAP_SKILL_MD, VISUALIZE_REPO_SKILL_MD, @@ -161,6 +166,40 @@ export const BUILT_IN_APP_SKILLS = { }), skillMarkdown: CONTENT_SKILL_MD, }, + rewind: { + skillName: "rewind", + screenMemoryMcp: true, + manifest: normalizeAppSkillManifest({ + schemaVersion: 1, + id: "rewind", + displayName: "Rewind", + description: + "Retrieve recent local Clips screen memory, chapters, transcripts, and exact frames through a privacy-preserving agent workflow.", + hosted: { + url: "https://clips.agent-native.com", + mcpUrl: "https://clips.agent-native.com/mcp", + }, + mcp: { serverName: "clips-screen-memory" }, + auth: { mode: "none" }, + surfaces: [ + { + id: "rewind-memory", + path: "/", + description: + "Search and inspect the user's local Clips Rewind memory through a bounded local MCP broker.", + }, + ], + skills: [ + { + path: "skills/rewind", + visibility: "exported", + exportAs: "rewind", + }, + ], + hostAdapters: ["plain-skill", "claude-skill", "generic-mcp"], + }), + skillMarkdown: REWIND_SKILL_MD, + }, design: { skillName: "design-exploration", extraSkills: { @@ -356,6 +395,7 @@ export const BUILT_IN_APP_SKILLS = { */ extraFiles?: Record>; localOnly?: boolean; + screenMemoryMcp?: boolean; } >; @@ -379,6 +419,10 @@ const BUILT_IN_APP_SKILL_ALIASES = { "local-content": "content", "content-local-files": "content", "agent-native-content": "content", + rewind: "rewind", + "screen-memory": "rewind", + "clips-rewind": "rewind", + "agent-native-rewind": "rewind", design: "design", "ui-design": "design", "ux-design": "design", @@ -421,6 +465,7 @@ const BUILT_IN_APP_SKILL_DISPLAY_ALIASES = { "content-local-files", "agent-native-content", ], + rewind: ["screen-memory", "clips-rewind", "agent-native-rewind"], design: [ "design-exploration", "visual-edit", @@ -811,6 +856,12 @@ function isLocalOnlyBuiltInSkill( return Boolean(entry && "localOnly" in entry && entry.localOnly); } +function isScreenMemoryMcpBuiltInSkill( + entry: (typeof BUILT_IN_APP_SKILLS)[BuiltInAppSkillId] | null | undefined, +): boolean { + return Boolean(entry && "screenMemoryMcp" in entry && entry.screenMemoryMcp); +} + function targetSupportsInstallMode( targetId: string | undefined, ): targetId is ModeAwareAppSkillId { @@ -1019,6 +1070,7 @@ function skillFilesForBuiltIn( skillName, mcpUrl: isLocalOnlyBuiltInSkill(entry) || + isScreenMemoryMcpBuiltInSkill(entry) || localFilesModeSkipsMcp(appSkillId, options.planMode) ? "" : (options.mcpUrl ?? entry.manifest.hosted.mcpUrl), @@ -1238,9 +1290,10 @@ function builtInSkillsRootForAgent( ): string { const home = homeDir() ?? baseDir; if (scope === "project") { - if (agent === "codex") return path.join(baseDir, ".agents", "skills"); - if (agent === "pi") return path.join(baseDir, ".agents", "skills"); - return path.join(baseDir, ".claude", "skills"); + if (agent === "claude-code" || agent === "claude-code-cli") { + return path.join(baseDir, ".claude", "skills"); + } + return path.join(baseDir, ".agents", "skills"); } if (agent === "codex") { return process.env.CODEX_HOME @@ -1250,6 +1303,13 @@ function builtInSkillsRootForAgent( if (agent === "pi") { return path.join(home, ".agents", "skills"); } + if ( + agent === "cursor" || + agent === "opencode" || + agent === "github-copilot" + ) { + return path.join(home, ".agents", "skills"); + } return path.join(home, ".claude", "skills"); } @@ -2128,6 +2188,11 @@ const BUILT_IN_SKILL_PROMPT_OPTIONS: SkillsTargetPromptContext["options"] = [ label: "content", hint: BUILT_IN_APP_SKILLS.content.manifest.description, }, + { + value: "rewind", + label: "rewind", + hint: BUILT_IN_APP_SKILLS.rewind.manifest.description, + }, { value: "design-exploration", label: "design-exploration", @@ -3309,7 +3374,13 @@ export async function addAgentNativeSkill( ); } const knownBuiltIn = knownTarget ? BUILT_IN_APP_SKILLS[knownTarget] : null; + const installsScreenMemoryMcp = isScreenMemoryMcpBuiltInSkill(knownBuiltIn); const baseDir = options.baseDir ?? process.cwd(); + if (installsScreenMemoryMcp && parsed.mcpUrl) { + throw new Error( + "Rewind uses the local Clips Screen Memory MCP and does not accept --mcp-url.", + ); + } if (isLocalOnlyBuiltInSkill(knownBuiltIn)) { if (parsed.planMode) { throw new Error( @@ -3415,7 +3486,9 @@ export async function addAgentNativeSkill( ); } installTarget = preserveMcpUrlAppPathOverride(installTarget, parsed.mcpUrl); - const skillsAgents = skillsAgentsForClients(clients); + const skillsAgents = installsScreenMemoryMcp + ? clients + : skillsAgentsForClients(clients); if (parsed.dryRun) { try { const localManifestPath = shouldWriteContentLocalFilesManifest( @@ -3443,9 +3516,11 @@ export async function addAgentNativeSkill( displayName: installTarget.displayName, skillNames: installTarget.skillNames, skillsAgents, - mcpUrl: localFilesModeSkipsMcp(modeAwareTargetId, planMode) + mcpUrl: installsScreenMemoryMcp ? "" - : installTarget.loaded.manifest.hosted.mcpUrl, + : localFilesModeSkipsMcp(modeAwareTargetId, planMode) + ? "" + : installTarget.loaded.manifest.hosted.mcpUrl, mcpClients: shouldRegisterMcp ? mcpClients : [], dryRun: true, commands: [ @@ -3545,7 +3620,29 @@ export async function addAgentNativeSkill( dryRun: Boolean(parsed.dryRun), }); - if (shouldRegisterMcp) { + if (shouldRegisterMcp && installsScreenMemoryMcp) { + const screenMemoryDir = resolveScreenMemoryStoreDir(); + if (!screenMemoryDir) { + throw new Error( + "No local Clips Screen Memory store was found. Turn Rewind on in Clips, then run the setup again.", + ); + } + registeredMcpClients = mcpClients.map((client) => { + commands.push( + `npx @agent-native/core@latest mcp install-screen-memory --client ${client} --scope ${parsed.scope}`, + ); + installScreenMemoryForClient( + client, + screenMemoryDir, + baseDir, + parsed.scope, + ); + return client; + }); + options.telemetry?.track("skills_cli mcp registered", { + skills: installTarget.skillNames.join(","), + }); + } else if (shouldRegisterMcp) { commands.push( `npx @agent-native/core@latest app-skill ensure --manifest ${installTarget.loaded.file} --client ${parsed.client} --scope ${parsed.scope} --yes`, ); @@ -3655,9 +3752,11 @@ export async function addAgentNativeSkill( instructionSource, skillNames: installTarget.skillNames, skillsAgents, - mcpUrl: localFilesModeSkipsMcp(modeAwareTargetId, planMode) + mcpUrl: installsScreenMemoryMcp ? "" - : installTarget.loaded.manifest.hosted.mcpUrl, + : localFilesModeSkipsMcp(modeAwareTargetId, planMode) + ? "" + : installTarget.loaded.manifest.hosted.mcpUrl, mcpClients: registeredMcpClients, dryRun: parsed.dryRun, commands, @@ -3689,10 +3788,13 @@ function listSkills(options: RunSkillsOptions = {}) { ] ?? [], name: entry.manifest.displayName, description: entry.manifest.description, - mcpUrl: isLocalOnlyBuiltInSkill(entry) - ? "" - : entry.manifest.hosted.mcpUrl, - local: isLocalOnlyBuiltInSkill(entry), + mcpUrl: + isLocalOnlyBuiltInSkill(entry) || isScreenMemoryMcpBuiltInSkill(entry) + ? "" + : entry.manifest.hosted.mcpUrl, + local: + isLocalOnlyBuiltInSkill(entry) || + isScreenMemoryMcpBuiltInSkill(entry), source: "agent-native", })), ...publicSkillEntries(options).map((entry) => ({ diff --git a/packages/core/src/mcp-client/index.ts b/packages/core/src/mcp-client/index.ts index fe7e0e56de..24e5f657e5 100644 --- a/packages/core/src/mcp-client/index.ts +++ b/packages/core/src/mcp-client/index.ts @@ -113,12 +113,21 @@ export { } from "./tool-policy.js"; export { configureScreenMemory, + queryScreenMemoryForAgent, queryScreenMemoryContext, readScreenMemoryStatus, type ScreenMemoryConfig, + type ScreenMemoryAgentQueryResult, type ScreenMemoryContextItem, + type ScreenMemoryCoverageGap, + type ScreenMemoryEvidenceItem, + type ScreenMemoryEvidenceSourceType, type ScreenMemoryQueryResult, + type ScreenMemoryRetrievalCoverage, + type ScreenMemorySegmentReference, type ScreenMemoryStatus, + type ScreenMemoryTimeRange, + type ScreenMemoryTruncation, } from "./screen-memory-local.js"; export { MCP_ACTION_RESULT_MARKER, diff --git a/packages/core/src/mcp-client/screen-memory-local.spec.ts b/packages/core/src/mcp-client/screen-memory-local.spec.ts index e68c3e2a15..e8e4932b19 100644 --- a/packages/core/src/mcp-client/screen-memory-local.spec.ts +++ b/packages/core/src/mcp-client/screen-memory-local.spec.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -6,6 +6,7 @@ import { describe, expect, it } from "vitest"; import { configureScreenMemory, + queryScreenMemoryForAgent, queryScreenMemoryContext, readScreenMemoryStatus, } from "./screen-memory-local.js"; @@ -48,6 +49,22 @@ describe("local Screen Memory helpers", () => { expect(status.enabled).toBe(false); expect(status.state).toBe("disabled"); expect(status.captureCount).toBe(0); + expect(status.config).toMatchObject({ + retentionHours: 8, + captureMode: "visuals", + reviewBeforeSending: true, + agentClipRetention: "forever", + excludePrivateWindows: false, + }); + expect(status.config.excludedBundleIds).toContain( + "com.1password.1password", + ); + + const result = await queryScreenMemoryContext({}, options); + expect(result.evidence).toEqual([]); + expect(result.coverage.gaps).toEqual([ + expect.objectContaining({ reason: "no-context-files" }), + ]); }); it("updates local config and queries bounded context records", async () => { @@ -63,7 +80,18 @@ describe("local Screen Memory helpers", () => { "utf8", ); - const status = await configureScreenMemory({ enabled: true }, options); + const status = await configureScreenMemory( + { + enabled: true, + retentionHours: 72, + captureMode: "visuals-audio", + reviewBeforeSending: false, + agentClipRetention: "7-days", + excludedBundleIds: [" COM.Example.Secret ", "com.example.secret"], + excludePrivateWindows: false, + }, + options, + ); const result = await queryScreenMemoryContext( { query: "enabled", limit: 5 }, options, @@ -71,12 +99,167 @@ describe("local Screen Memory helpers", () => { expect(status.enabled).toBe(true); expect(status.state).toBe("ready"); + expect(status.config).toMatchObject({ + retentionHours: 24, + captureMode: "visuals-audio", + reviewBeforeSending: false, + agentClipRetention: "7-days", + excludedBundleIds: ["com.example.secret"], + excludePrivateWindows: false, + }); expect(result.count).toBe(1); expect(result.items[0]).toMatchObject({ appName: "Clips", windowTitle: "Settings", text: "Screen Memory is enabled", }); + expect(result.evidence[0]).toMatchObject({ + sourceType: "app-context", + excerpt: "Screen Memory is enabled", + jumpTarget: { kind: "screen-memory-moment" }, + }); + }); + + it("applies configured privacy exclusions before searching local rows", async () => { + const { root, options } = await tempScreenMemoryEnv(); + await writeFile( + join(root, "context.jsonl"), + [ + { + capturedAt: "2026-06-29T12:00:00.000Z", + bundleId: "com.example.secret", + windowTitle: "Secret notes", + text: "excluded bundle words", + }, + { + capturedAt: "2026-06-29T12:01:00.000Z", + bundleId: "com.example.browser", + windowTitle: "Private Browsing", + text: "private window words", + }, + { + capturedAt: "2026-06-29T12:01:30.000Z", + bundleId: "com.example.browser", + windowTitle: "Browser", + isPrivate: true, + text: "explicit private words", + }, + { + capturedAt: "2026-06-29T12:02:00.000Z", + bundleId: "com.example.clips", + windowTitle: "Clips", + text: "visible local words", + }, + ] + .map((row) => JSON.stringify(row)) + .join("\n") + "\n", + "utf8", + ); + await configureScreenMemory( + { + enabled: true, + excludedBundleIds: ["com.example.secret"], + excludePrivateWindows: true, + }, + options, + ); + + const result = await queryScreenMemoryContext({}, options); + + expect(result.items).toHaveLength(1); + expect(result.evidence).toEqual([ + expect.objectContaining({ excerpt: "visible local words" }), + ]); + expect(JSON.stringify(result)).not.toContain("excluded bundle words"); + expect(JSON.stringify(result)).not.toContain("private window words"); + expect(JSON.stringify(result)).not.toContain("explicit private words"); + expect(result.coverage.gaps).toContainEqual({ + startedAt: "2026-06-29T12:00:00.000Z", + endedAt: "2026-06-29T12:01:30.000Z", + reason: "privacy-excluded-or-unretained", + }); + }); + + it("redacts paths and credentials and logs only a content-free receipt", async () => { + const { root, options } = await tempScreenMemoryEnv(); + await writeFile( + join(root, "egress.jsonl"), + `${JSON.stringify({ + requestId: "legacy-request", + occurredAt: "2026-06-29T11:59:00.000Z", + state: "prepared", + packet: { + question: "legacy private question", + evidence: [ + { + id: "legacy-evidence", + momentId: "legacy-moment", + sourceType: "transcript", + capturedAt: "2026-06-29T11:58:00.000Z", + excerpt: "legacy private excerpt", + }, + ], + }, + evidenceCount: 1, + packetBytes: 100, + error: null, + })}\n`, + ); + await writeFile( + join(root, "context.jsonl"), + `${JSON.stringify({ + capturedAt: "2026-06-29T12:00:00.000Z", + text: "api_key=super-secret-value", + })}\n`, + "utf8", + ); + const result = await queryScreenMemoryForAgent( + { query: "api_key=super-secret-value", limit: 5 }, + options, + ); + + expect(result.contextFiles).toEqual([]); + expect(result.items[0]?.sourceFile).toBe("local-screen-memory"); + expect(result.evidence[0]?.excerpt).toBe("api_key=[REDACTED]"); + expect(result.egress.packet.question).toBe("api_key=[REDACTED]"); + const log = await readFile(join(root, "egress.jsonl"), "utf8"); + const events = log + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(events.map((event) => event.state)).toEqual([ + "prepared", + "prepared", + "completed", + ]); + expect(events[1]).toMatchObject({ + operation: "agent-query", + receipt: { + evidence: [ + expect.objectContaining({ + id: result.evidence[0]?.id, + capturedAt: "2026-06-29T12:00:00.000Z", + }), + ], + }, + }); + expect(events[1].receipt).not.toHaveProperty("excerpt"); + expect(events[1].receipt).not.toHaveProperty("question"); + expect(log).not.toContain("super-secret"); + expect(log).not.toContain("legacy private question"); + expect(log).not.toContain("legacy private excerpt"); + expect(events[0]).toMatchObject({ + requestId: "legacy-request", + receipt: { + evidence: [ + expect.objectContaining({ + id: "legacy-evidence", + capturedAt: "2026-06-29T11:58:00.000Z", + }), + ], + }, + }); + expect(log).not.toContain(options.env.AGENT_NATIVE_SCREEN_MEMORY_DIR); }); it("reads linux screen memory data from the app-data directory", async () => { @@ -117,4 +300,318 @@ describe("local Screen Memory helpers", () => { text: "Screen Memory is visible on Linux", }); }); + + it("reports stale coverage and bounds excerpts and item output", async () => { + const { root, options } = await tempScreenMemoryEnv(); + const now = new Date("2026-06-29T12:10:00.000Z"); + await writeFile( + join(root, "context.jsonl"), + [ + { + capturedAt: "2026-06-29T12:00:00.000Z", + text: "x".repeat(1_300), + }, + { + capturedAt: "2026-06-29T11:59:00.000Z", + text: "second evidence", + }, + { + capturedAt: "2026-06-29T11:58:00.000Z", + text: "third evidence", + }, + ] + .map((row) => JSON.stringify(row)) + .join("\n") + "\n", + "utf8", + ); + + const result = await queryScreenMemoryContext( + { sinceMinutes: 15, limit: 2 }, + { ...options, now: () => now }, + ); + + expect(result.items).toHaveLength(2); // Legacy context remains bounded too. + expect(result.evidence).toHaveLength(2); + expect(result.evidence[0]?.excerpt).toHaveLength(1_200); + expect(result.evidence[0]?.excerptTruncated).toBe(true); + expect(result.truncation).toMatchObject({ + itemLimit: 2, + returnedItems: 2, + omittedItems: 1, + excerptsTruncated: 1, + }); + expect(result.coverage.gaps).toEqual( + expect.arrayContaining([ + expect.objectContaining({ reason: "missing-before-first-evidence" }), + expect.objectContaining({ reason: "capture-stale" }), + ]), + ); + }); + + it("normalizes local OCR and transcript fields without inventing either", async () => { + const { root, options } = await tempScreenMemoryEnv(); + await writeFile( + join(root, "context.jsonl"), + `${JSON.stringify({ + capturedAt: "2026-06-29T12:00:00.000Z", + appName: "Clips", + text: "legacy app context", + ocrText: "Visible settings label", + transcript: "Spoken local words", + segmentId: "segment-1", + })}\n${JSON.stringify({ + capturedAt: "2026-06-29T12:01:00.000Z", + text: "only legacy context", + })}\n`, + "utf8", + ); + await writeFile(join(root, "segment-1.mp4"), "retained-test-media", "utf8"); + await writeFile( + join(root, "segment-1.json"), + `${JSON.stringify({ + id: "segment-1", + startedAt: "2026-06-29T11:59:00.000Z", + endedAt: "2026-06-29T12:02:00.000Z", + path: join(root, "segment-1.mp4"), + bytes: 123, + })}\n`, + "utf8", + ); + + const result = await queryScreenMemoryContext({ limit: 10 }, options); + + expect(result.evidence.map((item) => item.sourceType)).toEqual([ + "app-context", + "transcript", + "ocr", + ]); + expect(result.evidence[1]?.segmentRefs).toEqual([ + { + id: "segment-1", + startedAt: "2026-06-29T11:59:00.000Z", + endedAt: "2026-06-29T12:02:00.000Z", + }, + ]); + expect(JSON.stringify(result.evidence)).not.toContain("segment-1.mp4"); + expect(JSON.stringify(result.evidence)).not.toContain('"bytes"'); + }); + + it("reads OCR sidecars and exposes incomplete local indexing as coverage gaps", async () => { + const { root, options } = await tempScreenMemoryEnv(); + await writeFile( + join(root, "segment-1.ocr.jsonl"), + `${JSON.stringify({ + schemaVersion: 1, + segmentId: "segment-1", + capturedAt: "2026-06-29T12:00:00.000Z", + offsetMs: 1000, + source: "ocr", + ocrText: "Local visible text", + confidence: 0.9, + frameWidth: 1920, + frameHeight: 1080, + })}\n`, + "utf8", + ); + await writeFile( + join(root, "segment-1.json"), + `${JSON.stringify({ id: "segment-1", startedAt: "2026-06-29T11:59:00.000Z", endedAt: "2026-06-29T12:02:00.000Z" })}\n`, + "utf8", + ); + await writeFile( + join(root, "segment-1.ocr-status.json"), + `${JSON.stringify({ state: "failed" })}\n`, + "utf8", + ); + + const result = await queryScreenMemoryContext( + { query: "visible" }, + options, + ); + expect(result.evidence).toEqual([ + expect.objectContaining({ + sourceType: "ocr", + excerpt: "Local visible text", + }), + ]); + expect(result.coverage.gaps).toContainEqual( + expect.objectContaining({ + reason: "index-failed", + startedAt: "2026-06-29T11:59:00.000Z", + }), + ); + }); + + it("reads local transcript sidecars as transcript evidence", async () => { + const { root, options } = await tempScreenMemoryEnv(); + await writeFile( + join(root, "segment-2.transcript.jsonl"), + `${JSON.stringify({ + schemaVersion: 1, + segmentId: "segment-2", + capturedAt: "2026-06-29T12:03:02.000Z", + source: "mixed-audio", + startMs: 2000, + endMs: 3200, + text: "A locally transcribed sentence", + })}\n`, + "utf8", + ); + await writeFile( + join(root, "segment-2.json"), + `${JSON.stringify({ id: "segment-2", startedAt: "2026-06-29T12:03:00.000Z", endedAt: "2026-06-29T12:04:00.000Z" })}\n`, + "utf8", + ); + await writeFile( + join(root, "segment-2.transcript-status.json"), + `${JSON.stringify({ state: "transcribing" })}\n`, + "utf8", + ); + + const result = await queryScreenMemoryContext( + { query: "transcribed" }, + options, + ); + expect(result.evidence).toEqual([ + expect.objectContaining({ + sourceType: "transcript", + excerpt: "A locally transcribed sentence", + segmentRefs: [expect.objectContaining({ id: "segment-2" })], + }), + ]); + expect(result.coverage.gaps).toContainEqual( + expect.objectContaining({ + reason: "index-pending", + startedAt: "2026-06-29T12:03:00.000Z", + }), + ); + }); + + it("never returns evidence backed only by tainted or pruned segments", async () => { + const { root, options } = await tempScreenMemoryEnv(); + await writeFile( + join(root, "events.jsonl"), + [ + { + segmentId: "segment-tainted", + capturedAt: "2026-06-29T12:00:10.000Z", + text: "private excluded window", + }, + { + segmentId: "segment-pruned", + capturedAt: "2026-06-29T12:01:10.000Z", + text: "already pruned media", + }, + ] + .map((row) => JSON.stringify(row)) + .join("\n") + "\n", + "utf8", + ); + await writeFile( + join(root, "segment-tainted.mp4"), + "retained-test-media", + "utf8", + ); + await writeFile( + join(root, "segment-tainted.json"), + `${JSON.stringify({ + id: "segment-tainted", + startedAt: "2026-06-29T12:00:00.000Z", + endedAt: "2026-06-29T12:01:00.000Z", + path: join(root, "segment-tainted.mp4"), + exclusionTainted: true, + })}\n`, + "utf8", + ); + await writeFile( + join(root, "segment-pruned.json"), + `${JSON.stringify({ + id: "segment-pruned", + startedAt: "2026-06-29T12:01:00.000Z", + endedAt: "2026-06-29T12:02:00.000Z", + path: join(root, "segment-pruned.mp4"), + })}\n`, + "utf8", + ); + + const result = await queryScreenMemoryContext({ limit: 10 }, options); + + expect(result.items).toEqual([]); + expect(result.evidence).toEqual([]); + expect(result.coverage.gaps).toContainEqual( + expect.objectContaining({ + reason: "privacy-excluded-or-unretained", + }), + ); + }); + + it("joins continuous transcript rows without crossing pauses or audio sources", async () => { + const { root, options } = await tempScreenMemoryEnv(); + const transcriptRows = [ + { + schemaVersion: 1, + segmentId: "segment-3", + capturedAt: "2026-06-29T12:00:00.000Z", + source: "microphone", + startMs: 0, + endMs: 1_000, + text: "Clips and screen captures", + }, + { + schemaVersion: 1, + segmentId: "segment-3", + capturedAt: "2026-06-29T12:00:01.100Z", + source: "microphone", + startMs: 1_100, + endMs: 2_000, + text: "obey the visibility toggle.", + }, + { + schemaVersion: 1, + segmentId: "segment-3", + capturedAt: "2026-06-29T12:00:02.100Z", + source: "system-audio", + startMs: 2_100, + endMs: 3_000, + text: "A different audio source stays separate.", + }, + { + schemaVersion: 1, + segmentId: "segment-3", + capturedAt: "2026-06-29T12:00:05.100Z", + source: "microphone", + startMs: 5_100, + endMs: 6_000, + text: "After a real pause stays separate.", + }, + ]; + await writeFile( + join(root, "segment-3.transcript.jsonl"), + `${transcriptRows.map((row) => JSON.stringify(row)).join("\n")}\n`, + "utf8", + ); + await writeFile( + join(root, "segment-3.json"), + `${JSON.stringify({ id: "segment-3", startedAt: "2026-06-29T12:00:00.000Z", endedAt: "2026-06-29T12:01:00.000Z" })}\n`, + "utf8", + ); + + const result = await queryScreenMemoryContext({ limit: 10 }, options); + expect(result.evidence.map((item) => item.excerpt)).toEqual([ + "After a real pause stays separate.", + "A different audio source stays separate.", + "Clips and screen captures obey the visibility toggle.", + ]); + + const crossRowSearch = await queryScreenMemoryContext( + { query: "captures obey", limit: 10 }, + options, + ); + expect(crossRowSearch.evidence).toEqual([ + expect.objectContaining({ + sourceType: "transcript", + excerpt: "Clips and screen captures obey the visibility toggle.", + }), + ]); + }); }); diff --git a/packages/core/src/mcp-client/screen-memory-local.ts b/packages/core/src/mcp-client/screen-memory-local.ts index fb169cf70f..1ba041588e 100644 --- a/packages/core/src/mcp-client/screen-memory-local.ts +++ b/packages/core/src/mcp-client/screen-memory-local.ts @@ -5,6 +5,11 @@ export interface ScreenMemoryConfig { maxBytes: number; segmentSeconds: number; sampleIntervalSeconds: number; + captureMode: "visuals" | "visuals-audio"; + reviewBeforeSending: boolean; + agentClipRetention: "forever" | "24-hours" | "7-days" | "30-days"; + excludedBundleIds: string[]; + excludePrivateWindows: boolean; } export interface ScreenMemoryStatus { @@ -37,6 +42,74 @@ export interface ScreenMemoryContextItem { sourceFile: string; } +export type ScreenMemoryEvidenceSourceType = + | "app-context" + | "transcript" + | "ocr"; + +export interface ScreenMemoryTimeRange { + startedAt: string | null; + endedAt: string; +} + +export interface ScreenMemoryCoverageGap { + startedAt: string | null; + endedAt: string | null; + reason: + | "no-context-files" + | "no-evidence-in-requested-range" + | "missing-before-first-evidence" + | "capture-stale" + | "timestamps-unavailable" + | "index-pending" + | "index-failed" + | "index-skipped" + | "privacy-excluded-or-unretained"; +} + +export interface ScreenMemorySegmentReference { + id: string; + startedAt: string | null; + endedAt: string | null; +} + +export interface ScreenMemoryEvidenceItem { + id: string; + momentId: string; + capturedAt: string | null; + sourceType: ScreenMemoryEvidenceSourceType; + excerpt: string; + excerptTruncated: boolean; + appName: string | null; + windowTitle: string | null; + bundleId: string | null; + url: string | null; + title: string | null; + segmentRefs: ScreenMemorySegmentReference[]; + jumpTarget: { + kind: "screen-memory-moment"; + momentId: string; + capturedAt: string | null; + segmentId: string | null; + }; +} + +export interface ScreenMemoryRetrievalCoverage { + requestedRange: ScreenMemoryTimeRange; + coveredRange: { startedAt: string | null; endedAt: string | null }; + gaps: ScreenMemoryCoverageGap[]; +} + +export interface ScreenMemoryTruncation { + itemLimit: number; + returnedItems: number; + omittedItems: number; + maxExcerptChars: number; + excerptsTruncated: number; + sourceRowsReadLimit: number; + sourceRowsReadTruncated: boolean; +} + export interface ScreenMemoryQueryResult { feature: "screen-memory"; localOnly: true; @@ -46,6 +119,10 @@ export interface ScreenMemoryQueryResult { sinceMinutes: number | null; count: number; items: ScreenMemoryContextItem[]; + /** Stable, bounded local retrieval contract. `items` remains for legacy callers. */ + evidence: ScreenMemoryEvidenceItem[]; + coverage: ScreenMemoryRetrievalCoverage; + truncation: ScreenMemoryTruncation; contextFiles: string[]; note: string; } @@ -54,15 +131,45 @@ export interface ScreenMemoryLocalOptions { env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; homeDir?: string; + /** Test-only clock injection; retrieval never reads network time. */ + now?: () => Date; +} + +export interface ScreenMemoryAgentQueryResult extends ScreenMemoryQueryResult { + egress: { + requestId: string; + packet: { + question: string; + evidence: Array<{ + id: string; + momentId: string; + sourceType: ScreenMemoryEvidenceSourceType; + capturedAt: string | null; + excerpt: string; + }>; + }; + note: string; + }; } const DEFAULT_CONFIG: ScreenMemoryConfig = { enabled: false, paused: false, - retentionHours: 24, + retentionHours: 8, maxBytes: 20 * 1024 * 1024 * 1024, segmentSeconds: 5 * 60, sampleIntervalSeconds: 10, + captureMode: "visuals", + reviewBeforeSending: true, + agentClipRetention: "forever", + excludedBundleIds: [ + "com.1password.1password", + "com.agilebits.onepassword7", + "com.bitwarden.desktop", + "com.dashlane.dashlane", + "com.lastpass.lastpass", + ], + excludePrivateWindows: false, }; const JSONL_NAMES = [ @@ -71,6 +178,15 @@ const JSONL_NAMES = [ "snapshots.jsonl", "screen-memory.jsonl", ]; +const MAX_SOURCE_ROWS = 10_000; +const MAX_EXCERPT_CHARS = 1_200; +const MAX_TRANSCRIPT_JOIN_GAP_MS = 2_000; +type ScreenMemoryOcrIndexState = + | "pending" + | "indexing" + | "ready" + | "failed" + | "skipped"; function asRecord(value: unknown): Record | null { return value && typeof value === "object" && !Array.isArray(value) @@ -87,14 +203,36 @@ function booleanValue(value: unknown, fallback: boolean): boolean { return typeof value === "boolean" ? value : fallback; } +function enumValue( + value: unknown, + allowed: readonly T[], + fallback: T, +): T { + return typeof value === "string" && allowed.includes(value as T) + ? (value as T) + : fallback; +} + +function bundleIds(value: unknown, fallback: string[]): string[] { + if (!Array.isArray(value)) return [...fallback]; + return [ + ...new Set( + value + .filter((item): item is string => typeof item === "string") + .map((item) => item.trim().toLowerCase()) + .filter(Boolean), + ), + ].sort(); +} + function normalizeConfig(value: unknown): ScreenMemoryConfig { const raw = asRecord(value) ?? {}; return { enabled: booleanValue(raw.enabled, DEFAULT_CONFIG.enabled), paused: booleanValue(raw.paused, DEFAULT_CONFIG.paused), - retentionHours: positiveNumber( - raw.retentionHours, - DEFAULT_CONFIG.retentionHours, + retentionHours: Math.min( + positiveNumber(raw.retentionHours, DEFAULT_CONFIG.retentionHours), + 24, ), maxBytes: positiveNumber(raw.maxBytes, DEFAULT_CONFIG.maxBytes), segmentSeconds: positiveNumber( @@ -105,6 +243,28 @@ function normalizeConfig(value: unknown): ScreenMemoryConfig { raw.sampleIntervalSeconds, DEFAULT_CONFIG.sampleIntervalSeconds, ), + captureMode: enumValue( + raw.captureMode, + ["visuals", "visuals-audio"], + DEFAULT_CONFIG.captureMode, + ), + reviewBeforeSending: + typeof raw.reviewBeforeSending === "boolean" + ? raw.reviewBeforeSending + : DEFAULT_CONFIG.reviewBeforeSending, + agentClipRetention: enumValue( + raw.agentClipRetention, + ["forever", "24-hours", "7-days", "30-days"], + DEFAULT_CONFIG.agentClipRetention, + ), + excludedBundleIds: bundleIds( + raw.excludedBundleIds, + DEFAULT_CONFIG.excludedBundleIds, + ), + excludePrivateWindows: + typeof raw.excludePrivateWindows === "boolean" + ? raw.excludePrivateWindows + : DEFAULT_CONFIG.excludePrivateWindows, }; } @@ -287,6 +447,22 @@ async function contextFilesFor( // ignore missing candidate files } } + try { + const names = await fs.readdir(dir); + for (const name of names.filter( + (entry) => + entry.endsWith(".ocr.jsonl") || entry.endsWith(".transcript.jsonl"), + )) { + const candidate = path.join(dir, name); + const stat = await fs.stat(candidate); + if (stat.isFile()) { + files.push(candidate); + storageBytes += stat.size; + } + } + } catch { + // ignore unavailable local directories + } } return { files, storageBytes }; @@ -303,6 +479,28 @@ function firstString( return null; } +function firstFiniteNumber( + raw: Record, + keys: string[], +): number | null { + for (const key of keys) { + const value = raw[key]; + if (typeof value === "number" && Number.isFinite(value)) return value; + } + return null; +} + +function firstBoolean( + raw: Record, + keys: string[], +): boolean | null { + for (const key of keys) { + const value = raw[key]; + if (typeof value === "boolean") return value; + } + return null; +} + function normalizeContextItem( value: unknown, sourceFile: string, @@ -342,9 +540,307 @@ function itemTime(item: ScreenMemoryContextItem): number { return Number.isFinite(parsed) ? parsed : 0; } -async function readItems(files: string[]): Promise { +interface LocalContextRow { + item: ScreenMemoryContextItem; + raw: Record; +} + +function isPrivacyExcludedRow( + row: LocalContextRow, + config: ScreenMemoryConfig, +): boolean { + const bundleId = row.item.bundleId?.trim().toLowerCase(); + if ( + bundleId && + config.excludedBundleIds.some( + (excluded) => excluded.trim().toLowerCase() === bundleId, + ) + ) { + return true; + } + + if (!config.excludePrivateWindows) return false; + if ( + firstBoolean(row.raw, [ + "isPrivate", + "is_private", + "privateWindow", + "private_window", + "incognito", + ]) === true + ) { + return true; + } + const title = row.item.windowTitle?.toLowerCase() ?? ""; + return ["incognito", "inprivate", "private browsing", "private window"].some( + (marker) => title.includes(marker), + ); +} + +function rowTimeRange( + rows: LocalContextRow[], + fallback: ScreenMemoryTimeRange, +): ScreenMemoryTimeRange { + const times = rows + .map(({ item }) => itemTime(item)) + .filter((time) => time > 0); + if (times.length === 0) return fallback; + return { + startedAt: new Date(Math.min(...times)).toISOString(), + endedAt: new Date(Math.max(...times)).toISOString(), + }; +} + +interface LocalTranscriptSpan { + row: LocalContextRow; + segmentId: string; + source: string; + startMs: number; + endMs: number; +} + +interface LocalSegment { + id: string; + startedAt: string | null; + endedAt: string | null; + clean: boolean; +} + +function stableId(prefix: string, value: string): string { + let hash = 0x811c9dc5; + for (let index = 0; index < value.length; index += 1) { + hash ^= value.charCodeAt(index); + hash = Math.imul(hash, 0x01000193); + } + return `${prefix}-${(hash >>> 0).toString(36)}`; +} + +function boundedExcerpt(value: string): { + excerpt: string; + excerptTruncated: boolean; +} { + const normalized = value.replace(/\s+/g, " ").trim(); + if (normalized.length <= MAX_EXCERPT_CHARS) { + return { excerpt: normalized, excerptTruncated: false }; + } + return { + excerpt: `${normalized.slice(0, MAX_EXCERPT_CHARS - 1)}…`, + excerptTruncated: true, + }; +} + +function redactCredentialText(value: string): string { + return value + .replace( + /\b(?:sk-[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9]{12,}|AKIA[A-Z0-9]{16})\b/g, + "[REDACTED CREDENTIAL]", + ) + .replace(/\bBearer\s+[A-Za-z0-9._~+\/-]{8,}/gi, "Bearer [REDACTED]") + .replace( + /\b(api[_-]?key|access[_-]?token|password|secret)\s*[:=]\s*([^\s,;]{4,})/gi, + "$1=[REDACTED]", + ); +} + +function sourceTexts( + raw: Record, + fallback: string, +): Array<{ sourceType: ScreenMemoryEvidenceSourceType; text: string }> { + const localTranscriptText = + firstString(raw, ["text"]) && + firstString(raw, ["segmentId", "segment_id"]) && + (typeof raw.startMs === "number" || typeof raw.start_ms === "number") + ? firstString(raw, ["text"]) + : null; + const values: Array<{ + sourceType: ScreenMemoryEvidenceSourceType; + keys: string[]; + }> = [ + { sourceType: "transcript", keys: ["transcript", "transcriptText"] }, + { sourceType: "ocr", keys: ["ocrText", "visibleText", "ocr"] }, + ]; + const evidence = values.flatMap(({ sourceType, keys }) => { + const text = firstString(raw, keys); + return text ? [{ sourceType, text }] : []; + }); + if (localTranscriptText) { + evidence.unshift({ + sourceType: "transcript", + text: localTranscriptText, + }); + } + return evidence.length > 0 + ? evidence + : fallback + ? [{ sourceType: "app-context", text: fallback }] + : []; +} + +function segmentReferences( + raw: Record, + capturedAt: string | null, + segments: LocalSegment[], +): ScreenMemorySegmentReference[] { + const directId = firstString(raw, ["segmentId", "segment_id"]); + const matched = directId + ? segments.filter((segment) => segment.id === directId) + : capturedAt + ? segments.filter((segment) => { + const moment = Date.parse(capturedAt); + const started = segment.startedAt + ? Date.parse(segment.startedAt) + : NaN; + const ended = segment.endedAt ? Date.parse(segment.endedAt) : NaN; + return moment >= started && moment <= ended; + }) + : []; + return matched.slice(0, 3).map((segment) => ({ + id: segment.id, + startedAt: segment.startedAt, + endedAt: segment.endedAt, + })); +} + +function normalizeEvidence( + row: LocalContextRow, + segments: LocalSegment[], +): ScreenMemoryEvidenceItem[] { + const refs = segmentReferences(row.raw, row.item.capturedAt, segments); + const base = [ + row.item.sourceFile, + row.item.capturedAt ?? "unknown-time", + row.item.appName ?? "", + row.item.windowTitle ?? "", + ].join("|"); + const momentId = + firstString(row.raw, ["momentId", "moment_id", "eventId", "captureId"]) ?? + stableId("moment", base); + const appContext = + row.item.text || + [row.item.appName, row.item.windowTitle, row.item.title, row.item.url] + .filter((value): value is string => Boolean(value)) + .join(" — "); + return sourceTexts(row.raw, appContext).map(({ sourceType, text }) => { + const { excerpt, excerptTruncated } = boundedExcerpt(text); + const id = + firstString(row.raw, ["evidenceId", "evidence_id"]) ?? + stableId("evidence", `${momentId}|${sourceType}|${text}`); + return { + id, + momentId, + capturedAt: row.item.capturedAt, + sourceType, + excerpt, + excerptTruncated, + appName: row.item.appName, + windowTitle: row.item.windowTitle, + bundleId: row.item.bundleId, + url: row.item.url, + title: row.item.title, + segmentRefs: refs, + jumpTarget: { + kind: "screen-memory-moment", + momentId, + capturedAt: row.item.capturedAt, + segmentId: refs[0]?.id ?? null, + }, + }; + }); +} + +function transcriptSpan(row: LocalContextRow): LocalTranscriptSpan | null { + const segmentId = firstString(row.raw, ["segmentId", "segment_id"]); + const source = firstString(row.raw, ["source", "kind"]); + const startMs = firstFiniteNumber(row.raw, ["startMs", "start_ms"]); + const endMs = firstFiniteNumber(row.raw, ["endMs", "end_ms"]); + if ( + !segmentId || + !source || + startMs === null || + endMs === null || + endMs <= startMs || + !firstString(row.raw, ["text"]) + ) { + return null; + } + return { row, segmentId, source, startMs, endMs }; +} + +/** + * Whisper emits phrase-sized rows, which can split an ordinary sentence in + * the middle. Join only rows that are demonstrably continuous in the same + * finalized segment and audio source. Stored sidecars remain untouched, and a + * real pause, source switch, segment boundary, or excerpt bound starts a new + * evidence item. + */ +function coalesceTranscriptRows(rows: LocalContextRow[]): LocalContextRow[] { + const passthrough: LocalContextRow[] = []; + const groups = new Map(); + + for (const row of rows) { + const span = transcriptSpan(row); + if (!span) { + passthrough.push(row); + continue; + } + const key = [row.item.sourceFile, span.segmentId, span.source].join("\0"); + const group = groups.get(key) ?? []; + group.push(span); + groups.set(key, group); + } + + const joined = [...groups.values()].flatMap((group) => { + const ordered = group.sort((a, b) => a.startMs - b.startMs); + const output: LocalContextRow[] = []; + let current: LocalTranscriptSpan | null = null; + + for (const next of ordered) { + const currentText = current + ? (firstString(current.row.raw, ["text"]) ?? "") + : ""; + const nextText = firstString(next.row.raw, ["text"]) ?? ""; + const gapMs = current ? next.startMs - current.endMs : Infinity; + const combinedText = [currentText, nextText].filter(Boolean).join(" "); + const canJoin = + current !== null && + gapMs >= 0 && + gapMs <= MAX_TRANSCRIPT_JOIN_GAP_MS && + combinedText.length <= MAX_EXCERPT_CHARS; + + if (!current || !canJoin) { + if (current) output.push(current.row); + current = { + ...next, + row: { + item: { ...next.row.item }, + raw: { ...next.row.raw }, + }, + }; + continue; + } + + current.endMs = next.endMs; + current.row.item.text = combinedText; + current.row.raw.text = combinedText; + if ("end_ms" in current.row.raw) current.row.raw.end_ms = next.endMs; + else current.row.raw.endMs = next.endMs; + } + if (current) output.push(current.row); + return output; + }); + + return [...passthrough, ...joined].sort( + (a, b) => itemTime(b.item) - itemTime(a.item), + ); +} + +async function readRows(files: string[]): Promise<{ + rows: LocalContextRow[]; + sourceRowsReadTruncated: boolean; +}> { const { fs } = await nodeModules(); - const items: ScreenMemoryContextItem[] = []; + const rows: LocalContextRow[] = []; + let sourceRowsReadTruncated = false; for (const file of files) { let text = ""; @@ -353,18 +849,106 @@ async function readItems(files: string[]): Promise { } catch { continue; } - const lines = text.split(/\r?\n/).filter(Boolean).slice(-10_000); + const allLines = text.split(/\r?\n/).filter(Boolean); + if (allLines.length > MAX_SOURCE_ROWS) sourceRowsReadTruncated = true; + const lines = allLines.slice(-MAX_SOURCE_ROWS); for (const line of lines) { try { - const item = normalizeContextItem(JSON.parse(line), file); - if (item) items.push(item); + const raw = asRecord(JSON.parse(line)); + const item = raw ? normalizeContextItem(raw, file) : null; + if (item && raw) rows.push({ item, raw }); } catch { // Keep one malformed row from hiding the rest of the local context. } } } - return items.sort((a, b) => itemTime(b) - itemTime(a)); + return { + rows: coalesceTranscriptRows(rows), + sourceRowsReadTruncated, + }; +} + +async function readItems(files: string[]): Promise { + const { rows } = await readRows(files); + return rows.map(({ item }) => item); +} + +async function readSegments(dataDirs: string[]): Promise { + const { fs, path } = await nodeModules(); + const segments: LocalSegment[] = []; + for (const dir of dataDirs) { + let names: string[] = []; + try { + names = await fs.readdir(dir); + } catch { + continue; + } + for (const name of names.filter((candidate) => + candidate.endsWith(".json"), + )) { + const raw = asRecord(await readJson(path.join(dir, name))); + const id = raw && firstString(raw, ["id"]); + if (!raw || !id) continue; + const mediaPath = firstString(raw, ["path"]); + let retained = true; + if (mediaPath) { + try { + await fs.access(mediaPath); + } catch { + retained = false; + } + } + segments.push({ + id, + startedAt: firstString(raw, ["startedAt"]), + endedAt: firstString(raw, ["endedAt"]), + clean: + retained && + raw.exclusionTainted !== true && + raw.corrupt !== true && + !firstString(raw, ["error"]), + }); + } + } + return segments; +} + +async function readOcrIndexStates( + dataDirs: string[], +): Promise> { + const { fs, path } = await nodeModules(); + const states: Array<{ segmentId: string; state: ScreenMemoryOcrIndexState }> = + []; + for (const dir of dataDirs) { + let names: string[] = []; + try { + names = await fs.readdir(dir); + } catch { + continue; + } + for (const name of names.filter( + (entry) => + entry.endsWith(".ocr-status.json") || + entry.endsWith(".transcript-status.json"), + )) { + const raw = asRecord(await readJson(path.join(dir, name))); + const rawState = String(raw?.state); + const state = rawState === "transcribing" ? "indexing" : rawState; + const suffix = name.endsWith(".ocr-status.json") + ? ".ocr-status.json" + : ".transcript-status.json"; + const segmentId = name.slice(0, -suffix.length); + if ( + ["pending", "indexing", "ready", "failed", "skipped"].includes( + String(state), + ) + ) { + states.push({ segmentId, state: state as ScreenMemoryOcrIndexState }); + } + } + } + return states; } export async function queryScreenMemoryContext( @@ -384,18 +968,130 @@ export async function queryScreenMemoryContext( typeof args.sinceMinutes === "number" && Number.isFinite(args.sinceMinutes) ? Math.max(args.sinceMinutes, 0) : null; + const now = options.now?.() ?? new Date(); const cutoff = - sinceMinutes === null ? null : Date.now() - sinceMinutes * 60 * 1000; + sinceMinutes === null ? null : now.getTime() - sinceMinutes * 60 * 1000; const needle = query?.toLowerCase() ?? null; - const items = (await readItems(files)).filter((item) => { + const source = await readRows(files); + const rangeRows = source.rows.filter((row) => { + const { item } = row; if (cutoff !== null) { const time = itemTime(item); if (!time || time < cutoff) return false; } + return true; + }); + const privacyExcludedRows = rangeRows.filter((row) => + isPrivacyExcludedRow(row, info.config), + ); + const candidateRows = rangeRows.filter((row) => { + if (isPrivacyExcludedRow(row, info.config)) return false; if (!needle) return true; - return JSON.stringify(item).toLowerCase().includes(needle); + const { item } = row; + return JSON.stringify({ item, raw: row.raw }) + .toLowerCase() + .includes(needle); }); + const segments = await readSegments(paths.dataDirs); + const cleanSegments = segments.filter((segment) => segment.clean); + // Modern stores bind every evidence row to retained segment metadata. Once + // segment metadata exists, refuse rows that only point at tainted, corrupt, + // or pruned media. Legacy context-only stores (no segment metadata at all) + // remain readable for backwards compatibility. + const rows = + segments.length === 0 + ? candidateRows + : candidateRows.filter( + (row) => + segmentReferences(row.raw, row.item.capturedAt, cleanSegments) + .length > 0, + ); + const items = rows.map(({ item }) => item); + const ocrIndexStates = await readOcrIndexStates(paths.dataDirs); + const evidence = rows.flatMap((row) => normalizeEvidence(row, cleanSegments)); + const returnedEvidence = evidence.slice(0, limit); + const evidenceTimes = evidence + .map((item) => (item.capturedAt ? Date.parse(item.capturedAt) : NaN)) + .filter(Number.isFinite); + const coveredRange = evidenceTimes.length + ? { + startedAt: new Date(Math.min(...evidenceTimes)).toISOString(), + endedAt: new Date(Math.max(...evidenceTimes)).toISOString(), + } + : { startedAt: null, endedAt: null }; + const requestedRange = { + startedAt: cutoff === null ? null : new Date(cutoff).toISOString(), + endedAt: now.toISOString(), + }; + const gaps: ScreenMemoryCoverageGap[] = []; + if (privacyExcludedRows.length > 0) { + gaps.push({ + ...rowTimeRange(privacyExcludedRows, requestedRange), + reason: "privacy-excluded-or-unretained", + }); + } + if (candidateRows.length > rows.length) { + gaps.push({ + ...requestedRange, + reason: "privacy-excluded-or-unretained", + }); + } + for (const index of ocrIndexStates) { + if (index.state === "ready") continue; + const segment = segments.find( + (candidate) => candidate.id === index.segmentId, + ); + const reason = + index.state === "failed" + ? "index-failed" + : index.state === "skipped" + ? "index-skipped" + : "index-pending"; + gaps.push({ + startedAt: segment?.startedAt ?? null, + endedAt: segment?.endedAt ?? null, + reason, + }); + } + if (files.length === 0) { + gaps.push({ ...requestedRange, reason: "no-context-files" }); + } else if (evidence.length === 0) { + gaps.push({ ...requestedRange, reason: "no-evidence-in-requested-range" }); + } else { + if (evidence.some((item) => !item.capturedAt)) { + gaps.push({ + startedAt: null, + endedAt: null, + reason: "timestamps-unavailable", + }); + } + if ( + requestedRange.startedAt && + coveredRange.startedAt && + coveredRange.startedAt > requestedRange.startedAt + ) { + gaps.push({ + startedAt: requestedRange.startedAt, + endedAt: coveredRange.startedAt, + reason: "missing-before-first-evidence", + }); + } + const staleAfterMs = Math.max( + info.config.sampleIntervalSeconds * 3 * 1000, + 2 * 60 * 1000, + ); + if ( + coveredRange.endedAt && + Date.parse(coveredRange.endedAt) < now.getTime() - staleAfterMs + ) { + gaps.push({ + startedAt: coveredRange.endedAt, + endedAt: requestedRange.endedAt, + reason: "capture-stale", + }); + } + } return { feature: "screen-memory", @@ -406,6 +1102,19 @@ export async function queryScreenMemoryContext( sinceMinutes, count: items.length, items: items.slice(0, limit), + evidence: returnedEvidence, + coverage: { requestedRange, coveredRange, gaps }, + truncation: { + itemLimit: limit, + returnedItems: returnedEvidence.length, + omittedItems: Math.max(evidence.length - returnedEvidence.length, 0), + maxExcerptChars: MAX_EXCERPT_CHARS, + excerptsTruncated: returnedEvidence.filter( + (item) => item.excerptTruncated, + ).length, + sourceRowsReadLimit: MAX_SOURCE_ROWS, + sourceRowsReadTruncated: source.sourceRowsReadTruncated, + }, contextFiles: files, note: files.length === 0 @@ -414,6 +1123,131 @@ export async function queryScreenMemoryContext( }; } +/** + * Agent-facing retrieval boundary. Asking an agent to search Rewind is the + * authorization. This removes filesystem paths, redacts obvious + * credential-shaped text, and records a content-free activity receipt before + * it is returned to an action caller. + */ +export async function queryScreenMemoryForAgent( + args: { + query?: string | null; + limit?: number | null; + sinceMinutes?: number | null; + } = {}, + options: ScreenMemoryLocalOptions = {}, +): Promise { + const result = await queryScreenMemoryContext(args, options); + const items = result.items.map((item) => ({ + ...item, + text: redactCredentialText(item.text), + sourceFile: "local-screen-memory", + })); + const evidence = result.evidence.map((item) => ({ + ...item, + excerpt: redactCredentialText(item.excerpt), + })); + const packet = { + question: redactCredentialText( + args.query?.trim().slice(0, 4_000) || "Recent Screen Memory context", + ), + evidence: evidence.slice(0, 20).map((item) => ({ + id: item.id, + momentId: item.momentId, + sourceType: item.sourceType, + capturedAt: item.capturedAt, + excerpt: item.excerpt.slice(0, MAX_EXCERPT_CHARS), + })), + }; + const paths = await resolvePaths(options); + const { fs, path } = await nodeModules(); + const storeDir = paths.dataDirs[0]; + await fs.mkdir(storeDir, { recursive: true, mode: 0o700 }); + await fs.chmod(storeDir, 0o700); + const logPath = path.join(storeDir, "egress.jsonl"); + if (await exists(logPath)) { + const sanitized = (await fs.readFile(logPath, "utf8")) + .split("\n") + .filter(Boolean) + .flatMap((line) => { + try { + const event = JSON.parse(line) as Record; + const packet = event.packet as + | { evidence?: Array> } + | null + | undefined; + if (!event.receipt && Array.isArray(packet?.evidence)) { + event.receipt = { + evidence: packet.evidence.map((item) => ({ + id: item.id, + momentId: item.momentId, + sourceType: item.sourceType, + capturedAt: item.capturedAt ?? null, + })), + }; + } + delete event.packet; + delete event.reason; + event.packetBytes = + typeof event.packetBytes === "number" ? event.packetBytes : 0; + return [JSON.stringify(event)]; + } catch { + return []; + } + }); + const temporaryPath = `${logPath}.sanitize-${process.pid}`; + await fs.writeFile( + temporaryPath, + sanitized.length > 0 ? `${sanitized.join("\n")}\n` : "", + { encoding: "utf8", mode: 0o600 }, + ); + await fs.rename(temporaryPath, logPath); + } + const requestId = `egress-action-${Date.now()}-${process.pid}`; + const prepared = { + requestId, + occurredAt: new Date().toISOString(), + state: "prepared", + operation: "agent-query", + receipt: { + evidence: packet.evidence.map( + ({ id, momentId, sourceType, capturedAt }) => ({ + id, + momentId, + sourceType, + capturedAt, + }), + ), + }, + evidenceCount: packet.evidence.length, + packetBytes: 0, + error: null, + }; + const completed = { + ...prepared, + occurredAt: new Date().toISOString(), + state: "completed", + receipt: null, + }; + await fs.appendFile( + logPath, + `${JSON.stringify(prepared)}\n${JSON.stringify(completed)}\n`, + { encoding: "utf8", mode: 0o600 }, + ); + await fs.chmod(logPath, 0o600); + return { + ...result, + items, + evidence, + contextFiles: [], + egress: { + requestId, + packet, + note: "A content-free local activity receipt was recorded before this bounded text packet was returned.", + }, + }; +} + export async function readScreenMemoryStatus( options: ScreenMemoryLocalOptions = {}, ): Promise { diff --git a/packages/core/src/mcp/screen-memory-stdio.spec.ts b/packages/core/src/mcp/screen-memory-stdio.spec.ts new file mode 100644 index 0000000000..d3f4df6e89 --- /dev/null +++ b/packages/core/src/mcp/screen-memory-stdio.spec.ts @@ -0,0 +1,568 @@ +import { mkdtemp, readFile, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + appendLocalEvidenceReceipt, + contactSheetRangeIsValid, + projectScreenMemoryChapterCandidate, + redactCredentialText, + readScreenMemoryChapters, + readScreenMemoryFrame, + searchScreenMemoryChapters, + screenMemoryMcpToolDefinitions, + selectContactSheetTimestamps, +} from "./screen-memory-stdio.js"; + +describe("Screen Memory stdio MCP tools", () => { + it("keeps the legacy tool names while documenting the bounded evidence contract", () => { + const tools = screenMemoryMcpToolDefinitions(); + + expect(tools.map((tool) => tool.name)).toEqual([ + "screen_memory_status", + "screen_memory_recent_context", + "screen_memory_recent_segments", + "screen_memory_search_chapters", + "screen_memory_frame_at", + "screen_memory_contact_sheet", + "screen_memory_request_clip", + "screen_memory_handoff_status", + ]); + expect( + tools.find((tool) => tool.name === "screen_memory_recent_context") + ?.description, + ).toContain("coverage"); + expect( + tools.find((tool) => tool.name === "screen_memory_recent_context") + ?.description, + ).toContain("no media bytes or images"); + expect( + tools.find((tool) => tool.name === "screen_memory_recent_segments") + ?.description, + ).toContain("private Clip handoff boundary"); + expect( + tools.find((tool) => tool.name === "screen_memory_request_clip") + ?.description, + ).toContain("private Clip"); + }); + + it("redacts obvious credential-shaped text before an MCP packet can leave", () => { + const fakeBearer = ["abcdef", "ghijklmnop"].join(""); + expect( + redactCredentialText( + `api_key=super-secret-value Bearer ${fakeBearer} sk-abcdefghijklmnop`, + ), + ).toBe("api_key=[REDACTED] Bearer [REDACTED] [REDACTED CREDENTIAL]"); + }); + + it("parses a complete native chapters manifest and ranks semantic matches deterministically", async () => { + const store = await mkdtemp(join(tmpdir(), "screen-memory-chapters-")); + await writeFile( + join(store, "chapters.json"), + JSON.stringify({ + schemaVersion: 1, + generatedAt: "2026-07-18T12:00:00.000Z", + state: "ready", + coverage: { gaps: [] }, + chapters: [ + { + id: "clips", + startedAt: "2026-07-18T10:00:00.000Z", + endedAt: "2026-07-18T10:05:00.000Z", + durationMs: 300000, + label: "Reviewing Clips Rewind settings", + summary: "MCP frame tools and retention", + confidence: 0.8, + segmentRefs: [], + evidenceRefs: [{ sourceType: "ocr" }], + contexts: [{ appName: "Clips" }], + representativeMoments: [], + ambiguityReasons: [], + indexState: "ready", + }, + { + id: "content", + startedAt: "2026-07-18T10:05:00.000Z", + endedAt: "2026-07-18T10:10:00.000Z", + durationMs: 300000, + label: "Testing Content page", + summary: "Database cards", + confidence: 0.9, + segmentRefs: [], + evidenceRefs: [{ sourceType: "ocr" }], + contexts: [{ appName: "Content" }], + representativeMoments: [], + ambiguityReasons: [], + indexState: "partial", + }, + ], + }), + ); + const chapters = readScreenMemoryChapters(store); + expect(chapters?.chapters).toHaveLength(2); + expect(chapters?.schemaVersion).toBe(1); + expect(chapters?.chapters[0]).toMatchObject({ revision: 1, aliases: [] }); + expect( + searchScreenMemoryChapters(chapters!, "rewind frame", 5, "Content").map( + (chapter) => chapter.id, + ), + ).toEqual(["clips"]); + expect( + searchScreenMemoryChapters(chapters!, "content", 5, "Clips")[0], + ).toMatchObject({ + id: "content", + matchReasons: expect.arrayContaining([ + 'matched "content" in chapter or accessibility evidence', + ]), + }); + }); + + it("parses r5 chapter revisions, exact gaps, aliases, and representative coverage", async () => { + const store = await mkdtemp(join(tmpdir(), "screen-memory-r5-")); + await writeFile( + join(store, "chapters.json"), + JSON.stringify({ + schemaVersion: 2, + generatedAt: "2026-07-18T12:00:00.000Z", + state: "ready", + coverage: { + gaps: [ + { + startedAt: "2026-07-18T10:02:00.000Z", + endedAt: "2026-07-18T10:03:00.000Z", + reason: "excluded", + }, + ], + }, + chapters: [ + { + id: "verification", + revision: 3, + aliases: ["old-verification"], + startedAt: "2026-07-18T10:00:00.000Z", + endedAt: "2026-07-18T10:05:00.000Z", + label: "Account verification", + summary: "Browser and Mail verification task", + accessibilitySummary: "Verify account code in browser", + accessibilityKeywords: ["verification code"], + keywords: [], + confidence: 0.9, + segmentRefs: [], + sceneRefs: ["scene-a"], + evidenceRefs: [ + { + sourceType: "accessibility", + sourceKind: "focused-control", + keywords: [ + "verification code", + ...Array.from( + { length: 12 }, + (_, index) => `filler-${index}`, + ), + "global data model", + ], + confidence: 0.95, + }, + ], + contexts: [], + representativeMoments: [ + { + momentId: "middle", + capturedAt: "2026-07-18T10:02:30.000Z", + segmentId: "segment-a", + offsetMs: 150000, + reason: "meaningful internal change", + }, + ], + representativeCoverage: { + coveredScenes: 1, + totalScenes: 2, + truncated: true, + }, + ambiguityReasons: [], + indexState: "ready", + }, + ], + }), + ); + const document = readScreenMemoryChapters(store); + expect(document).toMatchObject({ + schemaVersion: 2, + coverage: { gaps: [{ reason: "excluded" }] }, + }); + expect(document?.chapters[0]).toMatchObject({ + revision: 3, + aliases: ["old-verification"], + sceneRefs: [{ id: "scene-a" }], + representativeCoverage: { + coveredScenes: 1, + totalScenes: 2, + truncated: true, + }, + }); + expect(document?.chapters[0].evidenceRefs[0].keywords).toContain( + "global data model", + ); + }); + + it("ranks exact visible evidence above a generic app match and exposes provenance", async () => { + const document = { + schemaVersion: 2 as const, + generatedAt: "2026-07-18T12:00:00.000Z", + state: "ready", + coverage: { gaps: [] }, + chapters: [ + { + id: "generic", + revision: 1, + aliases: [], + startedAt: "2026-07-18T10:00:00.000Z", + endedAt: "2026-07-18T10:05:00.000Z", + durationMs: 300000, + label: "Cursor", + summary: "Cursor app", + keywords: [], + accessibilitySummary: "", + accessibilityKeywords: [], + confidence: 1, + segmentRefs: [], + sceneRefs: [], + evidenceRefs: [], + contexts: [{ appName: "OAuth redirect mismatch - Cursor" }], + representativeMoments: [], + ambiguityReasons: [], + indexState: "ready" as const, + }, + { + id: "semantic", + revision: 2, + aliases: ["previous-semantic"], + startedAt: "2026-07-18T10:05:00.000Z", + endedAt: "2026-07-18T10:10:00.000Z", + durationMs: 300000, + label: "Reviewing authentication error", + summary: "", + keywords: [], + accessibilitySummary: + "OAuth redirect mismatch on account verification", + accessibilityKeywords: ["OAuth redirect mismatch"], + confidence: 0.7, + segmentRefs: [], + sceneRefs: [{ id: "scene-b" }], + evidenceRefs: [ + { + sourceType: "accessibility", + sourceKind: "document", + keywords: ["OAuth redirect mismatch"], + confidence: 0.9, + }, + ], + contexts: [{ appName: "Cursor" }], + representativeMoments: [], + ambiguityReasons: [], + indexState: "ready" as const, + }, + ], + }; + const result = searchScreenMemoryChapters( + document, + "OAuth redirect mismatch", + ); + expect(result.map((chapter) => chapter.id)).toEqual([ + "semantic", + "generic", + ]); + expect(result[0].matchReasons.join(" ")).toContain("exact phrase"); + expect(result[0].matchProvenance).toMatchObject([ + { sourceType: "accessibility", sourceKind: "document" }, + ]); + const candidate = projectScreenMemoryChapterCandidate(result[0]); + expect(candidate).toMatchObject({ + id: "semantic", + sceneCount: 1, + evidenceSources: ["accessibility/document"], + }); + expect(candidate).not.toHaveProperty("segmentRefs"); + expect(candidate).not.toHaveProperty("sceneRefs"); + expect(candidate).not.toHaveProperty("evidenceRefs"); + expect(JSON.stringify(candidate).length).toBeLessThan(4_000); + }); + + it("does not let uncorroborated microphone narration dominate visible work", () => { + const document = { + schemaVersion: 2 as const, + generatedAt: "2026-07-18T12:00:00.000Z", + state: "ready", + coverage: { gaps: [] }, + chapters: [ + { + id: "microphone", + revision: 1, + aliases: [], + startedAt: "2026-07-18T10:00:00.000Z", + endedAt: "2026-07-18T10:05:00.000Z", + durationMs: 300000, + label: "Reading", + summary: "", + keywords: [], + accessibilitySummary: "", + accessibilityKeywords: [], + confidence: 1, + segmentRefs: [], + sceneRefs: [], + evidenceRefs: [ + { + sourceType: "microphone", + sourceKind: "transcript", + keywords: ["astronomy comet"], + confidence: 1, + }, + ], + contexts: [], + representativeMoments: [], + ambiguityReasons: [], + indexState: "ready" as const, + }, + { + id: "visible", + revision: 1, + aliases: [], + startedAt: "2026-07-18T10:05:00.000Z", + endedAt: "2026-07-18T10:10:00.000Z", + durationMs: 300000, + label: "Astronomy research", + summary: "Comet observation notes", + keywords: ["astronomy comet"], + accessibilitySummary: "Comet observation notebook", + accessibilityKeywords: [], + confidence: 0.7, + segmentRefs: [], + sceneRefs: [], + evidenceRefs: [ + { + sourceType: "ocr", + keywords: ["astronomy comet"], + confidence: 0.8, + }, + ], + contexts: [], + representativeMoments: [], + ambiguityReasons: [], + indexState: "ready" as const, + }, + ], + }; + const result = searchScreenMemoryChapters(document, "astronomy comet"); + expect(result.map((chapter) => chapter.id)).toEqual(["visible"]); + }); + + it("ignores conversational stop words instead of creating generic matches", () => { + const document = { + schemaVersion: 2 as const, + generatedAt: "2026-07-18T12:00:00.000Z", + state: "ready", + coverage: { gaps: [] }, + chapters: [ + { + id: "generic", + revision: 1, + aliases: [], + startedAt: "2026-07-18T10:00:00.000Z", + endedAt: "2026-07-18T10:05:00.000Z", + durationMs: 300000, + label: "Working in an app", + summary: "the work that was open", + keywords: [], + accessibilitySummary: "", + accessibilityKeywords: [], + confidence: 0.5, + segmentRefs: [], + sceneRefs: [], + evidenceRefs: [], + contexts: [], + representativeMoments: [], + ambiguityReasons: [], + indexState: "ready" as const, + }, + ], + }; + expect(searchScreenMemoryChapters(document, "the work that was")).toEqual( + [], + ); + }); + + it("rejects missing or corrupt chapter manifests rather than leaking partial state", async () => { + const store = await mkdtemp(join(tmpdir(), "screen-memory-chapters-")); + expect(readScreenMemoryChapters(store)).toBeNull(); + await writeFile( + join(store, "chapters.json"), + JSON.stringify({ + schemaVersion: 1, + generatedAt: "bad", + state: "ready", + chapters: [], + }), + ); + expect(readScreenMemoryChapters(store)).toBeNull(); + }); + + it("selects the clean exact retained segment and returns a mocked local image without its path", async () => { + const store = await mkdtemp(join(tmpdir(), "screen-memory-frame-")); + const media = join(store, "private-segment.mp4"); + await writeFile(media, "not really a movie"); + await writeFile( + join(store, "segment-a.json"), + JSON.stringify({ + id: "segment-a", + startedAt: "2020-01-01T00:00:00.000Z", + endedAt: "2020-01-01T00:01:00.000Z", + path: media, + mimeType: "video/mp4", + bytes: 1, + durationMs: 60000, + }), + ); + const frame = readScreenMemoryFrame( + store, + "2020-01-01T00:00:20.000Z", + (_path, offsetMs) => { + expect(offsetMs).toBe(20_000); + return Buffer.from("fake-jpeg"); + }, + ); + expect(frame).toMatchObject({ + timestamp: "2020-01-01T00:00:20.000Z", + segmentId: "segment-a", + image: { type: "image", mimeType: "image/jpeg" }, + }); + expect(JSON.stringify(frame)).not.toContain(media); + }); + + it("rejects corrupt or exclusion-tainted segments and prefers representative contact-sheet timestamps", async () => { + const store = await mkdtemp(join(tmpdir(), "screen-memory-frame-")); + const media = join(store, "private-segment.mp4"); + await writeFile(media, "not really a movie"); + await writeFile( + join(store, "segment-a.json"), + JSON.stringify({ + id: "segment-a", + startedAt: "2020-01-01T00:00:00.000Z", + endedAt: "2020-01-01T00:01:00.000Z", + path: media, + mimeType: "video/mp4", + bytes: 1, + durationMs: 60000, + exclusionTainted: true, + }), + ); + expect(() => + readScreenMemoryFrame(store, "2020-01-01T00:00:20.000Z", () => + Buffer.from("jpeg"), + ), + ).toThrow("excluded"); + expect( + selectContactSheetTimestamps( + "2020-01-01T00:00:00.000Z", + "2020-01-01T00:01:00.000Z", + 3, + ["2020-01-01T00:00:05.000Z"], + )[0], + ).toBe("2020-01-01T00:00:05.000Z"); + expect( + selectContactSheetTimestamps( + "2020-01-01T00:00:00.000Z", + "2020-01-01T00:01:00.000Z", + 3, + [ + "2020-01-01T00:00:05.000Z", + "2020-01-01T00:00:25.000Z", + "2020-01-01T00:00:55.000Z", + ], + ), + ).toContain("2020-01-01T00:00:50.000Z"); + expect(contactSheetRangeIsValid(15 * 60_000, true)).toBe(true); + expect(contactSheetRangeIsValid(15 * 60_000, false)).toBe(false); + }); + + it("rejects retained-segment paths that escape the configured store", async () => { + const store = await mkdtemp(join(tmpdir(), "screen-memory-frame-store-")); + const outside = await mkdtemp( + join(tmpdir(), "screen-memory-frame-outside-"), + ); + const outsideMedia = join(outside, "private-segment.mp4"); + const escapedMedia = join(store, "escaped.mp4"); + await writeFile(outsideMedia, "not really a movie"); + await symlink(outsideMedia, escapedMedia); + await writeFile( + join(store, "segment-escaped.json"), + JSON.stringify({ + id: "segment-escaped", + startedAt: "2020-01-01T00:00:00.000Z", + endedAt: "2020-01-01T00:01:00.000Z", + path: escapedMedia, + mimeType: "video/mp4", + bytes: 1, + durationMs: 60000, + }), + ); + expect(() => + readScreenMemoryFrame(store, "2020-01-01T00:00:20.000Z", () => + Buffer.from("jpeg"), + ), + ).toThrow("outside the local memory store"); + }); + + it("keeps frame activity receipts content-free while retaining bounded provenance", async () => { + const store = await mkdtemp(join(tmpdir(), "screen-memory-chapters-")); + appendLocalEvidenceReceipt(store, "frame-at", [ + { + timestamp: "2026-07-19T12:00:00.000Z", + segmentId: "segment-opaque-123", + }, + ]); + const receipt = await readFile(join(store, "egress.jsonl"), "utf8"); + expect(JSON.parse(receipt)).toMatchObject({ + operation: "frame-at", + receipt: { + frames: [ + { + timestamp: "2026-07-19T12:00:00.000Z", + segmentId: "segment-opaque-123", + }, + ], + }, + }); + expect(receipt).not.toMatch(/\.mp4|mediaPath|reason|"packet"/i); + }); + + it("records every contact-sheet frame timestamp and opaque segment reference", async () => { + const store = await mkdtemp(join(tmpdir(), "screen-memory-chapters-")); + appendLocalEvidenceReceipt(store, "contact-sheet", [ + { + timestamp: "2026-07-19T12:00:00.000Z", + segmentId: "segment-opaque-123", + }, + { + timestamp: "2026-07-19T12:01:00.000Z", + segmentId: "segment-opaque-456", + }, + ]); + + const event = JSON.parse( + await readFile(join(store, "egress.jsonl"), "utf8"), + ); + expect(event.receipt.frames).toEqual([ + { + timestamp: "2026-07-19T12:00:00.000Z", + segmentId: "segment-opaque-123", + }, + { + timestamp: "2026-07-19T12:01:00.000Z", + segmentId: "segment-opaque-456", + }, + ]); + expect(JSON.stringify(event)).not.toMatch(/\.mp4|mediaPath|"packet"/i); + }); +}); diff --git a/packages/core/src/mcp/screen-memory-stdio.ts b/packages/core/src/mcp/screen-memory-stdio.ts index 039170cbd7..e929ed830a 100644 --- a/packages/core/src/mcp/screen-memory-stdio.ts +++ b/packages/core/src/mcp/screen-memory-stdio.ts @@ -1,14 +1,10 @@ +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -interface ScreenMemoryEvent { - capturedAt: string; - appName?: string | null; - windowTitle?: string | null; - bundleId?: string | null; - source: string; -} +import { queryScreenMemoryContext } from "../mcp-client/screen-memory-local.js"; interface ScreenMemorySegment { id: string; @@ -20,6 +16,65 @@ interface ScreenMemorySegment { bytes: number; durationMs: number; corrupt?: boolean; + excluded?: boolean; + exclusionTainted?: boolean; + tainted?: boolean; +} + +export interface ScreenMemoryChapter { + id: string; + /** In v2, chapter identity survives later semantic refinements. */ + revision: number; + aliases: string[]; + startedAt: string; + endedAt: string; + durationMs: number; + label: string; + summary: string; + keywords: string[]; + accessibilitySummary: string; + accessibilityKeywords: string[]; + confidence: number; + segmentRefs: Array<{ id: string; startedAt: string; endedAt: string }>; + sceneRefs: Array<{ id: string; startedAt?: string; endedAt?: string }>; + evidenceRefs: Array<{ + sourceType: string; + sourceKind?: string; + segmentId?: string; + offsetMs?: number; + capturedAt?: string; + keywords?: string[]; + confidence?: number; + }>; + contexts: Array<{ + appName?: string; + windowTitle?: string; + bundleId?: string; + }>; + representativeMoments: Array<{ + momentId: string; + capturedAt: string; + segmentId: string; + offsetMs: number; + reason: string; + }>; + representativeCoverage?: + | string + | { coveredScenes: number; totalScenes: number; truncated: boolean }; + ambiguityReasons: string[]; + indexState: "pending" | "partial" | "ready"; +} + +export interface ScreenMemoryChaptersDocument { + schemaVersion: 1 | 2; + generatedAt: string; + state: string; + coverage: unknown; + chapters: ScreenMemoryChapter[]; +} + +export interface ScreenMemoryFrameDecoder { + (segmentPath: string, offsetMs: number): Buffer; } interface ScreenMemoryConfig { @@ -27,14 +82,35 @@ interface ScreenMemoryConfig { paused?: boolean; retentionHours?: number; maxBytes?: number; + reviewBeforeSending?: boolean; + agentClipRetention?: "forever" | "24-hours" | "7-days" | "30-days"; } export interface RunScreenMemoryMCPStdioOptions { storeDir?: string; env?: NodeJS.ProcessEnv; + /** Test seam; production uses the local ffmpeg executable. */ + decodeFrame?: ScreenMemoryFrameDecoder; } -const EVENTS_JSONL = "events.jsonl"; +const MAX_CHAPTERS = 12; +const MAX_FRAME_BYTES = 900_000; +const MAX_FRAME_EDGE = 1280; +const SEARCH_STOP_WORDS = new Set([ + "and", + "for", + "from", + "that", + "the", + "this", + "was", + "were", + "what", + "when", + "with", + "work", + "working", +]); function log(msg: string): void { process.stderr.write(`[screen-memory-mcp] ${msg}\n`); @@ -64,26 +140,6 @@ function defaultStoreDir(env: NodeJS.ProcessEnv): string { return path.join(defaultAppDataDir(env), "screen-memory"); } -function readJsonl(file: string): T[] { - let body = ""; - try { - body = fs.readFileSync(file, "utf-8"); - } catch { - return []; - } - return body - .split(/\r?\n/) - .filter(Boolean) - .map((line) => { - try { - return JSON.parse(line) as T; - } catch { - return null; - } - }) - .filter((value): value is T => Boolean(value)); -} - function readFeatureConfig(storeDir: string): ScreenMemoryConfig { const configPath = path.join(path.dirname(storeDir), "feature-config.json"); try { @@ -100,30 +156,6 @@ function cutoffFor(minutes: number): number { return Date.now() - Math.max(1, Math.min(minutes, 24 * 60)) * 60_000; } -function eventMatches(event: ScreenMemoryEvent, query: string): boolean { - const needle = query.trim().toLowerCase(); - if (!needle) return true; - return [event.appName, event.windowTitle, event.bundleId, event.source].some( - (value) => value?.toLowerCase().includes(needle), - ); -} - -function recentEvents( - storeDir: string, - args: { query?: string; minutes?: number; limit?: number }, -): ScreenMemoryEvent[] { - const cutoff = cutoffFor(args.minutes ?? 30); - const limit = Math.max(1, Math.min(args.limit ?? 40, 200)); - return readJsonl(path.join(storeDir, EVENTS_JSONL)) - .filter((event) => { - const capturedAt = Date.parse(event.capturedAt); - return Number.isFinite(capturedAt) && capturedAt >= cutoff; - }) - .filter((event) => eventMatches(event, args.query ?? "")) - .sort((a, b) => b.capturedAt.localeCompare(a.capturedAt)) - .slice(0, limit); -} - function readSegments(storeDir: string): ScreenMemorySegment[] { let entries: string[] = []; try { @@ -145,6 +177,609 @@ function readSegments(storeDir: string): ScreenMemorySegment[] { .filter((segment): segment is ScreenMemorySegment => Boolean(segment)); } +function isTimestamp(value: unknown): value is string { + return typeof value === "string" && Number.isFinite(Date.parse(value)); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function cleanText(value: unknown, fallback = ""): string { + return typeof value === "string" ? value.trim().slice(0, 2_000) : fallback; +} + +function cleanKeywords(value: unknown, limit = 12): string[] { + return Array.isArray(value) + ? value + .filter((item): item is string => typeof item === "string") + .map((item) => cleanText(item).slice(0, 80)) + .filter(Boolean) + .slice(0, limit) + : []; +} + +function parseChapter(value: unknown): ScreenMemoryChapter | null { + if ( + !isRecord(value) || + !isTimestamp(value.startedAt) || + !isTimestamp(value.endedAt) + ) + return null; + const id = cleanText(value.id); + if (!id) return null; + const indexState = value.indexState; + if ( + indexState !== "pending" && + indexState !== "partial" && + indexState !== "ready" + ) + return null; + const segmentRefs = Array.isArray(value.segmentRefs) + ? value.segmentRefs.flatMap((ref) => + isRecord(ref) && + cleanText(ref.id) && + isTimestamp(ref.startedAt) && + isTimestamp(ref.endedAt) + ? [ + { + id: cleanText(ref.id), + startedAt: ref.startedAt, + endedAt: ref.endedAt, + }, + ] + : [], + ) + : []; + const evidenceRefs = Array.isArray(value.evidenceRefs) + ? value.evidenceRefs.flatMap((ref) => + isRecord(ref) && cleanText(ref.sourceType) + ? [ + { + sourceType: cleanText(ref.sourceType), + ...(cleanText(ref.sourceKind) + ? { sourceKind: cleanText(ref.sourceKind) } + : {}), + ...(cleanText(ref.segmentId) + ? { segmentId: cleanText(ref.segmentId) } + : {}), + ...(typeof ref.offsetMs === "number" && + Number.isFinite(ref.offsetMs) + ? { offsetMs: ref.offsetMs } + : {}), + ...(isTimestamp(ref.capturedAt) + ? { capturedAt: ref.capturedAt } + : {}), + ...(cleanKeywords(ref.keywords, 64).length + ? { keywords: cleanKeywords(ref.keywords, 64) } + : {}), + ...(typeof ref.confidence === "number" && + Number.isFinite(ref.confidence) + ? { confidence: Math.max(0, Math.min(1, ref.confidence)) } + : {}), + }, + ] + : [], + ) + : []; + const sceneRefs = Array.isArray(value.sceneRefs) + ? value.sceneRefs.flatMap((ref) => + typeof ref === "string" && cleanText(ref) + ? [{ id: cleanText(ref) }] + : isRecord(ref) && cleanText(ref.id) + ? [ + { + id: cleanText(ref.id), + ...(isTimestamp(ref.startedAt) + ? { startedAt: ref.startedAt } + : {}), + ...(isTimestamp(ref.endedAt) ? { endedAt: ref.endedAt } : {}), + }, + ] + : [], + ) + : []; + const representativeCoverage = isRecord(value.representativeCoverage) + ? typeof value.representativeCoverage.coveredScenes === "number" && + Number.isFinite(value.representativeCoverage.coveredScenes) && + typeof value.representativeCoverage.totalScenes === "number" && + Number.isFinite(value.representativeCoverage.totalScenes) && + typeof value.representativeCoverage.truncated === "boolean" + ? { + coveredScenes: Math.max( + 0, + Math.trunc(value.representativeCoverage.coveredScenes), + ), + totalScenes: Math.max( + 0, + Math.trunc(value.representativeCoverage.totalScenes), + ), + truncated: value.representativeCoverage.truncated, + } + : undefined + : cleanText(value.representativeCoverage) || undefined; + const contexts = Array.isArray(value.contexts) + ? value.contexts.flatMap((context) => + isRecord(context) + ? [ + { + ...(cleanText(context.appName) + ? { appName: cleanText(context.appName) } + : {}), + ...(cleanText(context.windowTitle) + ? { windowTitle: cleanText(context.windowTitle) } + : {}), + ...(cleanText(context.bundleId) + ? { bundleId: cleanText(context.bundleId) } + : {}), + }, + ] + : [], + ) + : []; + const representativeMoments = Array.isArray(value.representativeMoments) + ? value.representativeMoments.flatMap((moment) => + isRecord(moment) && + cleanText(moment.momentId) && + isTimestamp(moment.capturedAt) && + cleanText(moment.segmentId) && + typeof moment.offsetMs === "number" && + Number.isFinite(moment.offsetMs) + ? [ + { + momentId: cleanText(moment.momentId), + capturedAt: moment.capturedAt, + segmentId: cleanText(moment.segmentId), + offsetMs: moment.offsetMs, + reason: cleanText(moment.reason), + }, + ] + : [], + ) + : []; + return { + id, + revision: + typeof value.revision === "number" && + Number.isInteger(value.revision) && + value.revision >= 0 + ? value.revision + : 1, + aliases: cleanKeywords(value.aliases, 24), + startedAt: value.startedAt, + endedAt: value.endedAt, + durationMs: + typeof value.durationMs === "number" && Number.isFinite(value.durationMs) + ? value.durationMs + : Date.parse(value.endedAt) - Date.parse(value.startedAt), + label: cleanText(value.label, "Untitled work chapter"), + summary: cleanText(value.summary), + keywords: cleanKeywords(value.keywords), + accessibilitySummary: cleanText(value.accessibilitySummary), + accessibilityKeywords: cleanKeywords(value.accessibilityKeywords), + confidence: + typeof value.confidence === "number" && Number.isFinite(value.confidence) + ? Math.max(0, Math.min(1, value.confidence)) + : 0, + segmentRefs, + sceneRefs, + evidenceRefs, + contexts, + representativeMoments, + ...(representativeCoverage ? { representativeCoverage } : {}), + ambiguityReasons: Array.isArray(value.ambiguityReasons) + ? value.ambiguityReasons + .filter((item): item is string => typeof item === "string") + .map((item) => item.slice(0, 500)) + .slice(0, 8) + : [], + indexState, + }; +} + +/** Reads only the native, retention-bound chapter manifest; malformed rows never escape. */ +export function readScreenMemoryChapters( + storeDir: string, +): ScreenMemoryChaptersDocument | null { + try { + const raw: unknown = JSON.parse( + fs.readFileSync(path.join(storeDir, "chapters.json"), "utf8"), + ); + if ( + !isRecord(raw) || + (raw.schemaVersion !== 1 && raw.schemaVersion !== 2) || + !isTimestamp(raw.generatedAt) || + typeof raw.state !== "string" || + !Array.isArray(raw.chapters) + ) + return null; + const chapters = raw.chapters + .map(parseChapter) + .filter((chapter): chapter is ScreenMemoryChapter => Boolean(chapter)); + if (chapters.length !== raw.chapters.length) return null; + return { + schemaVersion: raw.schemaVersion, + generatedAt: raw.generatedAt, + state: raw.state, + coverage: raw.coverage ?? null, + chapters, + }; + } catch { + return null; + } +} + +type MatchProvenance = { + sourceType: string; + sourceKind?: string; + confidence?: number; + matchedTerms: string[]; +}; + +function textIncludes(value: string, term: string): boolean { + return value.toLowerCase().includes(term); +} + +function isMicrophoneEvidence( + ref: ScreenMemoryChapter["evidenceRefs"][number], +) { + return /microphone|mic/.test( + `${ref.sourceType} ${ref.sourceKind ?? ""}`.toLowerCase(), + ); +} + +function isStrongScreenEvidence( + ref: ScreenMemoryChapter["evidenceRefs"][number], +) { + return /accessibility|ocr|visible|screen|visual/.test( + `${ref.sourceType} ${ref.sourceKind ?? ""}`.toLowerCase(), + ); +} + +export function searchScreenMemoryChapters( + document: ScreenMemoryChaptersDocument, + query: string, + limit = 5, + clientHint?: string, +): Array< + ScreenMemoryChapter & { + score: number; + matchReasons: string[]; + matchProvenance: MatchProvenance[]; + } +> { + const terms = ( + query.toLowerCase().match(/[\p{L}\p{N}_-]{2,}/gu) ?? [] + ).filter((term) => !SEARCH_STOP_WORDS.has(term)); + const exactPhrase = query.trim().toLowerCase(); + const hint = clientHint?.toLowerCase().slice(0, 500) ?? ""; + return document.chapters + .map((chapter) => { + const strongText = [ + chapter.label, + chapter.summary, + ...chapter.keywords, + chapter.accessibilitySummary, + ...chapter.accessibilityKeywords, + ] + .filter(Boolean) + .join(" "); + const contextText = chapter.contexts + .flatMap((context) => [ + context.appName, + context.windowTitle, + context.bundleId, + ]) + .filter(Boolean) + .join(" "); + const strongMatched = terms.filter((term) => + textIncludes(strongText, term), + ); + const contextMatched = terms.filter((term) => + textIncludes(contextText, term), + ); + const evidenceMatches = chapter.evidenceRefs.flatMap((ref) => { + const matchedTerms = terms.filter((term) => + (ref.keywords ?? []).some((keyword) => textIncludes(keyword, term)), + ); + return matchedTerms.length ? [{ ref, matchedTerms }] : []; + }); + const corroboratedTerms = new Set([ + ...strongMatched, + ...evidenceMatches + .filter(({ ref }) => isStrongScreenEvidence(ref)) + .flatMap(({ matchedTerms }) => matchedTerms), + ]); + const evidenceScore = evidenceMatches.reduce( + (sum, { ref, matchedTerms }) => { + const sourceWeight = isStrongScreenEvidence(ref) + ? 1.5 + : isMicrophoneEvidence(ref) + ? matchedTerms.filter((term) => corroboratedTerms.has(term)) + .length + ? 0.12 + : 0 + : /transcript/.test( + `${ref.sourceType} ${ref.sourceKind ?? ""}`.toLowerCase(), + ) + ? 0.3 + : 0.5; + return ( + sum + sourceWeight * matchedTerms.length * (ref.confidence ?? 1) + ); + }, + 0, + ); + const exactStrong = + exactPhrase.length >= 2 && textIncludes(strongText, exactPhrase); + const exactEvidence = chapter.evidenceRefs.some( + (ref) => + isStrongScreenEvidence(ref) && + (ref.keywords ?? []).some((keyword) => + textIncludes(keyword, exactPhrase), + ), + ); + const semanticScore = terms.length + ? (strongMatched.length * 1.2 + + contextMatched.length * 0.2 + + evidenceScore) / + terms.length + + (exactStrong || exactEvidence ? 2 : 0) + : 0; + // A hint may only settle an otherwise close semantic result; it cannot make a non-match win. + const hintScore = + semanticScore > 0 && + hint && + (textIncludes(strongText, hint) || textIncludes(contextText, hint)) + ? 0.03 + : 0; + const score = semanticScore + hintScore + chapter.confidence * 0.001; + return { + ...chapter, + score, + semanticScore, + matchReasons: [ + ...(exactStrong || exactEvidence + ? [ + `exact phrase \"${exactPhrase}\" matched visible or accessibility evidence`, + ] + : []), + ...strongMatched.map( + (term) => + `matched \"${term}\" in chapter or accessibility evidence`, + ), + ...contextMatched.map( + (term) => `matched \"${term}\" in app or document context`, + ), + ...evidenceMatches.flatMap(({ ref, matchedTerms }) => + matchedTerms.map((term) => + isMicrophoneEvidence(ref) && !corroboratedTerms.has(term) + ? `ignored uncorroborated microphone match \"${term}\"` + : `matched \"${term}\" in ${ref.sourceType}${ref.sourceKind ? `/${ref.sourceKind}` : ""} evidence`, + ), + ), + ...(hintScore ? ["client hint weakly broke a close tie"] : []), + ], + matchProvenance: evidenceMatches.map(({ ref, matchedTerms }) => ({ + sourceType: ref.sourceType, + ...(ref.sourceKind ? { sourceKind: ref.sourceKind } : {}), + ...(typeof ref.confidence === "number" + ? { confidence: ref.confidence } + : {}), + matchedTerms, + })), + }; + }) + .filter((chapter) => chapter.semanticScore > 0) + .sort((a, b) => b.score - a.score || b.endedAt.localeCompare(a.endedAt)) + .slice(0, Math.min(Math.max(Math.trunc(limit), 1), MAX_CHAPTERS)) + .map(({ semanticScore: _semanticScore, ...chapter }) => chapter); +} + +export function projectScreenMemoryChapterCandidate( + chapter: ReturnType[number], +) { + const evidenceSources = [ + ...new Set( + chapter.evidenceRefs.map( + (reference) => + `${reference.sourceType}${reference.sourceKind ? `/${reference.sourceKind}` : ""}`, + ), + ), + ].slice(0, 8); + return { + id: chapter.id, + revision: chapter.revision, + aliases: chapter.aliases.slice(0, 8), + startedAt: chapter.startedAt, + endedAt: chapter.endedAt, + durationMs: chapter.durationMs, + label: redactCredentialText(chapter.label), + summary: redactCredentialText(chapter.summary), + keywords: chapter.keywords.slice(0, 8), + confidence: chapter.confidence, + sceneCount: chapter.sceneRefs.length, + evidenceSources, + contexts: chapter.contexts.slice(0, 6), + representativeMoments: chapter.representativeMoments + .slice(0, 8) + .map(({ momentId, capturedAt, reason }) => ({ + momentId, + capturedAt, + reason, + })), + representativeCoverage: chapter.representativeCoverage, + ambiguityReasons: chapter.ambiguityReasons.slice(0, 8), + indexState: chapter.indexState, + score: chapter.score, + matchReasons: chapter.matchReasons.slice(0, 8), + matchProvenance: chapter.matchProvenance.slice(0, 6), + }; +} + +function decodeFrameWithFfmpeg(segmentPath: string, offsetMs: number): Buffer { + const result = spawnSync( + "ffmpeg", + [ + "-v", + "error", + "-ss", + (Math.max(0, offsetMs) / 1000).toFixed(3), + "-i", + segmentPath, + "-frames:v", + "1", + "-vf", + `scale='min(${MAX_FRAME_EDGE},iw)':-2`, + "-f", + "image2pipe", + "-vcodec", + "mjpeg", + "pipe:1", + ], + { encoding: null, maxBuffer: MAX_FRAME_BYTES + 1 }, + ); + if (result.error || result.status !== 0 || !result.stdout?.length) + throw new Error( + "Local frame decoding is unavailable because ffmpeg could not decode this retained segment.", + ); + if (result.stdout.length > MAX_FRAME_BYTES) + throw new Error("The decoded frame exceeded the local byte cap."); + return result.stdout; +} + +function retainedSegmentAt( + storeDir: string, + timestamp: string, +): ScreenMemorySegment { + const target = Date.parse(timestamp); + if (!Number.isFinite(target)) + throw new Error("timestamp must be a valid RFC3339 timestamp."); + if (target > Date.now() + 5_000) + throw new Error("A Rewind frame cannot be requested from the future."); + const segment = readSegments(storeDir).find( + (candidate) => + Date.parse(candidate.startedAt) <= target && + target <= Date.parse(candidate.endedAt), + ); + if (!segment) + throw new Error( + "No clean retained Rewind segment covers that timestamp (it may be outside retention or in a coverage gap).", + ); + if (!segment.path || !fs.existsSync(segment.path)) + throw new Error("That Rewind segment is no longer retained locally."); + let storeRoot: string; + let canonicalSegmentPath: string; + try { + storeRoot = fs.realpathSync(storeDir); + canonicalSegmentPath = fs.realpathSync(segment.path); + } catch { + throw new Error("That Rewind segment is no longer retained locally."); + } + const relative = path.relative(storeRoot, canonicalSegmentPath); + if ( + path.isAbsolute(relative) || + relative === ".." || + relative.startsWith(`..${path.sep}`) || + !fs.statSync(canonicalSegmentPath).isFile() + ) { + throw new Error("That Rewind segment is outside the local memory store."); + } + if (segment.corrupt) + throw new Error( + "That Rewind segment is corrupt and cannot provide a frame.", + ); + if (segment.excluded || segment.exclusionTainted || segment.tainted) + throw new Error( + "That Rewind interval was excluded from capture and cannot provide a frame.", + ); + return { ...segment, path: canonicalSegmentPath }; +} + +function frameContent( + segment: ScreenMemorySegment, + timestamp: string, + decoder: ScreenMemoryFrameDecoder, +) { + const offsetMs = Date.parse(timestamp) - Date.parse(segment.startedAt); + const bytes = decoder(segment.path, offsetMs); + if (bytes.length > MAX_FRAME_BYTES) + throw new Error("The decoded frame exceeded the local byte cap."); + return { + timestamp: new Date(Date.parse(segment.startedAt) + offsetMs).toISOString(), + segmentId: segment.id, + image: { + type: "image" as const, + data: bytes.toString("base64"), + mimeType: "image/jpeg", + }, + }; +} + +export function readScreenMemoryFrame( + storeDir: string, + timestamp: string, + decoder: ScreenMemoryFrameDecoder = decodeFrameWithFfmpeg, +) { + return frameContent( + retainedSegmentAt(storeDir, timestamp), + timestamp, + decoder, + ); +} + +export function selectContactSheetTimestamps( + startAt: string, + endAt: string, + count: number, + representative: string[] = [], +): string[] { + const started = Date.parse(startAt); + const duration = Date.parse(endAt) - started; + const representatives = [ + ...new Set( + representative + .filter(isTimestamp) + .filter( + (timestamp) => + Date.parse(timestamp) >= started && + Date.parse(timestamp) <= started + duration, + ), + ), + ]; + // Reserve one slot for temporal coverage whenever possible: a contact sheet + // should not become a cluster of favorite frames with no sense of the span. + const preferred = representatives.slice(0, Math.max(0, count - 1)); + const even = Array.from({ length: count }, (_, index) => + new Date(started + (duration * (index + 0.5)) / count).toISOString(), + ); + const selected = [...preferred]; + while (selected.length < count) { + const next = even + .filter((timestamp) => !selected.includes(timestamp)) + .sort((a, b) => { + const distance = (timestamp: string) => + Math.min( + ...selected.map((chosen) => + Math.abs(Date.parse(timestamp) - Date.parse(chosen)), + ), + Number.POSITIVE_INFINITY, + ); + return distance(b) - distance(a); + })[0]; + if (!next) break; + selected.push(next); + } + return selected; +} + +export function contactSheetRangeIsValid( + durationMs: number, + usesChapterMoments: boolean, +): boolean { + return durationMs >= 0 && (usesChapterMoments || durationMs <= 5 * 60_000); +} + function recentSegments( storeDir: string, minutes: number, @@ -176,6 +811,341 @@ function textResult(value: unknown) { }; } +export function redactCredentialText(value: string): string { + return value + .replace( + /\b(?:sk-[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9]{12,}|AKIA[A-Z0-9]{16})\b/g, + "[REDACTED CREDENTIAL]", + ) + .replace(/\bBearer\s+[A-Za-z0-9._~+\/-]{8,}/gi, "Bearer [REDACTED]") + .replace( + /\b(api[_-]?key|access[_-]?token|password|secret)\s*[:=]\s*([^\s,;]{4,})/gi, + "$1=[REDACTED]", + ); +} + +const sanitizedEgressStores = new Set(); + +function sanitizeLegacyEgressLog(storeDir: string): void { + if (sanitizedEgressStores.has(storeDir)) return; + sanitizedEgressStores.add(storeDir); + const logPath = path.join(storeDir, "egress.jsonl"); + if (!fs.existsSync(logPath)) return; + const sanitized = fs + .readFileSync(logPath, "utf-8") + .split("\n") + .filter(Boolean) + .flatMap((line) => { + try { + const event = JSON.parse(line) as Record; + const packet = event.packet as + | { evidence?: Array> } + | null + | undefined; + if (!event.receipt && Array.isArray(packet?.evidence)) { + event.receipt = { + evidence: packet.evidence.map((evidence) => ({ + id: evidence.id, + momentId: evidence.momentId, + sourceType: evidence.sourceType, + capturedAt: evidence.capturedAt ?? null, + })), + }; + } + delete event.packet; + delete event.reason; + event.packetBytes = + typeof event.packetBytes === "number" ? event.packetBytes : 0; + return [JSON.stringify(event)]; + } catch { + // A malformed audit row cannot provide reliable provenance. Dropping it + // also prevents arbitrary stale text from surviving this migration. + return []; + } + }); + const temporaryPath = `${logPath}.sanitize-${process.pid}`; + fs.writeFileSync( + temporaryPath, + sanitized.length > 0 ? `${sanitized.join("\n")}\n` : "", + { encoding: "utf-8", mode: 0o600 }, + ); + fs.renameSync(temporaryPath, logPath); + fs.chmodSync(logPath, 0o600); +} + +function appendEgressEvent(storeDir: string, event: Record) { + fs.mkdirSync(storeDir, { recursive: true }); + fs.chmodSync(storeDir, 0o700); + sanitizeLegacyEgressLog(storeDir); + fs.appendFileSync( + path.join(storeDir, "egress.jsonl"), + `${JSON.stringify(event)}\n`, + { encoding: "utf-8", mode: 0o600 }, + ); + fs.chmodSync(path.join(storeDir, "egress.jsonl"), 0o600); +} + +type EgressEvidenceReference = { + id: string; + momentId: string; + sourceType: string; + capturedAt: string | null; +}; + +/** + * Records that bounded evidence was returned without preserving its text. + * The originating Screen Memory interval remains the only durable home for + * transcripts and chapter summaries. + */ +function appendEgressReceipt( + storeDir: string, + requestId: string, + operation: string, + evidence: EgressEvidenceReference[], +): void { + const occurredAt = new Date().toISOString(); + const receipt = { + evidence: evidence.map(({ id, momentId, sourceType, capturedAt }) => ({ + id, + momentId, + sourceType, + capturedAt, + })), + }; + appendEgressEvent(storeDir, { + requestId, + occurredAt, + state: "prepared", + operation, + receipt, + evidenceCount: receipt.evidence.length, + packetBytes: 0, + error: null, + }); + appendEgressEvent(storeDir, { + requestId, + occurredAt: new Date().toISOString(), + state: "completed", + operation, + receipt: null, + evidenceCount: receipt.evidence.length, + packetBytes: 0, + error: null, + }); +} + +export function appendLocalEvidenceReceipt( + storeDir: string, + operation: string, + evidence: Array<{ timestamp: string; segmentId: string }>, +): void { + appendEgressEvent(storeDir, { + requestId: `local-${operation}-${Date.now()}-${process.pid}`, + occurredAt: new Date().toISOString(), + state: "local-evidence-read", + operation, + receipt: { + frames: evidence.map(({ timestamp, segmentId }) => ({ + timestamp, + segmentId, + })), + }, + evidenceCount: evidence.length, + packetBytes: 0, + error: null, + }); +} + +function handoffDir(storeDir: string): string { + return path.join(storeDir, "agent-handoffs"); +} + +function handoffPath(storeDir: string, requestId: string): string { + const safeId = requestId.replace(/[^a-zA-Z0-9_-]/g, ""); + if (!safeId || safeId !== requestId) + throw new Error("Invalid handoff request ID."); + return path.join(handoffDir(storeDir), `${safeId}.json`); +} + +function writePrivateJson(file: string, value: unknown): void { + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`, { + encoding: "utf-8", + mode: 0o600, + }); + fs.chmodSync(file, 0o600); +} + +function parseHandoffRange(args: Record) { + if (typeof args.startAt !== "string" || typeof args.endAt !== "string") { + throw new Error("startAt and endAt must be RFC3339 timestamps."); + } + const startAt = new Date(args.startAt); + const endAt = new Date(args.endAt); + if ( + !Number.isFinite(startAt.getTime()) || + !Number.isFinite(endAt.getTime()) + ) { + throw new Error("startAt and endAt must be valid RFC3339 timestamps."); + } + const durationMs = endAt.getTime() - startAt.getTime(); + if (durationMs < 1_000 || durationMs > 5 * 60_000) { + throw new Error( + "A Rewind Clip handoff must be between one second and five minutes.", + ); + } + if (endAt.getTime() > Date.now() + 5_000) { + throw new Error("A Rewind Clip handoff cannot include future time."); + } + return { + startAt: startAt.toISOString(), + endAt: endAt.toISOString(), + durationMs, + }; +} + +export function screenMemoryMcpToolDefinitions() { + return [ + { + name: "screen_memory_status", + description: + "Read local Clips Screen Memory status, retention, disk usage, and connection health without exposing archive paths.", + inputSchema: { type: "object", properties: {} }, + }, + { + name: "screen_memory_recent_context", + description: + "Search bounded local Screen Memory evidence. Returns coverage, explicit gaps, typed app-context/OCR/transcript evidence when local files contain it, and no media bytes or images.", + inputSchema: { + type: "object", + properties: { + query: { + type: "string", + description: "Optional case-insensitive text filter.", + }, + minutes: { + type: "number", + description: "Lookback window in minutes. Defaults to 30.", + }, + limit: { + type: "number", + description: "Maximum evidence items to return. Defaults to 40.", + }, + }, + }, + }, + { + name: "screen_memory_recent_segments", + description: + "List recent local Screen Memory segment references and timestamps. Media paths and bytes stay behind Clips' bounded private Clip handoff boundary.", + inputSchema: { + type: "object", + properties: { + minutes: { + type: "number", + description: "Lookback window in minutes. Defaults to 30.", + }, + }, + }, + }, + { + name: "screen_memory_search_chapters", + description: + "Search locally generated, retention-bound Rewind work chapters. Returns bounded scored candidates, coverage and ambiguity for the connected agent; never archive paths or media bytes.", + inputSchema: { + type: "object", + required: ["query"], + properties: { + query: { + type: "string", + description: + "Question or topic to match against local chapter labels, summaries, context, and evidence types.", + }, + minutes: { + type: "number", + description: "Optional recent-window cap, up to 24 hours.", + }, + limit: { + type: "number", + description: "Maximum candidates, up to 12.", + }, + clientHint: { + type: "string", + description: + "Optional weak tie-breaker such as project or working directory; cannot hide a stronger semantic match.", + }, + }, + }, + }, + { + name: "screen_memory_frame_at", + description: + "Decode one bounded local JPEG at an exact RFC3339 Rewind timestamp. Returns image content and opaque segment reference only; no archive path, upload, motion, or audio.", + inputSchema: { + type: "object", + required: ["timestamp"], + properties: { + timestamp: { + type: "string", + description: "Exact RFC3339 timestamp.", + }, + reason: { + type: "string", + description: "Short reason for this bounded local evidence read.", + }, + }, + }, + }, + { + name: "screen_memory_contact_sheet", + description: + "Return up to eight bounded, timestamped local Rewind frames from one chapter or a range of no more than five minutes. Prefer representative moments; no paths or uploads.", + inputSchema: { + type: "object", + properties: { + chapterId: { type: "string" }, + startAt: { type: "string" }, + endAt: { type: "string" }, + count: { + type: "number", + description: "1 to 8 frames; defaults to 4.", + }, + reason: { type: "string" }, + }, + }, + }, + { + name: "screen_memory_request_clip", + description: + "Request one bounded private Clip from a Rewind interval when visual or audio inspection is necessary. Clips reviews it first by default, unless the user disabled review in Settings. Raw archive paths are never returned.", + inputSchema: { + type: "object", + required: ["startAt", "endAt"], + properties: { + startAt: { type: "string", description: "RFC3339 start timestamp." }, + endAt: { type: "string", description: "RFC3339 end timestamp." }, + reason: { + type: "string", + description: "Short user-facing reason this media range is needed.", + }, + includeMicrophone: { type: "boolean" }, + includeSystemAudio: { type: "boolean" }, + }, + }, + }, + { + name: "screen_memory_handoff_status", + description: + "Check whether a bounded Rewind Clip request is awaiting review, processing, ready, declined, or failed. Ready responses contain only the private agent URL and timestamps, never local archive paths.", + inputSchema: { + type: "object", + required: ["requestId"], + properties: { requestId: { type: "string" } }, + }, + }, + ]; +} + export async function runScreenMemoryMCPStdio( opts: RunScreenMemoryMCPStdioOptions = {}, ): Promise { @@ -194,50 +1164,7 @@ export async function runScreenMemoryMCPStdio( ); server.setRequestHandler(ListToolsRequestSchema, async () => ({ - tools: [ - { - name: "screen_memory_status", - description: - "Read local Clips Screen Memory status, retention, disk usage, and store path. Local-only.", - inputSchema: { type: "object", properties: {} }, - }, - { - name: "screen_memory_recent_context", - description: - "Search recent local Screen Memory app/window context. Returns timestamps, app names, window titles, and bundle ids; does not return images.", - inputSchema: { - type: "object", - properties: { - query: { - type: "string", - description: "Optional case-insensitive text filter.", - }, - minutes: { - type: "number", - description: "Lookback window in minutes. Defaults to 30.", - }, - limit: { - type: "number", - description: "Maximum events to return. Defaults to 40.", - }, - }, - }, - }, - { - name: "screen_memory_recent_segments", - description: - "List recent local Screen Memory video segment file paths and timestamps. Use only when the user asks to inspect or export local context.", - inputSchema: { - type: "object", - properties: { - minutes: { - type: "number", - description: "Lookback window in minutes. Defaults to 30.", - }, - }, - }, - }, - ], + tools: screenMemoryMcpToolDefinitions(), })); server.setRequestHandler(CallToolRequestSchema, async (request: any) => { @@ -256,24 +1183,353 @@ export async function runScreenMemoryMCPStdio( (sum, segment) => sum + (segment.bytes || 0), 0, ), - storeDir, + note: "The Screen Memory store is connected. Local archive paths are intentionally not exposed to agents.", }); } if (name === "screen_memory_recent_context") { - return textResult({ - storeDir, - events: recentEvents(storeDir, { + const result = await queryScreenMemoryContext( + { query: typeof args.query === "string" ? args.query : undefined, - minutes: typeof args.minutes === "number" ? args.minutes : undefined, + sinceMinutes: + typeof args.minutes === "number" ? args.minutes : undefined, limit: typeof args.limit === "number" ? args.limit : undefined, - }), - }); + }, + { env: { ...env, AGENT_NATIVE_SCREEN_MEMORY_DIR: storeDir } }, + ); + const sanitizedEvidence = result.evidence.map((item) => ({ + ...item, + excerpt: redactCredentialText(item.excerpt), + })); + const sanitizedItems = result.items.map((item) => ({ + ...item, + text: redactCredentialText(item.text), + sourceFile: "local-screen-memory", + })); + const requestId = `egress-mcp-${Date.now()}-${process.pid}`; + const packet = { + question: + typeof args.query === "string" && args.query.trim() + ? redactCredentialText(args.query.trim().slice(0, 4_000)) + : "Recent Screen Memory context", + evidence: sanitizedEvidence.slice(0, 20).map((item) => ({ + id: item.id, + momentId: item.momentId, + sourceType: item.sourceType, + capturedAt: item.capturedAt, + excerpt: item.excerpt.slice(0, 1_200), + })), + }; + appendEgressReceipt( + storeDir, + requestId, + "recent-context", + packet.evidence, + ); + return textResult( + // Keep the old MCP envelope available while adding the typed contract. + { + events: sanitizedItems, + ...result, + items: sanitizedItems, + evidence: sanitizedEvidence, + contextFiles: [], + egress: { + requestId, + packet, + note: "A content-free local activity receipt was recorded before this bounded text packet was returned.", + }, + }, + ); } if (name === "screen_memory_recent_segments") { const minutes = typeof args.minutes === "number" ? args.minutes : 30; + const segments = recentSegments(storeDir, minutes); + const requestId = `egress-mcp-${Date.now()}-${process.pid}`; + const packet = { + question: `List recent Rewind segment references from the previous ${Math.max(1, Math.min(minutes, 24 * 60))} minutes`, + evidence: segments.slice(0, 20).map((segment) => ({ + id: segment.id, + momentId: `segment:${segment.id}`, + sourceType: "app-context", + capturedAt: segment.startedAt, + excerpt: `Local media segment reference from ${segment.startedAt} to ${segment.endedAt}; media path and bytes were not exposed.`, + })), + }; + appendEgressReceipt( + storeDir, + requestId, + "recent-segments", + packet.evidence, + ); + return textResult({ + localOnly: true, + mediaApprovalRequired: true, + segments: segments.map((segment) => ({ + id: segment.id, + startedAt: segment.startedAt, + endedAt: segment.endedAt, + mimeType: segment.mimeType, + durationMs: segment.durationMs, + })), + egress: { + requestId, + packet, + note: "A content-free local activity receipt was recorded before this bounded metadata packet was returned.", + }, + }); + } + if (name === "screen_memory_search_chapters") { + if (typeof args.query !== "string" || !args.query.trim()) + throw new Error("query is required."); + const document = readScreenMemoryChapters(storeDir); + if (!document) + throw new Error( + "No valid local Rewind chapters index is available yet.", + ); + const minutes = + typeof args.minutes === "number" && Number.isFinite(args.minutes) + ? Math.max(1, Math.min(Math.trunc(args.minutes), 24 * 60)) + : null; + const cutoff = minutes ? Date.now() - minutes * 60_000 : null; + const boundedDocument = cutoff + ? { + ...document, + chapters: document.chapters.filter( + (chapter) => Date.parse(chapter.endedAt) >= cutoff, + ), + } + : document; + const candidates = searchScreenMemoryChapters( + boundedDocument, + args.query.trim(), + typeof args.limit === "number" ? args.limit : 5, + typeof args.clientHint === "string" ? args.clientHint : undefined, + ); + const ambiguous = + candidates.length > 1 && + Math.abs(candidates[0].score - candidates[1].score) <= 0.05; + const packet = { + question: redactCredentialText(args.query.trim().slice(0, 4_000)), + evidence: candidates.map((chapter) => ({ + id: chapter.id, + momentId: chapter.id, + sourceType: "chapter", + capturedAt: chapter.startedAt, + excerpt: redactCredentialText( + `${chapter.label}: ${chapter.summary}`, + ).slice(0, 1_200), + })), + }; + const requestId = `egress-mcp-${Date.now()}-${process.pid}`; + appendEgressReceipt( + storeDir, + requestId, + "search-chapters", + packet.evidence, + ); return textResult({ + localOnly: true, + generatedAt: document.generatedAt, + state: document.state, + coverage: document.coverage, + gaps: + isRecord(document.coverage) && Array.isArray(document.coverage.gaps) + ? document.coverage.gaps + : [], + indexState: candidates.some( + (candidate) => candidate.indexState === "pending", + ) + ? "pending" + : candidates.some((candidate) => candidate.indexState === "partial") + ? "partial" + : "ready", + ambiguous, + candidates: candidates.map(projectScreenMemoryChapterCandidate), + egress: { + requestId, + packet, + note: "A content-free local activity receipt was recorded before this bounded chapter packet was returned.", + }, + }); + } + if (name === "screen_memory_frame_at") { + if (typeof args.timestamp !== "string") + throw new Error("timestamp is required."); + const frame = readScreenMemoryFrame( + storeDir, + args.timestamp, + opts.decodeFrame ?? decodeFrameWithFfmpeg, + ); + appendLocalEvidenceReceipt(storeDir, "frame-at", [ + { timestamp: frame.timestamp, segmentId: frame.segmentId }, + ]); + return { + content: [ + { + type: "text" as const, + text: JSON.stringify({ + localOnly: true, + timestamp: frame.timestamp, + segmentId: frame.segmentId, + coverage: "retained clean segment", + note: "This is a bounded local image read. Request screen_memory_request_clip for motion or audio.", + }), + }, + frame.image, + ], + }; + } + if (name === "screen_memory_contact_sheet") { + const document = readScreenMemoryChapters(storeDir); + const chapter = + typeof args.chapterId === "string" + ? document?.chapters.find( + (candidate) => candidate.id === args.chapterId, + ) + : undefined; + const startAt = + chapter?.startedAt ?? + (typeof args.startAt === "string" ? args.startAt : null); + const endAt = + chapter?.endedAt ?? + (typeof args.endAt === "string" ? args.endAt : null); + if (!startAt || !endAt || !isTimestamp(startAt) || !isTimestamp(endAt)) + throw new Error( + "Provide a valid chapterId or RFC3339 startAt and endAt.", + ); + const durationMs = Date.parse(endAt) - Date.parse(startAt); + if (!contactSheetRangeIsValid(durationMs, Boolean(chapter))) + throw new Error( + "A contact sheet range must be between zero and five minutes.", + ); + if (Date.parse(endAt) > Date.now() + 5_000) + throw new Error("A Rewind contact sheet cannot include future time."); + const count = Math.min( + Math.max( + Math.trunc(typeof args.count === "number" ? args.count : 4), + 1, + ), + 8, + ); + const representative = (chapter?.representativeMoments ?? []) + .filter( + (moment) => + Date.parse(moment.capturedAt) >= Date.parse(startAt) && + Date.parse(moment.capturedAt) <= Date.parse(endAt), + ) + .map((moment) => moment.capturedAt); + const timestamps = selectContactSheetTimestamps( + startAt, + endAt, + count, + representative, + ); + const decoder = opts.decodeFrame ?? decodeFrameWithFfmpeg; + const frames = timestamps.flatMap((timestamp) => { + try { + return [readScreenMemoryFrame(storeDir, timestamp, decoder)]; + } catch { + // A chapter may span a brief coverage gap. Keep the clean retained + // frames instead of failing the entire bounded contact sheet. + return []; + } + }); + if (frames.length === 0) { + throw new Error( + "No clean retained Rewind frames cover the requested contact-sheet range.", + ); + } + appendLocalEvidenceReceipt( storeDir, - segments: recentSegments(storeDir, minutes), + "contact-sheet", + frames.map(({ timestamp, segmentId }) => ({ timestamp, segmentId })), + ); + return { + content: [ + { + type: "text" as const, + text: JSON.stringify({ + localOnly: true, + chapterId: chapter?.id ?? null, + startAt: new Date(Date.parse(startAt)).toISOString(), + endAt: new Date(Date.parse(endAt)).toISOString(), + frameCount: frames.length, + selection: representative.length + ? "representative moments then even coverage" + : "even temporal coverage", + frames: frames.map(({ timestamp, segmentId }) => ({ + timestamp, + segmentId, + })), + }), + }, + ...frames.map((frame) => frame.image), + ], + }; + } + if (name === "screen_memory_request_clip") { + const range = parseHandoffRange(args); + const config = readFeatureConfig(storeDir); + const requestId = `handoff-${randomUUID()}`; + const reason = + typeof args.reason === "string" + ? redactCredentialText(args.reason.trim().slice(0, 500)) + : "Visual or audio context requested by the agent"; + const request = { + requestId, + status: "pending", + requestedAt: new Date().toISOString(), + ...range, + reason, + includeMicrophone: args.includeMicrophone !== false, + includeSystemAudio: args.includeSystemAudio !== false, + reviewRequired: config.reviewBeforeSending !== false, + agentClipRetention: config.agentClipRetention ?? "forever", + }; + writePrivateJson(handoffPath(storeDir, requestId), request); + appendEgressEvent(storeDir, { + requestId, + occurredAt: request.requestedAt, + state: "handoff-requested", + operation: "request-clip", + receipt: { + mediaInterval: { startAt: range.startAt, endAt: range.endAt }, + reviewRequired: request.reviewRequired, + }, + evidenceCount: 0, + packetBytes: 0, + error: null, + }); + return textResult({ + requestId, + status: "pending", + startAt: range.startAt, + endAt: range.endAt, + reviewRequired: request.reviewRequired, + note: request.reviewRequired + ? "Clips is waiting for the user to review this bounded range. Poll screen_memory_handoff_status." + : "Clips is processing the explicitly requested bounded range. Poll screen_memory_handoff_status.", + }); + } + if (name === "screen_memory_handoff_status") { + if (typeof args.requestId !== "string") { + throw new Error("requestId is required."); + } + const parsed = JSON.parse( + fs.readFileSync(handoffPath(storeDir, args.requestId), "utf-8"), + ) as Record; + return textResult({ + requestId: parsed.requestId, + status: parsed.status, + startAt: parsed.startAt, + endAt: parsed.endAt, + recordingId: parsed.recordingId, + agentUrl: parsed.agentUrl, + contextUrl: parsed.contextUrl, + expiresAt: parsed.expiresAt, + autoDeleteAt: parsed.autoDeleteAt, + autoDeletedAt: parsed.autoDeletedAt, + error: parsed.error, }); } throw new Error(`Unknown Screen Memory tool: ${name}`); diff --git a/packages/skills/src/mcp-config-writers.ts b/packages/skills/src/mcp-config-writers.ts index d0a72d90b4..41ac468344 100644 --- a/packages/skills/src/mcp-config-writers.ts +++ b/packages/skills/src/mcp-config-writers.ts @@ -123,12 +123,13 @@ export function buildLocalMcpEntryForClient( client: ClientId, args: string[], env?: Record, + command = "agent-native", ): Record { const cleanEnv = env ? Object.fromEntries(Object.entries(env)) : {}; if (client === "opencode") { return { type: "local", - command: ["agent-native", ...args], + command: [command, ...args], enabled: true, ...(Object.keys(cleanEnv).length ? { environment: cleanEnv } : {}), }; @@ -136,13 +137,13 @@ export function buildLocalMcpEntryForClient( if (client === "github-copilot") { return { type: "stdio", - command: "agent-native", + command, args, ...(Object.keys(cleanEnv).length ? { env: cleanEnv } : {}), }; } return { - command: "agent-native", + command, args, ...(Object.keys(cleanEnv).length ? { env: cleanEnv } : {}), }; @@ -543,9 +544,10 @@ export function buildCodexLocalBlock( name: string, args: string[], env?: Record, + command = "agent-native", ): string { const lines: string[] = [codexMcpHeader(name)]; - lines.push(`command = "agent-native"`); + lines.push(`command = ${tomlQuote(command)}`); lines.push(`args = [${args.map(tomlQuote).join(", ")}]`); const cleanEnv = env ? Object.fromEntries(Object.entries(env)) : {}; if (Object.keys(cleanEnv).length) { diff --git a/templates/clips/actions/apply-rewind-extension.ts b/templates/clips/actions/apply-rewind-extension.ts new file mode 100644 index 0000000000..c3157b53e7 --- /dev/null +++ b/templates/clips/actions/apply-rewind-extension.ts @@ -0,0 +1,195 @@ +import { defineAction } from "@agent-native/core"; +import { + readAppState, + writeAppState, +} from "@agent-native/core/application-state"; +import { and, eq, sql } from "drizzle-orm"; +import { z } from "zod"; + +import { isPrivateClip } from "../app/lib/rewind-visibility.js"; +import { parseEdits, serializeEdits } from "../app/lib/timestamp-mapping.js"; +import { getDb, schema } from "../server/db/index.js"; +import { + getCurrentOwnerEmail, + ownerEmailMatches, +} from "../server/lib/recordings.js"; +import { parseTranscriptSegments } from "../shared/transcript-segments.js"; +import { assertNoDirectRecordingShares } from "./make-recording-private-for-rewind.js"; +import { + rewindExtensionKey, + type RewindExtensionRequest, +} from "./request-rewind-extension.js"; + +function shiftedJsonArray( + raw: string | null | undefined, + addedMs: number, + fields: string[], +): string { + try { + const parsed = JSON.parse(raw || "[]"); + if (!Array.isArray(parsed)) return raw || "[]"; + return JSON.stringify( + parsed.map((item) => { + if (!item || typeof item !== "object") return item; + const next = { ...item } as Record; + for (const field of fields) { + if (typeof next[field] === "number") { + next[field] = Math.max(0, Math.round(next[field] + addedMs)); + } + } + return next; + }), + ); + } catch { + return raw || "[]"; + } +} + +export default defineAction({ + description: + "Replace an owned Clip with an explicitly prepended local Rewind range while shifting its timestamped editor data and preserving the original-start marker.", + schema: z.object({ + recordingId: z.string(), + requestId: z.string(), + preRollRecordingId: z.string(), + videoUrl: z.string().min(1), + durationMs: z.number().int().positive(), + addedMs: z + .number() + .int() + .positive() + .max(5 * 60_000), + }), + run: async (args) => { + if (args.videoUrl.startsWith("data:")) { + throw new Error( + "The combined Clip must be uploaded before it is applied.", + ); + } + const db = getDb(); + const ownerEmail = getCurrentOwnerEmail(); + const [recording] = await db + .select() + .from(schema.recordings) + .where( + and( + eq(schema.recordings.id, args.recordingId), + ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail), + ), + ); + const [preRoll] = await db + .select() + .from(schema.recordings) + .where( + and( + eq(schema.recordings.id, args.preRollRecordingId), + ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail), + ), + ); + if (!recording || !preRoll) { + throw new Error("The Clip or its local Rewind pre-roll is unavailable."); + } + if ( + !isPrivateClip(recording.visibility) || + !isPrivateClip(preRoll.visibility) + ) { + throw new Error( + "Rewind history can only be added to a private Clip from a private pre-roll.", + ); + } + await assertNoDirectRecordingShares(args.recordingId); + const key = rewindExtensionKey(args.recordingId); + const request = (await readAppState(key)) as RewindExtensionRequest | null; + if ( + !request || + request.requestId !== args.requestId || + request.status !== "ready" || + request.preRollRecordingId !== args.preRollRecordingId + ) { + throw new Error("The Rewind pre-roll request is not ready to apply."); + } + const expectedDuration = recording.durationMs + args.addedMs; + if (Math.abs(args.durationMs - expectedDuration) > 2_000) { + throw new Error("The combined Clip duration does not match its sources."); + } + + const edits = parseEdits(recording.editsJson); + edits.trims = edits.trims.map((trim) => ({ + ...trim, + startMs: trim.startMs + args.addedMs, + endMs: trim.endMs + args.addedMs, + })); + edits.blurs = edits.blurs.map((blur) => ({ + ...blur, + startMs: blur.startMs + args.addedMs, + endMs: blur.endMs + args.addedMs, + })); + edits.rewindOriginalStartMs = args.addedMs; + + const [transcript] = await db + .select() + .from(schema.recordingTranscripts) + .where(eq(schema.recordingTranscripts.recordingId, args.recordingId)); + const shiftedTranscript = transcript + ? JSON.stringify( + parseTranscriptSegments(transcript.segmentsJson).map((segment) => ({ + ...segment, + startMs: segment.startMs + args.addedMs, + endMs: segment.endMs + args.addedMs, + })), + ) + : null; + const now = new Date().toISOString(); + + await db.transaction(async (tx) => { + await tx + .update(schema.recordings) + .set({ + videoUrl: args.videoUrl, + videoFormat: "mp4", + durationMs: args.durationMs, + editsJson: serializeEdits(edits), + chaptersJson: shiftedJsonArray(recording.chaptersJson, args.addedMs, [ + "startMs", + ]), + thumbnailUrl: null, + animatedThumbnailUrl: null, + updatedAt: now, + }) + .where(eq(schema.recordings.id, args.recordingId)); + if (transcript && shiftedTranscript) { + await tx + .update(schema.recordingTranscripts) + .set({ segmentsJson: shiftedTranscript, updatedAt: now }) + .where(eq(schema.recordingTranscripts.recordingId, args.recordingId)); + } + await tx + .update(schema.recordingComments) + .set({ + videoTimestampMs: sql`${schema.recordingComments.videoTimestampMs} + ${args.addedMs}`, + updatedAt: now, + }) + .where(eq(schema.recordingComments.recordingId, args.recordingId)); + await tx + .update(schema.recordingReactions) + .set({ + videoTimestampMs: sql`${schema.recordingReactions.videoTimestampMs} + ${args.addedMs}`, + }) + .where(eq(schema.recordingReactions.recordingId, args.recordingId)); + await tx + .update(schema.recordings) + .set({ trashedAt: now, expiresAt: now, updatedAt: now }) + .where(eq(schema.recordings.id, args.preRollRecordingId)); + }); + + const applied: RewindExtensionRequest = { + ...request, + status: "applied", + actualDurationMs: args.addedMs, + updatedAt: now, + }; + await writeAppState(key, applied); + await writeAppState("refresh-signal", { ts: Date.now() }); + return { recordingId: args.recordingId, request: applied }; + }, +}); diff --git a/templates/clips/actions/delete-agent-recording-if-unpromoted.ts b/templates/clips/actions/delete-agent-recording-if-unpromoted.ts new file mode 100644 index 0000000000..0af305f755 --- /dev/null +++ b/templates/clips/actions/delete-agent-recording-if-unpromoted.ts @@ -0,0 +1,64 @@ +import { defineAction } from "@agent-native/core"; +import { assertAccess } from "@agent-native/core/sharing"; +import { eq } from "drizzle-orm"; +import { z } from "zod"; + +import { getDb, schema } from "../server/db/index.js"; +import deleteRecordingPermanent from "./delete-recording-permanent.js"; + +export default defineAction({ + description: + "Delete an expired agent-created Clip only if it is still private and has not been renamed, shared, commented on, reacted to, tagged, archived, or trashed.", + schema: z.object({ id: z.string() }), + run: async ({ id }) => { + await assertAccess("recording", id, "editor"); + const db = getDb(); + const [recording] = await db + .select() + .from(schema.recordings) + .where(eq(schema.recordings.id, id)); + if (!recording) return { id, deleted: true, reason: "already-missing" }; + + const [share, comment, reaction, tag] = await Promise.all([ + db + .select({ id: schema.recordingShares.id }) + .from(schema.recordingShares) + .where(eq(schema.recordingShares.resourceId, id)) + .limit(1), + db + .select({ id: schema.recordingComments.id }) + .from(schema.recordingComments) + .where(eq(schema.recordingComments.recordingId, id)) + .limit(1), + db + .select({ id: schema.recordingReactions.id }) + .from(schema.recordingReactions) + .where(eq(schema.recordingReactions.recordingId, id)) + .limit(1), + db + .select({ id: schema.recordingTags.id }) + .from(schema.recordingTags) + .where(eq(schema.recordingTags.recordingId, id)) + .limit(1), + ]); + + const promoted = + recording.visibility !== "private" || + recording.titleSource === "manual" || + Boolean(recording.archivedAt || recording.trashedAt) || + share.length > 0 || + comment.length > 0 || + reaction.length > 0 || + tag.length > 0; + if (promoted) { + await db + .update(schema.recordings) + .set({ expiresAt: null, updatedAt: new Date().toISOString() }) + .where(eq(schema.recordings.id, id)); + return { id, deleted: false, reason: "promoted" }; + } + + await deleteRecordingPermanent.run({ id }); + return { id, deleted: true, reason: "retention-expired" }; + }, +}); diff --git a/templates/clips/actions/get-rewind-extension-request.ts b/templates/clips/actions/get-rewind-extension-request.ts new file mode 100644 index 0000000000..023f612838 --- /dev/null +++ b/templates/clips/actions/get-rewind-extension-request.ts @@ -0,0 +1,22 @@ +import { defineAction } from "@agent-native/core"; +import { readAppState } from "@agent-native/core/application-state"; +import { assertAccess } from "@agent-native/core/sharing"; +import { z } from "zod"; + +import { + rewindExtensionKey, + type RewindExtensionRequest, +} from "./request-rewind-extension.js"; + +export default defineAction({ + description: "Read the current explicit Rewind pre-roll request for a Clip.", + schema: z.object({ recordingId: z.string() }), + http: { method: "GET" }, + run: async ({ recordingId }) => { + await assertAccess("recording", recordingId, "owner"); + const request = (await readAppState( + rewindExtensionKey(recordingId), + )) as RewindExtensionRequest | null; + return { request }; + }, +}); diff --git a/templates/clips/actions/get-screen-memory-status.ts b/templates/clips/actions/get-screen-memory-status.ts index 929ed7a0e6..80d149a9e2 100644 --- a/templates/clips/actions/get-screen-memory-status.ts +++ b/templates/clips/actions/get-screen-memory-status.ts @@ -4,9 +4,25 @@ import { z } from "zod"; export default defineAction({ description: - "Check local Clips Screen Memory status for this machine: enabled/paused state, local storage files, and capture recency. Screen Memory is disabled by default and local-only.", + "Check local Clips Screen Memory status for this machine: enabled/paused state, local metadata files, and capture recency. Screen Memory is disabled by default, local-only, and does not expose media bytes or images.", schema: z.object({}), http: { method: "GET" }, readOnly: true, - run: async () => readScreenMemoryStatus(), + run: async () => { + const status = await readScreenMemoryStatus(); + return { + feature: status.feature, + localOnly: status.localOnly, + enabled: status.enabled, + paused: status.paused, + state: status.state, + captureMode: status.config.captureMode, + retentionHours: status.config.retentionHours, + captureCount: status.captureCount, + storageBytes: status.storageBytes, + oldestCaptureAt: status.oldestCaptureAt, + newestCaptureAt: status.newestCaptureAt, + note: status.note, + }; + }, }); diff --git a/templates/clips/actions/list-rewind-extension-requests.ts b/templates/clips/actions/list-rewind-extension-requests.ts new file mode 100644 index 0000000000..dd3ceef48e --- /dev/null +++ b/templates/clips/actions/list-rewind-extension-requests.ts @@ -0,0 +1,52 @@ +import { defineAction } from "@agent-native/core"; +import { listAppState } from "@agent-native/core/application-state"; +import { and, eq, inArray } from "drizzle-orm"; +import { z } from "zod"; + +import { getDb, schema } from "../server/db/index.js"; +import { + getCurrentOwnerEmail, + ownerEmailMatches, +} from "../server/lib/recordings.js"; +import { + REWIND_EXTENSION_PREFIX, + type RewindExtensionRequest, +} from "./request-rewind-extension.js"; + +export default defineAction({ + description: + "List pending local Rewind pre-roll requests for the signed-in Clips Alpha app.", + schema: z.object({}), + http: { method: "GET" }, + run: async () => { + const entries = await listAppState(REWIND_EXTENSION_PREFIX); + const staleProcessingCutoff = Date.now() - 2 * 60_000; + const requests = entries + .map((entry) => entry.value as unknown as RewindExtensionRequest) + .filter( + (request) => + request && + typeof request.recordingId === "string" && + (request.status === "pending" || + (request.status === "processing" && + Date.parse(request.updatedAt) < staleProcessingCutoff)), + ); + if (!requests.length) return { requests: [] }; + const ids = [...new Set(requests.map((request) => request.recordingId))]; + const ownerEmail = getCurrentOwnerEmail(); + const accessible = await getDb() + .select({ id: schema.recordings.id }) + .from(schema.recordings) + .where( + and( + ownerEmailMatches(schema.recordings.ownerEmail, ownerEmail), + inArray(schema.recordings.id, ids), + eq(schema.recordings.status, "ready"), + ), + ); + const allowed = new Set(accessible.map((row) => row.id)); + return { + requests: requests.filter((request) => allowed.has(request.recordingId)), + }; + }, +}); diff --git a/templates/clips/actions/make-recording-private-for-rewind.test.ts b/templates/clips/actions/make-recording-private-for-rewind.test.ts new file mode 100644 index 0000000000..363a64d68b --- /dev/null +++ b/templates/clips/actions/make-recording-private-for-rewind.test.ts @@ -0,0 +1,84 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mockAssertAccess = vi.hoisted(() => vi.fn(async () => undefined)); +const mockShareRows = vi.hoisted(() => vi.fn(async () => [])); +const mockReturning = vi.hoisted(() => vi.fn(async () => [{ id: "rec_1" }])); +const mockUpdate = vi.hoisted(() => + vi.fn(() => ({ + set: vi.fn(() => ({ + where: vi.fn(() => ({ returning: mockReturning })), + })), + })), +); +const mockDb = vi.hoisted(() => ({ + select: vi.fn(() => ({ + from: vi.fn(() => ({ + where: vi.fn(() => ({ limit: mockShareRows })), + })), + })), + update: mockUpdate, +})); + +vi.mock("@agent-native/core", () => ({ + defineAction: (options: unknown) => options, +})); + +vi.mock("@agent-native/core/sharing", () => ({ + assertAccess: (...args: unknown[]) => mockAssertAccess(...args), +})); + +vi.mock("drizzle-orm", () => ({ + and: (...conditions: unknown[]) => ({ conditions }), + eq: (column: unknown, value: unknown) => ({ column, value }), + notExists: (query: unknown) => ({ notExists: query }), +})); + +vi.mock("../server/db/index.js", () => ({ + getDb: () => mockDb, + schema: { + recordings: { + id: "recordings.id", + visibility: "recordings.visibility", + updatedAt: "recordings.updatedAt", + }, + recordingShares: { + id: "recordingShares.id", + resourceId: "recordingShares.resourceId", + }, + }, +})); + +import action, { + DIRECT_SHARE_REWIND_ERROR, +} from "./make-recording-private-for-rewind"; + +describe("make-recording-private-for-rewind", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockShareRows.mockResolvedValue([]); + mockReturning.mockResolvedValue([{ id: "rec_1" }]); + }); + + it("makes an unshared owned Clip private", async () => { + await expect(action.run({ recordingId: "rec_1" })).resolves.toEqual({ + recordingId: "rec_1", + visibility: "private", + }); + expect(mockAssertAccess).toHaveBeenCalledWith( + "recording", + "rec_1", + "owner", + ); + expect(mockUpdate).toHaveBeenCalledOnce(); + }); + + it("refuses to mutate a Clip with an active direct share", async () => { + mockShareRows.mockResolvedValue([{ id: "share_1" }]); + mockReturning.mockResolvedValue([]); + + await expect(action.run({ recordingId: "rec_1" })).rejects.toThrow( + DIRECT_SHARE_REWIND_ERROR, + ); + expect(mockUpdate).toHaveBeenCalledOnce(); + }); +}); diff --git a/templates/clips/actions/make-recording-private-for-rewind.ts b/templates/clips/actions/make-recording-private-for-rewind.ts new file mode 100644 index 0000000000..cd3b87648e --- /dev/null +++ b/templates/clips/actions/make-recording-private-for-rewind.ts @@ -0,0 +1,55 @@ +import { defineAction } from "@agent-native/core"; +import { assertAccess } from "@agent-native/core/sharing"; +import { and, eq, notExists } from "drizzle-orm"; +import { z } from "zod"; + +import { getDb, schema } from "../server/db/index.js"; + +export const DIRECT_SHARE_REWIND_ERROR = + "This Clip is still shared directly with other people. Remove their access in Share before adding local Rewind history."; + +export async function assertNoDirectRecordingShares( + recordingId: string, +): Promise { + const [share] = await getDb() + .select({ id: schema.recordingShares.id }) + .from(schema.recordingShares) + .where(eq(schema.recordingShares.resourceId, recordingId)) + .limit(1); + if (share) throw new Error(DIRECT_SHARE_REWIND_ERROR); +} + +export default defineAction({ + description: + "Make an owned recording private for a Rewind extension, refusing to continue while anyone still has direct access.", + schema: z.object({ recordingId: z.string() }), + run: async ({ recordingId }) => { + await assertAccess("recording", recordingId, "owner"); + + const now = new Date().toISOString(); + const db = getDb(); + const [recording] = await db + .update(schema.recordings) + .set({ visibility: "private", updatedAt: now }) + .where( + and( + eq(schema.recordings.id, recordingId), + notExists( + db + .select({ id: schema.recordingShares.id }) + .from(schema.recordingShares) + .where(eq(schema.recordingShares.resourceId, recordingId)), + ), + ), + ) + .returning({ id: schema.recordings.id }); + if (!recording) { + // The conditional UPDATE is the invariant. This follow-up read only + // chooses the useful error message after the atomic mutation declined. + await assertNoDirectRecordingShares(recordingId); + throw new Error("This Clip is unavailable."); + } + + return { recordingId, visibility: "private" as const }; + }, +}); diff --git a/templates/clips/actions/query-screen-memory-context.ts b/templates/clips/actions/query-screen-memory-context.ts index 04441498a7..b6dca38345 100644 --- a/templates/clips/actions/query-screen-memory-context.ts +++ b/templates/clips/actions/query-screen-memory-context.ts @@ -1,10 +1,10 @@ import { defineAction } from "@agent-native/core"; -import { queryScreenMemoryContext } from "@agent-native/core/mcp-client"; +import { queryScreenMemoryForAgent } from "@agent-native/core/mcp-client"; import { z } from "zod"; export default defineAction({ description: - "Search recent local Clips Screen Memory context from this machine. Returns bounded OCR/context snippets only; use get-screen-memory-status first if availability is unclear.", + "Search bounded local Clips Screen Memory evidence from this machine. Returns typed app-context, OCR, or transcript excerpts only when present in local files, plus coverage, gaps, segment references, jump targets, and truncation; never media bytes or images. Use get-screen-memory-status first if availability is unclear.", schema: z.object({ query: z .string() @@ -21,9 +21,9 @@ export default defineAction({ .min(1) .max(50) .default(10) - .describe("Maximum number of context snippets to return"), + .describe("Maximum number of bounded local evidence items to return"), }), http: { method: "GET" }, readOnly: true, - run: async (args) => queryScreenMemoryContext(args), + run: async (args) => queryScreenMemoryForAgent(args), }); diff --git a/templates/clips/actions/request-rewind-extension.ts b/templates/clips/actions/request-rewind-extension.ts new file mode 100644 index 0000000000..8b57c4deec --- /dev/null +++ b/templates/clips/actions/request-rewind-extension.ts @@ -0,0 +1,81 @@ +import { defineAction } from "@agent-native/core"; +import { + readAppState, + writeAppState, +} from "@agent-native/core/application-state"; +import { assertAccess } from "@agent-native/core/sharing"; +import { eq } from "drizzle-orm"; +import { z } from "zod"; + +import { isPrivateClip } from "../app/lib/rewind-visibility.js"; +import { getDb, schema } from "../server/db/index.js"; +import { nanoid } from "../server/lib/recordings.js"; +import { assertNoDirectRecordingShares } from "./make-recording-private-for-rewind.js"; + +export const REWIND_EXTENSION_PREFIX = "rewind-extension-request-"; + +export interface RewindExtensionRequest { + [key: string]: unknown; + requestId: string; + recordingId: string; + seconds: 30 | 300; + status: + | "pending" + | "processing" + | "ready" + | "applying" + | "applied" + | "failed"; + requestedAt: string; + updatedAt: string; + preRollRecordingId?: string; + actualDurationMs?: number; + error?: string; +} + +export function rewindExtensionKey(recordingId: string): string { + return `${REWIND_EXTENSION_PREFIX}${recordingId}`; +} + +export default defineAction({ + description: + "Ask the signed-in Clips Alpha app to retrieve an explicit 30-second or five-minute pre-roll from local Rewind for one owned recording.", + schema: z.object({ + recordingId: z.string(), + seconds: z.union([z.literal(30), z.literal(300)]), + }), + run: async ({ recordingId, seconds }) => { + await assertAccess("recording", recordingId, "owner"); + const [recording] = await getDb() + .select({ visibility: schema.recordings.visibility }) + .from(schema.recordings) + .where(eq(schema.recordings.id, recordingId)); + if (!recording || !isPrivateClip(recording.visibility)) { + throw new Error( + "Make this Clip private before adding local Rewind history.", + ); + } + await assertNoDirectRecordingShares(recordingId); + const key = rewindExtensionKey(recordingId); + const existing = (await readAppState(key)) as RewindExtensionRequest | null; + if (existing && !["failed", "applied"].includes(existing.status)) { + return existing; + } + if (existing?.status === "applied") { + throw new Error( + "This Clip already includes an explicit Rewind pre-roll.", + ); + } + const now = new Date().toISOString(); + const request: RewindExtensionRequest = { + requestId: `rewind-extend-${nanoid()}`, + recordingId, + seconds, + status: "pending", + requestedAt: now, + updatedAt: now, + }; + await writeAppState(key, request); + return request; + }, +}); diff --git a/templates/clips/actions/update-rewind-extension-request.ts b/templates/clips/actions/update-rewind-extension-request.ts new file mode 100644 index 0000000000..1c1d74e135 --- /dev/null +++ b/templates/clips/actions/update-rewind-extension-request.ts @@ -0,0 +1,47 @@ +import { defineAction } from "@agent-native/core"; +import { + readAppState, + writeAppState, +} from "@agent-native/core/application-state"; +import { assertAccess } from "@agent-native/core/sharing"; +import { z } from "zod"; + +import { + rewindExtensionKey, + type RewindExtensionRequest, +} from "./request-rewind-extension.js"; + +export default defineAction({ + description: + "Update the local Clips Alpha processing state for an explicit Rewind pre-roll request.", + schema: z.object({ + recordingId: z.string(), + requestId: z.string(), + status: z.enum(["processing", "ready", "failed"]), + preRollRecordingId: z.string().optional(), + actualDurationMs: z.number().int().positive().optional(), + error: z.string().max(1200).optional(), + }), + run: async (args) => { + await assertAccess("recording", args.recordingId, "owner"); + const key = rewindExtensionKey(args.recordingId); + const current = (await readAppState(key)) as RewindExtensionRequest | null; + if (!current || current.requestId !== args.requestId) { + throw new Error("Rewind extension request is no longer current."); + } + const next: RewindExtensionRequest = { + ...current, + status: args.status, + updatedAt: new Date().toISOString(), + ...(args.preRollRecordingId + ? { preRollRecordingId: args.preRollRecordingId } + : {}), + ...(args.actualDurationMs + ? { actualDurationMs: args.actualDurationMs } + : {}), + ...(args.error ? { error: args.error } : {}), + }; + await writeAppState(key, next); + return next; + }, +}); diff --git a/templates/clips/app/components/editor/editor-layout.tsx b/templates/clips/app/components/editor/editor-layout.tsx index 701eae49c7..44aea116fc 100644 --- a/templates/clips/app/components/editor/editor-layout.tsx +++ b/templates/clips/app/components/editor/editor-layout.tsx @@ -49,6 +49,7 @@ import { readPlaybackSpeedPreference, savePlaybackSpeedPreference, } from "@/lib/playback-speed"; +import { canOfferRewindHistory } from "@/lib/rewind-visibility"; import { parseEdits, getExcludedRanges, @@ -62,6 +63,7 @@ import { computePeaks, type WaveformPeaks } from "@/lib/waveform-peaks"; import { ChaptersEditor } from "./chapters-editor"; import { defaultSelectionRange } from "./editor-selection"; import { EditorToolbar } from "./editor-toolbar"; +import { RewindExtensionDialog } from "./rewind-extension-dialog"; import { StitchManager } from "./stitch-manager"; import { ThumbnailPicker } from "./thumbnail-picker"; import { Timeline } from "./timeline"; @@ -219,6 +221,7 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) { const [thumbOpen, setThumbOpen] = useState(false); const [stitchOpen, setStitchOpen] = useState(false); + const [rewindOpen, setRewindOpen] = useState(false); const [chaptersOpen, setChaptersOpen] = useState(false); const containerRef = useRef(null); @@ -636,6 +639,10 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) { onOpenThumbnailPicker={() => setThumbOpen(true)} onOpenChapters={() => setChaptersOpen((v) => !v)} onOpenStitch={() => setStitchOpen(true)} + onOpenRewind={() => setRewindOpen(true)} + rewindAlreadyAdded={Boolean(edits.rewindOriginalStartMs)} + rewindAvailable={canOfferRewindHistory(playerData?.role)} + rewindRequiresPrivate={recording?.visibility !== "private"} chaptersOpen={chaptersOpen} /> @@ -739,6 +746,7 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) { playheadMs={playheadMs} chapters={chapters} splitPoints={splitPoints} + originalStartMs={edits.rewindOriginalStartMs} onSeek={seek} onClickChapter={(c) => seek(c.startMs)} /> @@ -787,6 +795,23 @@ export function EditorLayout({ recordingId, className }: EditorLayoutProps) { onOpenChange={setStitchOpen} seedRecordingId={recordingId} /> + {canOfferRewindHistory(playerData?.role) ? ( + { + await playerDataQuery.refetch(); + }} + onApplied={async () => { + await playerDataQuery.refetch(); + }} + /> + ) : null} ); } diff --git a/templates/clips/app/components/editor/editor-toolbar.tsx b/templates/clips/app/components/editor/editor-toolbar.tsx index 2f12fbd93a..743074cdbd 100644 --- a/templates/clips/app/components/editor/editor-toolbar.tsx +++ b/templates/clips/app/components/editor/editor-toolbar.tsx @@ -16,6 +16,7 @@ import { IconDownload, IconLoader2, IconTrash, + IconHistory, } from "@tabler/icons-react"; import { useState } from "react"; import { toast } from "sonner"; @@ -84,6 +85,10 @@ export interface EditorToolbarProps { onOpenThumbnailPicker: () => void; onOpenChapters: () => void; onOpenStitch: () => void; + onOpenRewind: () => void; + rewindAlreadyAdded?: boolean; + rewindAvailable?: boolean; + rewindRequiresPrivate?: boolean; chaptersOpen?: boolean; } @@ -103,6 +108,10 @@ export function EditorToolbar({ onOpenThumbnailPicker, onOpenChapters, onOpenStitch, + onOpenRewind, + rewindAlreadyAdded, + rewindAvailable = true, + rewindRequiresPrivate = false, chaptersOpen, }: EditorToolbarProps) { const t = useT(); @@ -482,6 +491,19 @@ export function EditorToolbar({ {t("editorToolbar.stitchClips")} + {rewindAvailable ? ( + + + {rewindAlreadyAdded + ? "Rewind history added" + : rewindRequiresPrivate + ? "Make private and add Rewind history…" + : "Add what happened before…"} + + ) : null} setClearOpen(true)}> @@ -490,6 +512,20 @@ export function EditorToolbar({ + {rewindAvailable && !rewindAlreadyAdded ? ( + + ) : null} +
diff --git a/templates/clips/app/components/editor/rewind-extension-dialog.tsx b/templates/clips/app/components/editor/rewind-extension-dialog.tsx new file mode 100644 index 0000000000..2f5032f8d9 --- /dev/null +++ b/templates/clips/app/components/editor/rewind-extension-dialog.tsx @@ -0,0 +1,338 @@ +import { + agentNativePath, + appBasePath, +} from "@agent-native/core/client/api-path"; +import { useActionMutation } from "@agent-native/core/client/hooks"; +import { useT } from "@agent-native/core/client/i18n"; +import { IconHistory, IconLoader2 } from "@tabler/icons-react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { toast } from "sonner"; + +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { exportConcat } from "@/lib/ffmpeg-export"; +import { uploadFileClient } from "@/lib/upload-file-client"; + +interface RewindExtensionRequest { + requestId: string; + recordingId: string; + seconds: 30 | 300; + status: + | "pending" + | "processing" + | "ready" + | "applying" + | "applied" + | "failed"; + preRollRecordingId?: string; + actualDurationMs?: number; + error?: string; +} + +interface RewindExtensionDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + recordingId: string; + durationMs: number; + videoFormat: "webm" | "mp4"; + hasAudio: boolean; + visibility: "private" | "org" | "public"; + onVisibilityChanged: () => void | Promise; + onApplied: () => void | Promise; +} + +function wait(ms: number): Promise { + return new Promise((resolve) => window.setTimeout(resolve, ms)); +} + +async function readRequest( + recordingId: string, +): Promise { + const params = new URLSearchParams({ recordingId }); + const response = await fetch( + agentNativePath( + `/_agent-native/actions/get-rewind-extension-request?${params}`, + ), + ); + const json = await response.json().catch(() => null); + if (!response.ok) { + throw new Error(json?.error || json?.message || "Could not check Rewind."); + } + return (json?.result ?? json)?.request ?? null; +} + +export function RewindExtensionDialog({ + open, + onOpenChange, + recordingId, + durationMs, + videoFormat, + hasAudio, + visibility, + onVisibilityChanged, + onApplied, +}: RewindExtensionDialogProps) { + const t = useT(); + const makePrivateForRewind = useActionMutation( + "make-recording-private-for-rewind", + ); + const requestExtension = useActionMutation("request-rewind-extension"); + const applyExtension = useActionMutation("apply-rewind-extension"); + const requestTranscript = useActionMutation("request-transcript" as any); + const [busy, setBusy] = useState(false); + const [status, setStatus] = useState(null); + const [progress, setProgress] = useState(0); + const [privacyConfirmed, setPrivacyConfirmed] = useState(false); + const mounted = useRef(true); + + useEffect(() => { + return () => { + mounted.current = false; + }; + }, []); + + useEffect(() => { + if (!open && !busy) { + setStatus(null); + setProgress(0); + setPrivacyConfirmed(false); + } + }, [busy, open]); + + const makePrivate = useCallback(async () => { + setBusy(true); + setStatus("Making this Clip private…"); + try { + await makePrivateForRewind.mutateAsync({ recordingId }); + await onVisibilityChanged(); + setPrivacyConfirmed(true); + setStatus(null); + toast.success(t("rewindExtension.privateReady")); + } catch (error) { + toast.error(error instanceof Error ? error.message : String(error)); + setStatus(null); + } finally { + setBusy(false); + } + }, [makePrivateForRewind, onVisibilityChanged, recordingId, t]); + + const addFromRewind = useCallback( + async (seconds: 30 | 300) => { + setBusy(true); + setProgress(0); + try { + setStatus("Asking Clips Alpha for local Rewind history…"); + const created = (await requestExtension.mutateAsync({ + recordingId, + seconds, + })) as RewindExtensionRequest; + let request = created; + for (let attempt = 0; attempt < 160; attempt += 1) { + if (request.status === "failed") { + throw new Error( + request.error || "Clips Alpha could not read Rewind.", + ); + } + if ( + request.status === "ready" && + request.preRollRecordingId && + request.actualDurationMs + ) { + break; + } + setStatus( + request.status === "processing" + ? "Clips Alpha is preparing that local interval…" + : "Waiting for Clips Alpha…", + ); + await wait(1_500); + request = (await readRequest(recordingId)) ?? request; + } + if ( + request.status !== "ready" || + !request.preRollRecordingId || + !request.actualDurationMs + ) { + throw new Error( + "Clips Alpha did not finish the Rewind request in time.", + ); + } + + setStatus("Combining the selected history with this Clip…"); + const blob = await exportConcat( + [ + { + url: `${appBasePath()}/api/video/${encodeURIComponent(request.preRollRecordingId)}`, + format: "mp4", + hasAudio, + }, + { + url: `${appBasePath()}/api/video/${encodeURIComponent(recordingId)}`, + format: videoFormat, + hasAudio, + }, + ], + (next) => mounted.current && setProgress(next.progress), + ); + setStatus("Saving the longer Clip…"); + const upload = await uploadFileClient( + blob, + `rewind-${recordingId}.mp4`, + ); + if (!upload?.url) { + throw new Error( + "Connect video storage before adding Rewind history.", + ); + } + await applyExtension.mutateAsync({ + recordingId, + requestId: request.requestId, + preRollRecordingId: request.preRollRecordingId, + videoUrl: upload.url, + durationMs: durationMs + request.actualDurationMs, + addedMs: request.actualDurationMs, + }); + if (hasAudio) { + void requestTranscript + .mutateAsync({ recordingId, force: true, regenerate: true }) + .catch(() => { + toast.info( + "The longer Clip was saved; its transcript can be regenerated later.", + ); + }); + } + await onApplied(); + toast.success( + `${request.actualDurationMs >= 60_000 ? "Five minutes" : "Thirty seconds"} added from Rewind.`, + ); + onOpenChange(false); + } catch (error) { + toast.error(error instanceof Error ? error.message : String(error)); + setStatus(null); + } finally { + if (mounted.current) { + setBusy(false); + setProgress(0); + } + } + }, + [ + applyExtension, + durationMs, + hasAudio, + onApplied, + onOpenChange, + recordingId, + requestExtension, + requestTranscript, + videoFormat, + ], + ); + + return ( + !busy && onOpenChange(next)}> + + + + + {t("rewindExtension.title")} + + + {t("rewindExtension.description")} + + + + {busy ? ( +
+
+
+ {progress > 0 ? ( +
+
+
+ ) : null} +
+ ) : visibility !== "private" && !privacyConfirmed ? ( +
+
+ + {t("rewindExtension.privateFirstTitle")} + + + {t("rewindExtension.privateFirstDescription")} + +
+ +
+ ) : ( +
+ + +
+ )} + + + + + +
+ ); +} diff --git a/templates/clips/app/components/editor/stitch-manager.tsx b/templates/clips/app/components/editor/stitch-manager.tsx index 88ef76ad9a..1a178fd079 100644 --- a/templates/clips/app/components/editor/stitch-manager.tsx +++ b/templates/clips/app/components/editor/stitch-manager.tsx @@ -24,24 +24,9 @@ import { import { ScrollArea } from "@/components/ui/scroll-area"; import { exportConcat } from "@/lib/ffmpeg-export"; import { formatMs } from "@/lib/timestamp-mapping"; +import { uploadFileClient } from "@/lib/upload-file-client"; import { cn } from "@/lib/utils"; -/** Client-side upload via the framework's auto-mounted `/file-upload` route. */ -async function uploadFileClient( - blob: Blob, - filename: string, -): Promise<{ url: string } | null> { - const form = new FormData(); - form.append("file", blob, filename); - const res = await fetch(agentNativePath("/_agent-native/file-upload"), { - method: "POST", - body: form, - }); - if (!res.ok) return null; - const json = await res.json(); - return json?.url ? { url: json.url as string } : null; -} - export interface StitchManagerProps { open: boolean; onOpenChange: (open: boolean) => void; diff --git a/templates/clips/app/components/editor/timeline.tsx b/templates/clips/app/components/editor/timeline.tsx index df414378b7..1c6f13540d 100644 --- a/templates/clips/app/components/editor/timeline.tsx +++ b/templates/clips/app/components/editor/timeline.tsx @@ -1,3 +1,4 @@ +import { useT } from "@agent-native/core/client/i18n"; import { useMemo } from "react"; import { @@ -21,6 +22,8 @@ export interface TimelineProps { chapters?: TimelineChapter[]; excludedRanges?: Array<{ startMs: number; endMs: number }>; splitPoints?: number[]; + /** Countdown-complete start after explicit Rewind history was prepended. */ + originalStartMs?: number; onSeek?: (originalMs: number) => void; onClickChapter?: (chapter: TimelineChapter) => void; className?: string; @@ -44,10 +47,12 @@ export function Timeline({ chapters = [], excludedRanges = [], splitPoints = [], + originalStartMs, onSeek, onClickChapter, className, }: TimelineProps) { + const t = useT(); const ticks = useMemo(() => { if (durationMs <= 0) return []; // Target ~1 tick per 100px at the current zoom, rounded to a human interval. @@ -133,6 +138,27 @@ export function Timeline({ ); })} + {typeof originalStartMs === "number" && originalStartMs > 0 ? ( + + +
+ + {t("timeline.clipStartedHere")} + +
+ + + Recording began after the countdown · {formatMs(originalStartMs)} + + + ) : null} + {/* Playhead */}
, + sources: Array<{ + url: string; + format?: "webm" | "mp4"; + hasAudio?: boolean; + }>, onProgress?: (p: ExportProgress) => void, ): Promise { if (sources.length < 2) { @@ -377,38 +381,41 @@ export async function exportConcat( "-i", `src${i}.${s.format ?? "webm"}`, ]); + const includesAudio = sources.every((source) => source.hasAudio !== false); const filterParts: string[] = []; const concatInputs: string[] = []; for (let i = 0; i < sources.length; i++) { filterParts.push(`[${i}:v]setpts=PTS-STARTPTS[v${i}]`); - filterParts.push(`[${i}:a]asetpts=PTS-STARTPTS[a${i}]`); - concatInputs.push(`[v${i}][a${i}]`); + if (includesAudio) { + filterParts.push(`[${i}:a]asetpts=PTS-STARTPTS[a${i}]`); + concatInputs.push(`[v${i}][a${i}]`); + } else { + concatInputs.push(`[v${i}]`); + } } filterParts.push( - `${concatInputs.join("")}concat=n=${sources.length}:v=1:a=1[outv][outa]`, + `${concatInputs.join("")}concat=n=${sources.length}:v=1:a=${includesAudio ? 1 : 0}[outv]${includesAudio ? "[outa]" : ""}`, ); - await ffmpeg.exec([ + const outputArgs = [ ...inputArgs, "-filter_complex", filterParts.join(";"), "-map", "[outv]", - "-map", - "[outa]", "-c:v", "libx264", "-preset", "veryfast", "-crf", "22", - "-c:a", - "aac", - "-b:a", - "128k", + ...(includesAudio + ? ["-map", "[outa]", "-c:a", "aac", "-b:a", "128k"] + : []), "-movflags", "+faststart", "stitched.mp4", - ]); + ]; + await ffmpeg.exec(outputArgs); const data = (await ffmpeg.readFile("stitched.mp4")) as Uint8Array; const blob = new Blob([data as BlobPart], { type: "video/mp4" }); diff --git a/templates/clips/app/lib/rewind-visibility.test.ts b/templates/clips/app/lib/rewind-visibility.test.ts new file mode 100644 index 0000000000..52b4e9eca6 --- /dev/null +++ b/templates/clips/app/lib/rewind-visibility.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; + +import { + canAddRewindHistory, + canOfferRewindHistory, + isPrivateClip, + rewindHistoryUnavailableReason, +} from "./rewind-visibility"; + +describe("Rewind history visibility", () => { + it("only permits an owner to extend a private Clip", () => { + expect(canAddRewindHistory("owner", "private")).toBe(true); + expect(canAddRewindHistory("owner", "org")).toBe(false); + expect(canAddRewindHistory("owner", "public")).toBe(false); + expect(canAddRewindHistory("viewer", "private")).toBe(false); + expect(canOfferRewindHistory("owner")).toBe(true); + expect(canOfferRewindHistory("viewer")).toBe(false); + expect(isPrivateClip("private")).toBe(true); + expect(isPrivateClip("org")).toBe(false); + }); + + it("explains how to make an owned shared Clip eligible", () => { + expect(rewindHistoryUnavailableReason("owner", "public")).toBe( + "Make this Clip private before adding Rewind history", + ); + expect(rewindHistoryUnavailableReason("viewer", "private")).toBe( + "Only the owner can add Rewind history", + ); + }); +}); diff --git a/templates/clips/app/lib/rewind-visibility.ts b/templates/clips/app/lib/rewind-visibility.ts new file mode 100644 index 0000000000..c62043b926 --- /dev/null +++ b/templates/clips/app/lib/rewind-visibility.ts @@ -0,0 +1,31 @@ +export type ClipVisibility = "private" | "org" | "public"; + +export function isPrivateClip( + visibility: ClipVisibility | string | null | undefined, +): boolean { + return visibility === "private"; +} + +export function canAddRewindHistory( + role: string | null | undefined, + visibility: ClipVisibility | string | null | undefined, +): boolean { + return role === "owner" && isPrivateClip(visibility); +} + +export function canOfferRewindHistory( + role: string | null | undefined, +): boolean { + return role === "owner"; +} + +export function rewindHistoryUnavailableReason( + role: string | null | undefined, + visibility: ClipVisibility | string | null | undefined, +): string | undefined { + if (role !== "owner") return "Only the owner can add Rewind history"; + if (visibility !== "private") { + return "Make this Clip private before adding Rewind history"; + } + return undefined; +} diff --git a/templates/clips/app/lib/timestamp-mapping.test.ts b/templates/clips/app/lib/timestamp-mapping.test.ts index 354cb9f84a..89ad8d4426 100644 --- a/templates/clips/app/lib/timestamp-mapping.test.ts +++ b/templates/clips/app/lib/timestamp-mapping.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "vitest"; -import { skipExcludedRange } from "./timestamp-mapping"; +import { + parseEdits, + serializeEdits, + skipExcludedRange, +} from "./timestamp-mapping"; describe("skipExcludedRange", () => { const cuts = [ @@ -24,3 +28,27 @@ describe("skipExcludedRange", () => { ).toBe(5_000); }); }); + +describe("Rewind original-start provenance", () => { + it("round-trips a positive countdown-complete boundary", () => { + const parsed = parseEdits( + serializeEdits({ + version: 1, + trims: [], + blurs: [], + rewindOriginalStartMs: 30_042.4, + }), + ); + + expect(parsed.rewindOriginalStartMs).toBe(30_042); + }); + + it("drops invalid or non-positive boundaries", () => { + expect( + parseEdits('{"rewindOriginalStartMs":0}').rewindOriginalStartMs, + ).toBeUndefined(); + expect( + parseEdits('{"rewindOriginalStartMs":"30000"}').rewindOriginalStartMs, + ).toBeUndefined(); + }); +}); diff --git a/templates/clips/app/lib/timestamp-mapping.ts b/templates/clips/app/lib/timestamp-mapping.ts index 4b4a6d8555..c83558b0c3 100644 --- a/templates/clips/app/lib/timestamp-mapping.ts +++ b/templates/clips/app/lib/timestamp-mapping.ts @@ -45,6 +45,8 @@ export interface EditsJson { thumbnail?: ThumbnailSpec | null; /** Provenance: source recording IDs when this recording was created via stitch-recordings. */ stitchedFrom?: string[]; + /** Original countdown-complete boundary after an explicit Rewind pre-roll was prepended. */ + rewindOriginalStartMs?: number; } export const DEFAULT_EDITS: EditsJson = { @@ -73,6 +75,11 @@ export function parseEdits(raw: string | null | undefined): EditsJson { ...(Array.isArray(j.stitchedFrom) ? { stitchedFrom: j.stitchedFrom as string[] } : {}), + ...(typeof j.rewindOriginalStartMs === "number" && + Number.isFinite(j.rewindOriginalStartMs) && + j.rewindOriginalStartMs > 0 + ? { rewindOriginalStartMs: Math.round(j.rewindOriginalStartMs) } + : {}), }; } catch { return { ...DEFAULT_EDITS }; diff --git a/templates/clips/app/lib/upload-file-client.ts b/templates/clips/app/lib/upload-file-client.ts new file mode 100644 index 0000000000..471a54adad --- /dev/null +++ b/templates/clips/app/lib/upload-file-client.ts @@ -0,0 +1,17 @@ +import { agentNativePath } from "@agent-native/core/client/api-path"; + +/** Upload a browser-produced file through Clips' authenticated file route. */ +export async function uploadFileClient( + blob: Blob, + filename: string, +): Promise<{ url: string } | null> { + const form = new FormData(); + form.append("file", blob, filename); + const response = await fetch(agentNativePath("/_agent-native/file-upload"), { + method: "POST", + body: form, + }); + if (!response.ok) return null; + const json = await response.json(); + return json?.url ? { url: json.url as string } : null; +} diff --git a/templates/clips/changelog/2026-07-14-rewind-keeps-recent-screen-context-local-and-turns-it-into-clips.md b/templates/clips/changelog/2026-07-14-rewind-keeps-recent-screen-context-local-and-turns-it-into-clips.md new file mode 100644 index 0000000000..9843e9d91e --- /dev/null +++ b/templates/clips/changelog/2026-07-14-rewind-keeps-recent-screen-context-local-and-turns-it-into-clips.md @@ -0,0 +1,6 @@ +--- +type: added +date: 2026-07-14 +--- + +Rewind can keep recent screen context local, make it searchable on your Mac, and explicitly include the previous 30 seconds or 5 minutes in a Clip without starting another recorder diff --git a/templates/clips/changelog/2026-07-17-active-clips-keep-rewind-stable-and-settings-clear.md b/templates/clips/changelog/2026-07-17-active-clips-keep-rewind-stable-and-settings-clear.md new file mode 100644 index 0000000000..9b178e693e --- /dev/null +++ b/templates/clips/changelog/2026-07-17-active-clips-keep-rewind-stable-and-settings-clear.md @@ -0,0 +1,12 @@ +--- +type: fixed +date: 2026-07-17 +--- + +Active Clips now keep Rewind's shared screen and audio capture stable until the Clip ends. Rewind's on/off, pause, and capture-mode controls visibly lock during recording, with a native safeguard for non-UI callers, so changing Rewind can no longer strand a Clip in optimization with a coverage gap. + +Rewind Settings also use application exclusions as the single privacy model, return Back through the surface that opened them, describe local-memory maintenance before asking someone to act, and speak plainly about local agents finding moments on request. + +The recording toolbar's timer now opens Clips without stopping the active recording, and Dock, tray, and shortcut opens all restore the active popover. Rewind-derived hosted Clips now create the resumable storage session they need before recording, and exact-range encoding uses an interactive-speed preset so Stop no longer appears frozen for roughly the Clip's full duration. The compact Rewind settings rows also reserve separate label and control columns to prevent overlap. + +Recorder Home now keeps its bottom navigation fixed while the content region adapts to the available popover height. After Rewind's first-time setup, its single Home switch means remembering or paused; the duplicate Pause button and active-recording lock paragraph are gone. Full disable and capture-mode setup remain in Rewind Settings, while the active-Clip switch uses an unmistakable disabled treatment until recording ends. diff --git a/templates/clips/changelog/2026-07-17-clicking-the-clips-icon-during-a-recording-now-opens-the-app.md b/templates/clips/changelog/2026-07-17-clicking-the-clips-icon-during-a-recording-now-opens-the-app.md new file mode 100644 index 0000000000..94439e97cc --- /dev/null +++ b/templates/clips/changelog/2026-07-17-clicking-the-clips-icon-during-a-recording-now-opens-the-app.md @@ -0,0 +1,6 @@ +--- +type: fixed +date: 2026-07-17 +--- + +Clicking the Clips icon during a recording now opens the app without stopping capture. diff --git a/templates/clips/changelog/2026-07-17-default-mic-now-uses-the-mac-default-and-transcription-shares-one-capture.md b/templates/clips/changelog/2026-07-17-default-mic-now-uses-the-mac-default-and-transcription-shares-one-capture.md new file mode 100644 index 0000000000..05a94c7e80 --- /dev/null +++ b/templates/clips/changelog/2026-07-17-default-mic-now-uses-the-mac-default-and-transcription-shares-one-capture.md @@ -0,0 +1,6 @@ +--- +type: fixed +date: 2026-07-17 +--- + +Clips now uses the Mac's real default microphone and shares one physical mic/system capture with live transcription, preventing spoken recordings from becoming silent. diff --git a/templates/clips/changelog/2026-07-17-full-screen-clips-finalize-even-when-screencapturekit-stop-hangs.md b/templates/clips/changelog/2026-07-17-full-screen-clips-finalize-even-when-screencapturekit-stop-hangs.md new file mode 100644 index 0000000000..a6545d5fe5 --- /dev/null +++ b/templates/clips/changelog/2026-07-17-full-screen-clips-finalize-even-when-screencapturekit-stop-hangs.md @@ -0,0 +1,6 @@ +--- +type: fixed +date: 2026-07-17 +--- + +Full-screen Clips now finish saving when macOS stops capture but fails to return from ScreenCaptureKit's synchronous stop callback. diff --git a/templates/clips/changelog/2026-07-17-normal-full-screen-clips-no-longer-stop-immediately-after-th.md b/templates/clips/changelog/2026-07-17-normal-full-screen-clips-no-longer-stop-immediately-after-th.md new file mode 100644 index 0000000000..5f967de04c --- /dev/null +++ b/templates/clips/changelog/2026-07-17-normal-full-screen-clips-no-longer-stop-immediately-after-th.md @@ -0,0 +1,6 @@ +--- +type: fixed +date: 2026-07-17 +--- + +Normal full-screen Clips no longer stop immediately after the countdown. diff --git a/templates/clips/changelog/2026-07-17-rewind-can-hand-one-bounded-private-clip-to-your-agent.md b/templates/clips/changelog/2026-07-17-rewind-can-hand-one-bounded-private-clip-to-your-agent.md new file mode 100644 index 0000000000..648d56b9f8 --- /dev/null +++ b/templates/clips/changelog/2026-07-17-rewind-can-hand-one-bounded-private-clip-to-your-agent.md @@ -0,0 +1,12 @@ +--- +title: Rewind can hand one bounded private Clip to your agent +date: 2026-07-17 +--- + +Rewind now connects directly to Codex or Claude Code through a dedicated local Screen Memory connection. When you ask an agent about something recent, bounded matching text can be returned without exposing local archive paths. + +If the agent needs to see or hear the moment, it can request one timestamp range of up to five minutes. Clips shows **Review before sending** by default, including the exact interval, local preview, microphone and Mac-audio choices, and a clear statement that only the selected range becomes a private Clip. You can turn review off in Rewind Settings when the agent request itself is enough approval; Clips still leaves a visible handoff receipt. + +Agent-created Clips are kept in the Library by default. An optional retention setting can remove future agent-created Clips after 24 hours, 7 days, or 30 days. Renaming, sharing, commenting on, reacting to, tagging, or archiving one preserves it. + +Raw Rewind recordings and the complete local index remain on the Mac. The old Private versus Cloud-assisted mode gate has been removed; the boundary is now the bounded request and, for media, the private Clip handoff. diff --git a/templates/clips/changelog/2026-07-17-rewind-home-can-copy-a-local-agent-prompt.md b/templates/clips/changelog/2026-07-17-rewind-home-can-copy-a-local-agent-prompt.md new file mode 100644 index 0000000000..f64e5e4ff7 --- /dev/null +++ b/templates/clips/changelog/2026-07-17-rewind-home-can-copy-a-local-agent-prompt.md @@ -0,0 +1,6 @@ +--- +type: improved +date: 2026-07-17 +--- + +Rewind's Home status is now one calm line instead of repeating retention settings. Its typography matches the recorder controls, and a new copy button provides a ready-to-paste prompt that helps a local agent recover the newest relevant Rewind context without uploading raw media. Pausing and resuming also restore Recorder Home to its complete natural height, so the Clip controls and fixed footer never overlap after the shorter paused state. diff --git a/templates/clips/changelog/2026-07-17-rewind-now-has-a-calm-home-status-focused-first-run-consent-.md b/templates/clips/changelog/2026-07-17-rewind-now-has-a-calm-home-status-focused-first-run-consent-.md new file mode 100644 index 0000000000..4b6f67c0ba --- /dev/null +++ b/templates/clips/changelog/2026-07-17-rewind-now-has-a-calm-home-status-focused-first-run-consent-.md @@ -0,0 +1,6 @@ +--- +type: improved +date: 2026-07-17 +--- + +Rewind now has a calm home status, focused first-run consent, local Memory search, and dedicated privacy settings. diff --git a/templates/clips/changelog/2026-07-17-rewind-settings-are-smaller-clearer-and-agent-first.md b/templates/clips/changelog/2026-07-17-rewind-settings-are-smaller-clearer-and-agent-first.md new file mode 100644 index 0000000000..de44159096 --- /dev/null +++ b/templates/clips/changelog/2026-07-17-rewind-settings-are-smaller-clearer-and-agent-first.md @@ -0,0 +1,6 @@ +--- +type: improved +date: 2026-07-17 +--- + +Rewind now fits the Clips popover instead of taking it over. Home shows one compact status row with Pause and an on/off switch; configuration lives in Settings. Privacy language is plain, disk limits are visible, excluded apps use a native application picker instead of bundle IDs, and agent retrieval is the primary story while manual search remains a quiet local fallback. diff --git a/templates/clips/changelog/2026-07-17-the-full-clips-interface-stays-available-while-recording.md b/templates/clips/changelog/2026-07-17-the-full-clips-interface-stays-available-while-recording.md new file mode 100644 index 0000000000..6ab35941f2 --- /dev/null +++ b/templates/clips/changelog/2026-07-17-the-full-clips-interface-stays-available-while-recording.md @@ -0,0 +1,6 @@ +--- +type: improved +date: 2026-07-17 +--- + +The full Clips interface now stays available while a recording is active. Its compact header uses a clear live REC signal and an explicit Stop button without explaining the interface back to the user. Reopening the interface respects **Show Clips in screen captures**: it remains usable but capture-excluded by default, and appears in recordings only when the user turns that setting on. diff --git a/templates/clips/changelog/2026-07-18-rewind-organizes-recent-work-into-local-searchable-chapters.md b/templates/clips/changelog/2026-07-18-rewind-organizes-recent-work-into-local-searchable-chapters.md new file mode 100644 index 0000000000..eec2638465 --- /dev/null +++ b/templates/clips/changelog/2026-07-18-rewind-organizes-recent-work-into-local-searchable-chapters.md @@ -0,0 +1,8 @@ +--- +type: improved +date: 2026-07-18 +--- + +Rewind now divides retained screen memory into local work chapters that any connected agent can search. Agents can inspect an exact local frame or a small contact sheet before asking to turn a bounded range into a private Clip. Raw Rewind media and archive paths remain on the Mac. + +The copied Rewind prompt can repair the local connection, handles ambiguous chapters explicitly, and only escalates to a Clip when audio, motion, deeper processing, or durable queryability requires it. Rewind Settings also make agent activity and local-export receipts easier to find. diff --git a/templates/clips/changelog/2026-07-18-searchable-rewind-chapters.md b/templates/clips/changelog/2026-07-18-searchable-rewind-chapters.md new file mode 100644 index 0000000000..1a591c056a --- /dev/null +++ b/templates/clips/changelog/2026-07-18-searchable-rewind-chapters.md @@ -0,0 +1,6 @@ +--- +type: improved +date: 2026-07-18 +--- + +Rewind now organizes recent activity into searchable work chapters with stronger local context, representative moments, and truthful coverage gaps. diff --git a/templates/clips/changelog/2026-07-19-rewind-backed-recordings-now-start-immediately-after-the-cou.md b/templates/clips/changelog/2026-07-19-rewind-backed-recordings-now-start-immediately-after-the-cou.md new file mode 100644 index 0000000000..0a230f811a --- /dev/null +++ b/templates/clips/changelog/2026-07-19-rewind-backed-recordings-now-start-immediately-after-the-cou.md @@ -0,0 +1,6 @@ +--- +type: improved +date: 2026-07-19 +--- + +Rewind-backed recordings now start immediately after the countdown and upload progressively while you record. diff --git a/templates/clips/changelog/2026-07-20-rewind-can-set-up-your-local-agent-once-so-future-requests-w.md b/templates/clips/changelog/2026-07-20-rewind-can-set-up-your-local-agent-once-so-future-requests-w.md new file mode 100644 index 0000000000..acded5b6bf --- /dev/null +++ b/templates/clips/changelog/2026-07-20-rewind-can-set-up-your-local-agent-once-so-future-requests-w.md @@ -0,0 +1,6 @@ +--- +type: improved +date: 2026-07-20 +--- + +Rewind can set up your local agent once, so future requests work with a simple ‘Look at Rewind.’ diff --git a/templates/clips/desktop/package.json b/templates/clips/desktop/package.json index 9800d78fa9..1d8a0d8790 100644 --- a/templates/clips/desktop/package.json +++ b/templates/clips/desktop/package.json @@ -10,6 +10,7 @@ "dev": "pnpm run check:prereqs && pnpm run kill:installed && tauri dev", "build": "vite build", "build:tauri": "CLIPS_DESKTOP_LOCAL_BUILD=1 pnpm run check:prereqs && CLIPS_DESKTOP_LOCAL_BUILD=1 tauri build --config '{\"bundle\":{\"createUpdaterArtifacts\":false},\"plugins\":{\"updater\":{\"active\":false}}}' && node ./scripts/sign-macos-local.ts", + "build:alpha": "CLIPS_DESKTOP_LOCAL_BUILD=1 RUSTUP_TOOLCHAIN=1.88.0 pnpm run check:prereqs && CLIPS_DESKTOP_LOCAL_BUILD=1 RUSTUP_TOOLCHAIN=1.88.0 tauri build --debug --config src-tauri/tauri.alpha.conf.json && CLIPS_MACOS_APP_NAME='Clips Alpha' CLIPS_MACOS_BUILD_PROFILE=debug CLIPS_MACOS_BUNDLE_ID='com.clips.tray.alpha' node ./scripts/sign-macos-local.ts", "tauri": "tauri", "vite:dev": "vite", "vite:build": "vite build", diff --git a/templates/clips/desktop/scripts/sign-macos-local.ts b/templates/clips/desktop/scripts/sign-macos-local.ts index cbd8debba5..d280aa6ffc 100644 --- a/templates/clips/desktop/scripts/sign-macos-local.ts +++ b/templates/clips/desktop/scripts/sign-macos-local.ts @@ -12,11 +12,14 @@ if (process.platform !== "darwin") { const scriptDir = dirname(fileURLToPath(import.meta.url)); const desktopDir = resolve(scriptDir, ".."); +const appName = process.env.CLIPS_MACOS_APP_NAME || "Clips"; +const buildProfile = process.env.CLIPS_MACOS_BUILD_PROFILE || "release"; const appPath = resolve( desktopDir, - "src-tauri/target/release/bundle/macos/Clips.app", + `src-tauri/target/${buildProfile}/bundle/macos/${appName}.app`, ); const entitlementsPath = resolve(desktopDir, "src-tauri/Entitlements.plist"); +const stableAdHocBundleId = process.env.CLIPS_MACOS_BUNDLE_ID?.trim(); if (!existsSync(appPath)) { console.warn(`[clips-desktop] No macOS app bundle found at ${appPath}`); @@ -24,6 +27,13 @@ if (!existsSync(appPath)) { } function sign(identity: string) { + const stableRequirement = + identity === "-" && stableAdHocBundleId + ? [ + "--requirements", + `=designated => identifier \"${stableAdHocBundleId.replaceAll('"', "")}\"`, + ] + : []; const result = spawnSync( "codesign", [ @@ -31,6 +41,7 @@ function sign(identity: string) { "--deep", "--sign", identity, + ...stableRequirement, "--options", "runtime", "--entitlements", diff --git a/templates/clips/desktop/src-tauri/Cargo.lock b/templates/clips/desktop/src-tauri/Cargo.lock index b6ef40d57f..5673c9b767 100644 --- a/templates/clips/desktop/src-tauri/Cargo.lock +++ b/templates/clips/desktop/src-tauri/Cargo.lock @@ -727,6 +727,7 @@ dependencies = [ "core-graphics", "libc", "objc2", + "objc2-app-kit", "objc2-audio-toolbox", "objc2-avf-audio", "objc2-core-audio", diff --git a/templates/clips/desktop/src-tauri/Cargo.toml b/templates/clips/desktop/src-tauri/Cargo.toml index c8847b596c..164e5fc0a7 100644 --- a/templates/clips/desktop/src-tauri/Cargo.toml +++ b/templates/clips/desktop/src-tauri/Cargo.toml @@ -77,6 +77,11 @@ libc = "0.2" core-foundation = "0.10" core-graphics = "0.25" objc2 = { version = "0.6", features = ["exception"] } +objc2-app-kit = { version = "0.3", default-features = false, features = [ + "std", + "NSEvent", + "block2", +] } # Grapheme-cluster-aware chunking for voice-dictation Unicode typing in # terminal apps, so a chunk boundary never splits a multi-scalar emoji/ # combining-mark sequence across two synthetic keyboard events. Already diff --git a/templates/clips/desktop/src-tauri/build.rs b/templates/clips/desktop/src-tauri/build.rs index 0a7d885c3d..6962aceb47 100644 --- a/templates/clips/desktop/src-tauri/build.rs +++ b/templates/clips/desktop/src-tauri/build.rs @@ -1,12 +1,78 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::Command; fn main() { emit_sentry_env_reruns(); + compile_screen_memory_ocr_helper(); add_swift_runtime_rpaths(); tauri_build::build() } +/// Build the tiny, macOS-only AVFoundation/Vision bridge used by local Screen +/// Memory OCR. Keeping this outside the Rust dependency graph avoids adding a +/// large Objective-C binding surface for a single, OS-provided capability. +fn compile_screen_memory_ocr_helper() { + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("macos") { + return; + } + + let source = Path::new("native/screen_memory_ocr.swift"); + println!("cargo:rerun-if-changed={}", source.display()); + + let out_dir = PathBuf::from(std::env::var("OUT_DIR").expect("Cargo sets OUT_DIR")); + let object = out_dir.join("screen_memory_ocr_helper.o"); + let archive = out_dir.join("libscreen_memory_ocr_helper.a"); + + // `tauri build --target universal-apple-darwin` invokes Cargo once per + // architecture. swiftc otherwise emits an object for the runner's host + // architecture, which makes the x86_64 link fail on arm64 CI runners. + let swift_arch = match std::env::var("CARGO_CFG_TARGET_ARCH").as_deref() { + Ok("aarch64") => "arm64", + Ok("x86_64") => "x86_64", + Ok(arch) => panic!("unsupported macOS OCR helper architecture: {arch}"), + Err(error) => panic!("Cargo target architecture is required: {error}"), + }; + let deployment_target = + std::env::var("MACOSX_DEPLOYMENT_TARGET").unwrap_or_else(|_| "13.0".to_string()); + let swift_target = format!("{swift_arch}-apple-macosx{deployment_target}"); + + let swift_status = Command::new("xcrun") + .args([ + "swiftc", + "-parse-as-library", + "-emit-object", + "-target", + &swift_target, + source.to_str().expect("UTF-8 source path"), + "-o", + object.to_str().expect("UTF-8 output path"), + ]) + .status() + .expect("Xcode's swiftc is required to build the macOS OCR helper"); + assert!(swift_status.success(), "failed to compile macOS OCR helper"); + + let archive_status = Command::new("ar") + .args([ + "crus", + archive.to_str().expect("UTF-8 archive path"), + object.to_str().expect("UTF-8 object path"), + ]) + .status() + .expect("ar is required to archive the macOS OCR helper"); + assert!( + archive_status.success(), + "failed to archive macOS OCR helper" + ); + + println!("cargo:rustc-link-search=native={}", out_dir.display()); + println!("cargo:rustc-link-lib=static=screen_memory_ocr_helper"); + // The helper is written in Swift, while the rest of the native desktop + // stack already carries Swift runtime rpaths through ScreenCaptureKit. + for library in ["swiftCore", "swiftFoundation", "swift_Concurrency"] { + println!("cargo:rustc-link-lib=dylib={library}"); + } +} + fn emit_sentry_env_reruns() { for name in [ "CLIPS_DESKTOP_SENTRY_DSN", @@ -47,6 +113,8 @@ fn add_swift_runtime_rpaths() { // CMTime symbols we touch via raw `msg_send!` / `extern "C"`. println!("cargo:rustc-link-lib=framework=AVFoundation"); println!("cargo:rustc-link-lib=framework=CoreMedia"); + println!("cargo:rustc-link-lib=framework=Vision"); + println!("cargo:rustc-link-lib=framework=CoreGraphics"); println!("cargo:rustc-link-lib=framework=IOKit"); // The screencapturekit crate builds a Swift bridge. Its build script adds diff --git a/templates/clips/desktop/src-tauri/native/screen_memory_ocr.swift b/templates/clips/desktop/src-tauri/native/screen_memory_ocr.swift new file mode 100644 index 0000000000..f38f212f25 --- /dev/null +++ b/templates/clips/desktop/src-tauri/native/screen_memory_ocr.swift @@ -0,0 +1,103 @@ +import AVFoundation +import CoreGraphics +import Darwin +import Foundation +import Vision + +private let maxFrames = 30 +private let maxTextBytes = 8 * 1_024 + +private func response(_ object: [String: Any]) -> UnsafeMutablePointer? { + guard JSONSerialization.isValidJSONObject(object), + let data = try? JSONSerialization.data(withJSONObject: object), + let json = String(data: data, encoding: .utf8) else { + return strdup("{\"ok\":false,\"error\":\"OCR response encoding failed\"}") + } + return strdup(json) +} + +private func milliseconds(_ time: CMTime) -> Int64 { + guard time.isNumeric else { return 0 } + return Int64((CMTimeGetSeconds(time) * 1_000).rounded()) +} + +private func boundedText(_ text: String) -> String { + guard text.utf8.count > maxTextBytes else { return text } + // Decoding the byte prefix replaces a partial final scalar safely rather + // than splitting Swift's Unicode storage at an arbitrary index. + return String(decoding: text.utf8.prefix(maxTextBytes), as: UTF8.self) +} + +/// Runs wholly on-device. The result is JSON allocated by `strdup`; Rust must +/// return it through `clips_screen_memory_ocr_free` exactly once. +@_cdecl("clips_screen_memory_ocr_json") +public func clipsScreenMemoryOcrJson( + _ videoPath: UnsafePointer?, + _ sampleIntervalSeconds: UInt64 +) -> UnsafeMutablePointer? { + guard let videoPath else { + return response(["ok": false, "error": "missing video path"]) + } + + let path = String(cString: videoPath) + let asset = AVURLAsset(url: URL(fileURLWithPath: path)) + let duration = asset.duration + guard duration.isNumeric, CMTimeGetSeconds(duration) >= 0 else { + return response(["ok": false, "error": "video duration unavailable"]) + } + + let requestedInterval = max(1, min(sampleIntervalSeconds, UInt64(Int.max))) + let generator = AVAssetImageGenerator(asset: asset) + generator.appliesPreferredTrackTransform = true + generator.requestedTimeToleranceBefore = .zero + generator.requestedTimeToleranceAfter = .zero + + let seconds = CMTimeGetSeconds(duration) + var requestedTimes: [CMTime] = [] + var second: Double = 0 + while second <= seconds && requestedTimes.count < maxFrames { + requestedTimes.append(CMTime(seconds: second, preferredTimescale: 1_000)) + second += Double(requestedInterval) + } + if requestedTimes.isEmpty { + requestedTimes.append(.zero) + } + + var frames: [[String: Any]] = [] + for requestedTime in requestedTimes { + autoreleasepool { + var actualTime = CMTime.zero + guard let image = try? generator.copyCGImage(at: requestedTime, actualTime: &actualTime) else { + return + } + + let request = VNRecognizeTextRequest() + request.recognitionLevel = .fast + request.usesLanguageCorrection = false + request.minimumTextHeight = 0.015 + let handler = VNImageRequestHandler(cgImage: image, options: [:]) + guard (try? handler.perform([request])) != nil else { return } + + let observations = request.results ?? [] + let candidates = observations.compactMap { $0.topCandidates(1).first } + let text = boundedText(candidates.map(\.string).joined(separator: "\n")) + guard !text.isEmpty else { return } + let confidence = candidates.map { Double($0.confidence) }.reduce(0, +) + / Double(max(candidates.count, 1)) + frames.append([ + "offsetMs": milliseconds(actualTime), + "text": text, + "confidence": confidence, + "width": image.width, + "height": image.height, + ]) + } + } + + return response(["ok": true, "frames": frames]) +} + +@_cdecl("clips_screen_memory_ocr_free") +public func clipsScreenMemoryOcrFree(_ responsePointer: UnsafeMutablePointer?) { + free(responsePointer) +} diff --git a/templates/clips/desktop/src-tauri/src/accessibility.rs b/templates/clips/desktop/src-tauri/src/accessibility.rs index fbe07e50b6..b1db4c683b 100644 --- a/templates/clips/desktop/src-tauri/src/accessibility.rs +++ b/templates/clips/desktop/src-tauri/src/accessibility.rs @@ -54,7 +54,120 @@ //! - AXValue can also be a CFNumber (sliders) or CFURL — we only return //! a String when it's a CFString; otherwise "". -use serde::Serialize; +use serde::{Deserialize, Serialize}; + +/// A deliberately small, retention-bound summary of the current accessibility +/// surface. This is evidence for local Rewind scenes, not a serialised AX tree. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct AccessibilityFingerprint { + #[serde(skip_serializing_if = "Option::is_none")] + pub document: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub focused: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub visible_labels: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct AccessibilitySemanticNode { + pub role: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub document: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub selected_text: Option, +} + +const MAX_AX_NODES: usize = 6; +const MAX_AX_DEPTH: usize = 1; +const MAX_AX_STRING_BYTES: usize = 160; +const MAX_AX_LABELS: usize = 8; +const MAX_AX_TOTAL_BYTES: usize = 1_200; + +/// Keep text useful for scene continuity without retaining arbitrary field +/// contents. Password-like controls and credential-shaped values are omitted. +fn safe_ax_text(value: Option, remaining: &mut usize) -> Option { + let value = value?; + let value = value.split_whitespace().collect::>().join(" "); + if value.is_empty() || looks_like_credential(&value) { + return None; + } + let limit = MAX_AX_STRING_BYTES.min(*remaining); + if limit == 0 { + return None; + } + let mut end = value.len().min(limit); + while end > 0 && !value.is_char_boundary(end) { + end -= 1; + } + let value = value[..end].to_string(); + if value.is_empty() { + return None; + } + *remaining = remaining.saturating_sub(value.len()); + Some(value) +} + +fn looks_like_credential(value: &str) -> bool { + let lower = value.to_ascii_lowercase(); + if [ + "password", + "passcode", + "secret=", + "client secret", + "api key", + "api_key", + "access token", + "auth token", + "bearer ", + ] + .iter() + .any(|marker| lower.contains(marker)) + { + return true; + } + let compact = value + .chars() + .filter(|c| !c.is_whitespace()) + .collect::(); + let tokenish = compact.len() >= 32 + && compact + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.')); + let jwtish = compact.matches('.').count() == 2 && compact.len() >= 32; + let private_key = value.contains("-----BEGIN") && value.contains("PRIVATE KEY-----"); + tokenish || jwtish || private_key +} + +fn role_is_secure(role: &str) -> bool { + let role = role.to_ascii_lowercase(); + role.contains("secure") || role.contains("password") +} + +fn safe_ax_url(url: String, remaining: &mut usize) -> Option { + let url = url.split(['?', '#']).next()?.to_string(); + // Never retain a URL that embeds credentials in its authority component. + let authority = url + .split_once("://")? + .1 + .split('/') + .next() + .unwrap_or_default(); + if authority.contains('@') { + return None; + } + if url.split('/').skip(3).any(looks_like_credential) { + return None; + } + safe_ax_text(Some(url), remaining) +} #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] @@ -122,9 +235,53 @@ pub async fn accessibility_request_permission() -> Result { } } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ax_text_is_bounded_and_budgeted() { + let mut remaining = 12; + let value = safe_ax_text(Some("a very long accessible title".into()), &mut remaining) + .expect("safe title"); + assert_eq!(value.len(), 12); + assert_eq!(remaining, 0); + assert!(safe_ax_text(Some("another title".into()), &mut remaining).is_none()); + } + + #[test] + fn credential_shaped_text_is_not_retained() { + let mut remaining = MAX_AX_TOTAL_BYTES; + assert!(safe_ax_text(Some("password: hunter2".into()), &mut remaining).is_none()); + assert!( + safe_ax_text(Some("API key: short-but-sensitive".into()), &mut remaining).is_none() + ); + assert!(safe_ax_text( + Some("eyJhbGciOiJIUzI1NiJ9.abcdefghijklmnopqrstuvwxyz.0123456789abcdef".into()), + &mut remaining + ) + .is_none()); + assert!(role_is_secure("AXSecureTextField")); + assert!(safe_ax_url( + "https://alice:secret@example.test/document".into(), + &mut remaining + ) + .is_none()); + assert!(safe_ax_url( + "https://example.test/session/abcdefghijklmnopqrstuvwxyz123456".into(), + &mut remaining + ) + .is_none()); + } +} + #[cfg(target_os = "macos")] pub(crate) mod macos { - use super::ActiveWindowContext; + use super::{ + role_is_secure, safe_ax_text, safe_ax_url, AccessibilityFingerprint, + AccessibilitySemanticNode, ActiveWindowContext, MAX_AX_DEPTH, MAX_AX_LABELS, MAX_AX_NODES, + MAX_AX_TOTAL_BYTES, + }; use core_foundation::array::CFArray; use core_foundation::base::{CFType, TCFType}; use core_foundation::dictionary::CFDictionary; @@ -140,6 +297,7 @@ pub(crate) mod macos { type CFTypeRef = *const c_void; type CFStringRef = *const c_void; type CFDictionaryRef = *const c_void; + type CFArrayRef = *const c_void; type CFBooleanRef = *const c_void; type CFAllocatorRef = *const c_void; type CFIndex = isize; @@ -166,6 +324,7 @@ pub(crate) mod macos { extern "C" { fn AXIsProcessTrustedWithOptions(options: CFDictionaryRef) -> Boolean; fn AXUIElementCreateSystemWide() -> AXUIElementRef; + fn AXUIElementCreateApplication(pid: i32) -> AXUIElementRef; fn AXUIElementCopyAttributeValue( element: AXUIElementRef, attribute: CFStringRef, @@ -194,6 +353,10 @@ pub(crate) mod macos { fn CFRelease(cf: CFTypeRef); fn CFGetTypeID(cf: CFTypeRef) -> CFTypeID; fn CFStringGetTypeID() -> CFTypeID; + fn CFURLGetTypeID() -> CFTypeID; + fn CFArrayGetCount(array: CFArrayRef) -> CFIndex; + fn CFArrayGetValueAtIndex(array: CFArrayRef, index: CFIndex) -> *const c_void; + fn CFURLGetString(url: CFTypeRef) -> CFStringRef; fn CFDictionaryCreate( allocator: CFAllocatorRef, keys: *const *const c_void, @@ -341,25 +504,214 @@ pub(crate) mod macos { } } + /// Collect the semantic surface from the same application selected by the + /// Core Graphics foreground-window sample. This avoids combining an + /// ignored Clips utility window's AX tree with the underlying work app. + pub fn semantic_fingerprint_for_pid_impl(pid: i32) -> Option { + unsafe { + if !is_trusted(false) { + return None; + } + let application = AXUIElementCreateApplication(pid); + if application.is_null() { + return None; + } + let window = copy_ax_element_attribute(application, "AXFocusedWindow"); + let focused = copy_ax_element_attribute(application, "AXFocusedUIElement"); + CFRelease(application as CFTypeRef); + + let document_element = window.or(focused)?; + let mut remaining = MAX_AX_TOTAL_BYTES; + let document = semantic_node(document_element, false, &mut remaining); + let focused_node = + focused.and_then(|element| semantic_node(element, true, &mut remaining)); + let mut labels = Vec::new(); + collect_visible_labels(document_element, 0, &mut labels, &mut remaining); + + if let Some(window) = window { + CFRelease(window); + } + if let Some(focused) = focused { + CFRelease(focused); + } + + if document.is_none() && focused_node.is_none() && labels.is_empty() { + None + } else { + Some(AccessibilityFingerprint { + document, + focused: focused_node, + visible_labels: labels, + }) + } + } + } + + unsafe fn copy_ax_element_attribute( + element: AXUIElementRef, + attribute: &str, + ) -> Option { + let attribute = cfstr(attribute); + if attribute.is_null() { + return None; + } + let mut value = ptr::null(); + let error = AXUIElementCopyAttributeValue(element, attribute, &mut value); + CFRelease(attribute as CFTypeRef); + (error == AX_ERROR_SUCCESS && !value.is_null()).then_some(value as AXUIElementRef) + } + + unsafe fn ax_string_attribute(element: AXUIElementRef, attribute: &str) -> Option { + let attribute = cfstr(attribute); + if attribute.is_null() { + return None; + } + let mut raw_value = ptr::null(); + let error = AXUIElementCopyAttributeValue(element, attribute, &mut raw_value); + CFRelease(attribute as CFTypeRef); + if error != AX_ERROR_SUCCESS || raw_value.is_null() { + return None; + } + let value = if CFGetTypeID(raw_value) == CFStringGetTypeID() { + cfstring_to_string(raw_value as CFStringRef) + } else { + None + }; + CFRelease(raw_value); + value + } + + unsafe fn ax_url_attribute(element: AXUIElementRef) -> Option { + let attribute = cfstr("AXURL"); + if attribute.is_null() { + return None; + } + let mut value = ptr::null(); + let error = AXUIElementCopyAttributeValue(element, attribute, &mut value); + CFRelease(attribute as CFTypeRef); + if error != AX_ERROR_SUCCESS || value.is_null() { + return None; + } + // URLs may contain query-string credentials. Keep only the stable + // origin/path portion, and omit anything that cannot be read safely. + let string = if CFGetTypeID(value) == CFStringGetTypeID() { + value as CFStringRef + } else if CFGetTypeID(value) == CFURLGetTypeID() { + CFURLGetString(value) + } else { + ptr::null() + }; + let result = cfstring_to_string(string) + .and_then(|url| url.split(['?', '#']).next().map(str::to_string)); + CFRelease(value); + result + } + + unsafe fn semantic_node( + element: AXUIElementRef, + focused: bool, + remaining: &mut usize, + ) -> Option { + let role = ax_string_attribute(element, "AXRole")?; + if role_is_secure(&role) { + return None; + } + let role = safe_ax_text(Some(role), remaining)?; + let title = safe_ax_text(ax_string_attribute(element, "AXTitle"), remaining); + let description = safe_ax_text(ax_string_attribute(element, "AXDescription"), remaining); + let document = safe_ax_text(ax_string_attribute(element, "AXDocument"), remaining); + let url = ax_url_attribute(element).and_then(|url| safe_ax_url(url, remaining)); + let selected_text = focused + .then(|| ax_string_attribute(element, "AXSelectedText")) + .flatten() + .and_then(|text| safe_ax_text(Some(text), remaining)); + Some(AccessibilitySemanticNode { + role, + title, + description, + document, + url, + selected_text, + }) + } + + unsafe fn collect_visible_labels( + element: AXUIElementRef, + depth: usize, + labels: &mut Vec, + remaining: &mut usize, + ) { + if depth >= MAX_AX_DEPTH || labels.len() >= MAX_AX_LABELS || *remaining == 0 { + return; + } + let attribute = cfstr("AXChildren"); + if attribute.is_null() { + return; + } + let mut children = ptr::null(); + let error = AXUIElementCopyAttributeValue(element, attribute, &mut children); + CFRelease(attribute as CFTypeRef); + if error != AX_ERROR_SUCCESS || children.is_null() { + return; + } + let count = (CFArrayGetCount(children) as usize).min(MAX_AX_NODES); + for index in 0..count { + if labels.len() >= MAX_AX_LABELS || *remaining == 0 { + break; + } + let child = CFArrayGetValueAtIndex(children, index as CFIndex) as AXUIElementRef; + if child.is_null() { + continue; + } + let role = ax_string_attribute(child, "AXRole"); + if role.as_deref().is_some_and(role_is_secure) { + continue; + } + let label = ax_string_attribute(child, "AXTitle") + .or_else(|| ax_string_attribute(child, "AXDescription")) + .and_then(|text| safe_ax_text(Some(text), remaining)); + if let Some(label) = label { + labels.push(label); + } + } + CFRelease(children); + } + pub fn active_window_context_impl() -> ActiveWindowContext { + active_window_context_with_pid_impl().0 + } + + pub(crate) fn active_window_context_with_pid_impl() -> (ActiveWindowContext, Option) { let window = active_window_from_core_graphics(); let frontmost = frontmost_application(); - - ActiveWindowContext { - app_name: window - .as_ref() - .and_then(|w| w.app_name.clone()) - .or_else(|| frontmost.as_ref().and_then(|app| app.app_name.clone())), - window_title: window.and_then(|w| w.window_title), - bundle_id: frontmost.and_then(|app| app.bundle_id), - source: "core-graphics".to_string(), - } + let owner = window + .as_ref() + .and_then(|window| window.owner_pid) + .and_then(application_for_pid); + let owner_pid = window.as_ref().and_then(|window| window.owner_pid); + + ( + ActiveWindowContext { + app_name: window + .as_ref() + .and_then(|w| w.app_name.clone()) + .or_else(|| owner.as_ref().and_then(|app| app.app_name.clone())) + .or_else(|| frontmost.as_ref().and_then(|app| app.app_name.clone())), + window_title: window.and_then(|w| w.window_title), + bundle_id: owner + .and_then(|app| app.bundle_id) + .or_else(|| frontmost.and_then(|app| app.bundle_id)), + source: "core-graphics".to_string(), + }, + owner_pid, + ) } #[derive(Debug, Clone)] struct WindowInfo { app_name: Option, window_title: Option, + owner_pid: Option, } #[derive(Debug, Clone)] @@ -400,6 +752,8 @@ pub(crate) mod macos { return Some(WindowInfo { app_name: owner, window_title: title, + owner_pid: dict_number_i64(&window, "kCGWindowOwnerPID") + .and_then(|pid| i32::try_from(pid).ok()), }); } } @@ -430,6 +784,26 @@ pub(crate) mod macos { } } + fn application_for_pid(pid: i32) -> Option { + use objc2::msg_send; + use objc2::runtime::{AnyClass, AnyObject}; + + unsafe { + let class_name = std::ffi::CString::new("NSRunningApplication").ok()?; + let cls: &AnyClass = AnyClass::get(&class_name)?; + let app: *mut AnyObject = msg_send![cls, runningApplicationWithProcessIdentifier: pid]; + if app.is_null() { + return None; + } + let app_name: *mut AnyObject = msg_send![app, localizedName]; + let bundle_id: *mut AnyObject = msg_send![app, bundleIdentifier]; + Some(AppInfo { + app_name: ns_string_to_owned(app_name), + bundle_id: ns_string_to_owned(bundle_id), + }) + } + } + fn dict_string(dict: &CFDictionary, key: &'static str) -> Option { let key = CFString::from_static_string(key); let value = dict.find(&key)?; diff --git a/templates/clips/desktop/src-tauri/src/capture_audio_bus.rs b/templates/clips/desktop/src-tauri/src/capture_audio_bus.rs new file mode 100644 index 0000000000..6af4600d0e --- /dev/null +++ b/templates/clips/desktop/src-tauri/src/capture_audio_bus.rs @@ -0,0 +1,382 @@ +//! In-process fan-out for PCM already produced by the Rewind ScreenCaptureKit stream. +//! +//! The bus deliberately owns no hardware. A single producer advertises the +//! sources it has opened and publishes mono f32 buffers; consumers either +//! subscribe to that producer or, when none exists, may open their legacy +//! physical capture path. An existing producer that lacks a requested source +//! is a hard error, never permission to start a second recorder. + +use std::collections::BTreeMap; +use std::fmt; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; + +pub(crate) type AudioCallback = Arc; + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct AudioSources { + pub microphone: bool, + pub system: bool, +} + +impl AudioSources { + pub(crate) const fn new(microphone: bool, system: bool) -> Self { + Self { microphone, system } + } + + fn contains(self, requested: Self) -> bool { + (!requested.microphone || self.microphone) && (!requested.system || self.system) + } + + fn missing_names(self, requested: Self) -> Vec<&'static str> { + let mut missing = Vec::new(); + if requested.microphone && !self.microphone { + missing.push("microphone"); + } + if requested.system && !self.system { + missing.push("system"); + } + missing + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum CapturePlan { + OpenPhysicalCapture, + SubscribeToSharedProducer, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum SubscribeError { + ProducerLacksSources { + available: AudioSources, + requested: AudioSources, + }, +} + +impl fmt::Display for SubscribeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::ProducerLacksSources { + available, + requested, + } => write!( + formatter, + "shared-audio-producer-lacks-sources:{}", + available.missing_names(*requested).join(",") + ), + } + } +} + +impl std::error::Error for SubscribeError {} + +pub(crate) fn capture_plan( + available: Option, + requested: AudioSources, +) -> Result { + match available { + None => Ok(CapturePlan::OpenPhysicalCapture), + Some(available) if available.contains(requested) => { + Ok(CapturePlan::SubscribeToSharedProducer) + } + Some(available) => Err(SubscribeError::ProducerLacksSources { + available, + requested, + }), + } +} + +struct Subscriber { + active: Arc, + microphone: Option, + system: Option, +} + +struct ProducerState { + id: u64, + sources: AudioSources, + subscribers: BTreeMap, +} + +#[derive(Default)] +struct BusState { + next_id: u64, + producer: Option, +} + +fn bus() -> &'static Mutex { + static BUS: OnceLock> = OnceLock::new(); + BUS.get_or_init(|| Mutex::new(BusState::default())) +} + +struct ProducerRegistration { + id: u64, + active: AtomicBool, +} + +impl ProducerRegistration { + fn deactivate(&self) { + if !self.active.swap(false, Ordering::SeqCst) { + return; + } + if let Ok(mut state) = bus().lock() { + if state + .producer + .as_ref() + .is_some_and(|producer| producer.id == self.id) + { + if let Some(producer) = state.producer.take() { + for subscriber in producer.subscribers.into_values() { + subscriber.active.store(false, Ordering::SeqCst); + } + } + } + } + } +} + +impl Drop for ProducerRegistration { + fn drop(&mut self) { + self.deactivate(); + } +} + +/// Cloneable producer handle. The registration remains live through stream +/// rebuilds until explicitly deactivated or the final clone is dropped. +#[derive(Clone)] +pub(crate) struct AudioProducer { + registration: Arc, +} + +impl AudioProducer { + pub(crate) fn register(sources: AudioSources) -> Result { + let mut state = bus().lock().map_err(|error| error.to_string())?; + if state.producer.is_some() { + return Err("shared-audio-producer-already-active".into()); + } + state.next_id = state.next_id.saturating_add(1); + let id = state.next_id; + state.producer = Some(ProducerState { + id, + sources, + subscribers: BTreeMap::new(), + }); + Ok(Self { + registration: Arc::new(ProducerRegistration { + id, + active: AtomicBool::new(true), + }), + }) + } + + pub(crate) fn deactivate(&self) { + self.registration.deactivate(); + } + + pub(crate) fn publish_microphone(&self, samples: &[f32], sample_rate: f64) { + self.publish(samples, sample_rate, true); + } + + pub(crate) fn publish_system(&self, samples: &[f32], sample_rate: f64) { + self.publish(samples, sample_rate, false); + } + + fn publish(&self, samples: &[f32], sample_rate: f64, microphone: bool) { + if samples.is_empty() || !self.registration.active.load(Ordering::SeqCst) { + return; + } + let callbacks: Vec<(Arc, AudioCallback)> = bus() + .lock() + .ok() + .and_then(|state| { + state.producer.as_ref().and_then(|producer| { + (producer.id == self.registration.id).then(|| { + producer + .subscribers + .values() + .filter_map(|subscriber| { + let callback = if microphone { + subscriber.microphone.as_ref() + } else { + subscriber.system.as_ref() + }?; + Some((subscriber.active.clone(), callback.clone())) + }) + .collect() + }) + }) + }) + .unwrap_or_default(); + // Never hold the bus lock while invoking client code. A callback may + // stop its session and drop its own subscription. + for (active, callback) in callbacks { + if active.load(Ordering::SeqCst) { + callback(samples, sample_rate); + } + } + } +} + +pub(crate) enum SubscriptionAttempt { + NoProducer, + Subscribed(AudioSubscription), +} + +pub(crate) struct AudioSubscription { + producer_id: u64, + subscriber_id: u64, + active: Arc, +} + +impl Drop for AudioSubscription { + fn drop(&mut self) { + self.active.store(false, Ordering::SeqCst); + if let Ok(mut state) = bus().lock() { + if let Some(producer) = state.producer.as_mut() { + if producer.id == self.producer_id { + producer.subscribers.remove(&self.subscriber_id); + } + } + } + } +} + +pub(crate) fn try_subscribe( + requested: AudioSources, + microphone: Option, + system: Option, +) -> Result { + let mut state = bus().lock().expect("capture audio bus lock poisoned"); + let available = state.producer.as_ref().map(|producer| producer.sources); + match capture_plan(available, requested)? { + CapturePlan::OpenPhysicalCapture => Ok(SubscriptionAttempt::NoProducer), + CapturePlan::SubscribeToSharedProducer => { + state.next_id = state.next_id.saturating_add(1); + let subscriber_id = state.next_id; + let producer = state + .producer + .as_mut() + .expect("capture plan observed an active producer"); + let producer_id = producer.id; + let active = Arc::new(AtomicBool::new(true)); + producer.subscribers.insert( + subscriber_id, + Subscriber { + active: active.clone(), + microphone, + system, + }, + ); + Ok(SubscriptionAttempt::Subscribed(AudioSubscription { + producer_id, + subscriber_id, + active, + })) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::MutexGuard; + + fn test_guard() -> MutexGuard<'static, ()> { + static TEST_LOCK: OnceLock> = OnceLock::new(); + TEST_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + fn reset() { + *bus().lock().unwrap() = BusState::default(); + } + + #[test] + fn routes_sources_to_independent_subscribers() { + let _guard = test_guard(); + reset(); + let producer = AudioProducer::register(AudioSources::new(true, true)).unwrap(); + let mic_frames = Arc::new(AtomicUsize::new(0)); + let sys_frames = Arc::new(AtomicUsize::new(0)); + let mic_seen = mic_frames.clone(); + let sys_seen = sys_frames.clone(); + let _subscription = match try_subscribe( + AudioSources::new(true, true), + Some(Arc::new(move |samples, rate| { + assert_eq!(rate, 44_100.0); + mic_seen.fetch_add(samples.len(), Ordering::SeqCst); + })), + Some(Arc::new(move |samples, rate| { + assert_eq!(rate, 48_000.0); + sys_seen.fetch_add(samples.len(), Ordering::SeqCst); + })), + ) + .unwrap() + { + SubscriptionAttempt::Subscribed(subscription) => subscription, + SubscriptionAttempt::NoProducer => panic!("producer disappeared"), + }; + producer.publish_microphone(&[0.1, 0.2], 44_100.0); + producer.publish_system(&[0.3, 0.4, 0.5], 48_000.0); + assert_eq!(mic_frames.load(Ordering::SeqCst), 2); + assert_eq!(sys_frames.load(Ordering::SeqCst), 3); + } + + #[test] + fn clone_refcount_and_subscription_drop_keep_teardown_safe() { + let _guard = test_guard(); + reset(); + let producer = AudioProducer::register(AudioSources::new(true, false)).unwrap(); + let producer_clone = producer.clone(); + drop(producer); + assert_eq!( + capture_plan( + bus().lock().unwrap().producer.as_ref().map(|p| p.sources), + AudioSources::new(true, false) + ), + Ok(CapturePlan::SubscribeToSharedProducer) + ); + + let calls = Arc::new(AtomicUsize::new(0)); + let seen = calls.clone(); + let subscription = match try_subscribe( + AudioSources::new(true, false), + Some(Arc::new(move |_, _| { + seen.fetch_add(1, Ordering::SeqCst); + })), + None, + ) + .unwrap() + { + SubscriptionAttempt::Subscribed(subscription) => subscription, + SubscriptionAttempt::NoProducer => panic!("producer disappeared"), + }; + producer_clone.publish_microphone(&[0.1], 48_000.0); + drop(subscription); + producer_clone.publish_microphone(&[0.2], 48_000.0); + assert_eq!(calls.load(Ordering::SeqCst), 1); + + drop(producer_clone); + assert!(bus().lock().unwrap().producer.is_none()); + } + + #[test] + fn existing_producer_without_sources_fails_closed() { + let _guard = test_guard(); + reset(); + assert_eq!( + capture_plan( + Some(AudioSources::new(true, false)), + AudioSources::new(true, true) + ), + Err(SubscribeError::ProducerLacksSources { + available: AudioSources::new(true, false), + requested: AudioSources::new(true, true), + }) + ); + } +} diff --git a/templates/clips/desktop/src-tauri/src/capture_graph.rs b/templates/clips/desktop/src-tauri/src/capture_graph.rs new file mode 100644 index 0000000000..097ba5ed40 --- /dev/null +++ b/templates/clips/desktop/src-tauri/src/capture_graph.rs @@ -0,0 +1,777 @@ +//! A local coordinator for capture demand. +//! +//! This module intentionally does not start a screen, audio, or camera producer. +//! It is the small piece of shared state those producers will eventually obey: a +//! single monotonic graph clock, ref-counted source demand, and auditable leases. + +use chrono::{DateTime, Duration as ChronoDuration, Utc}; +use serde::Serialize; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +/// The largest interval a caller may explicitly request from a prior buffer. +pub(crate) const MAX_RETROSPECTIVE_EXTENSION: Duration = Duration::from_secs(5 * 60); + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum CaptureSource { + Screen, + SystemAudio, + Microphone, + Camera, +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum CaptureConsumer { + Rewind, + Clip, + Meeting, + Transcription, + VisualIndex, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum CoverageGapReason { + PermissionDenied, + ProducerUnavailable, + UserPaused, + PrivacyExclusion, + SystemInterrupted, + Unknown, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CaptureInterval { + pub started_at: String, + pub ended_at: String, + pub started_elapsed_ms: u64, + pub ended_elapsed_ms: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ConsumerLeaseSnapshot { + pub id: String, + pub consumer: CaptureConsumer, + pub sources: Vec, + pub interval: CaptureInterval, + pub retrospective_extension_ms: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ClosedConsumerLease { + pub lease: ConsumerLeaseSnapshot, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PinRecord { + pub id: String, + pub interval: CaptureInterval, + pub label: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CoverageGap { + pub id: String, + pub source: CaptureSource, + pub reason: CoverageGapReason, + pub interval: CaptureInterval, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ActiveSourceStatus { + pub source: CaptureSource, + pub consumer_count: usize, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CaptureGraphStatus { + pub graph_started_at: String, + pub graph_elapsed_ms: u64, + pub active_sources: Vec, + pub active_consumers: Vec, + pub coverage_gaps: Vec, + pub pins: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum CaptureGraphError { + UnknownLease(String), + DuplicateLease(String), + EmptySources, + InvalidInterval, + BeforeGraphStart, + RetrospectiveExtensionTooLarge { + requested: Duration, + maximum: Duration, + }, + RetrospectiveExtensionAlreadySet(String), +} + +impl fmt::Display for CaptureGraphError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::UnknownLease(id) => write!(formatter, "unknown capture lease {id}"), + Self::DuplicateLease(id) => write!(formatter, "capture lease {id} already exists"), + Self::EmptySources => write!( + formatter, + "a capture lease must request at least one source" + ), + Self::InvalidInterval => write!(formatter, "capture interval ends before it starts"), + Self::BeforeGraphStart => write!(formatter, "capture instant predates the graph clock"), + Self::RetrospectiveExtensionTooLarge { requested, maximum } => write!( + formatter, + "retrospective extension of {}ms exceeds {}ms", + requested.as_millis(), + maximum.as_millis() + ), + Self::RetrospectiveExtensionAlreadySet(id) => { + write!( + formatter, + "capture lease {id} already has a retrospective extension" + ) + } + } + } +} + +impl std::error::Error for CaptureGraphError {} + +#[derive(Debug)] +struct Lease { + id: String, + consumer: CaptureConsumer, + sources: BTreeSet, + started_at: Instant, + retrospective_extension: Option, +} + +/// In-memory source-demand graph. `Instant` is authoritative; wall clock values +/// are produced only at the serializable boundary. +#[derive(Debug)] +pub(crate) struct CaptureGraph { + clock_started_at: Instant, + epoch_started_at: DateTime, + active_leases: BTreeMap, + source_demand: BTreeMap, + coverage_gaps: Vec, + pins: Vec, + next_lease_id: u64, + next_gap_id: u64, + next_pin_id: u64, +} + +/// Tauri-managed holder. No command consumes this yet; keeping it managed makes +/// later producer integration share one coordinator rather than inventing a +/// second clock in each subsystem. +pub(crate) struct CaptureGraphState(pub Mutex); + +impl Default for CaptureGraphState { + fn default() -> Self { + Self(Mutex::new(CaptureGraph::new())) + } +} + +impl CaptureGraph { + pub(crate) fn new() -> Self { + Self::new_at(Instant::now(), Utc::now()) + } + + /// Stable identity for this in-memory monotonic clock. Persisted media may + /// compare elapsed offsets only when this identity also matches; elapsed + /// milliseconds restart from zero whenever the desktop app relaunches. + pub(crate) fn epoch_id(&self) -> String { + format_rfc3339_millis(self.epoch_started_at) + } + + fn new_at(clock_started_at: Instant, epoch_started_at: DateTime) -> Self { + Self { + clock_started_at, + epoch_started_at, + active_leases: BTreeMap::new(), + source_demand: BTreeMap::new(), + coverage_gaps: Vec::new(), + pins: Vec::new(), + next_lease_id: 1, + next_gap_id: 1, + next_pin_id: 1, + } + } + + /// Starts demand at `now`, never before it. Countdown handling belongs to a + /// caller: it must invoke this only when the countdown has completed. + pub(crate) fn start_consumer( + &mut self, + consumer: CaptureConsumer, + sources: impl IntoIterator, + ) -> Result { + self.start_consumer_at(consumer, sources, Instant::now()) + } + + pub(crate) fn start_consumer_at( + &mut self, + consumer: CaptureConsumer, + sources: impl IntoIterator, + now: Instant, + ) -> Result { + self.require_graph_instant(now)?; + let sources: BTreeSet<_> = sources.into_iter().collect(); + if sources.is_empty() { + return Err(CaptureGraphError::EmptySources); + } + let id = format!("lease-{}", self.next_lease_id); + self.next_lease_id += 1; + self.start_consumer_with_id_at(id, consumer, sources, now) + } + + fn start_consumer_with_id_at( + &mut self, + id: String, + consumer: CaptureConsumer, + sources: BTreeSet, + now: Instant, + ) -> Result { + if self.active_leases.contains_key(&id) { + return Err(CaptureGraphError::DuplicateLease(id)); + } + for source in &sources { + *self.source_demand.entry(*source).or_default() += 1; + } + let lease = Lease { + id: id.clone(), + consumer, + sources, + started_at: now, + retrospective_extension: None, + }; + let snapshot = self.open_lease_snapshot(&lease, now)?; + self.active_leases.insert(id, lease); + Ok(snapshot) + } + + /// Explicitly asks to include bounded buffered history. Starting a lease + /// never calls this implicitly; actual producer coverage remains separate. + pub(crate) fn extend_retrospectively( + &mut self, + lease_id: &str, + duration: Duration, + ) -> Result { + if duration > MAX_RETROSPECTIVE_EXTENSION { + return Err(CaptureGraphError::RetrospectiveExtensionTooLarge { + requested: duration, + maximum: MAX_RETROSPECTIVE_EXTENSION, + }); + } + { + let lease = self + .active_leases + .get_mut(lease_id) + .ok_or_else(|| CaptureGraphError::UnknownLease(lease_id.to_owned()))?; + if lease.retrospective_extension.is_some() { + return Err(CaptureGraphError::RetrospectiveExtensionAlreadySet( + lease_id.to_owned(), + )); + } + // Clamp at the graph epoch: a lease may not request time before its + // only reliable clock anchor. + let available = lease.started_at.duration_since(self.clock_started_at); + lease.retrospective_extension = Some(duration.min(available)); + } + let lease = self + .active_leases + .get(lease_id) + .expect("lease just updated"); + self.open_lease_snapshot(lease, lease.started_at) + } + + /// Explicitly extends a lease back to an absolute requested instant, such + /// as a meeting's recorded start. Unlike `extend_retrospectively`, this is + /// not subject to the short Rewind duration cap; it is clamped only to the + /// graph clock because the coordinator cannot describe coverage before it. + pub(crate) fn extend_retrospectively_to_start( + &mut self, + lease_id: &str, + requested_start: Instant, + ) -> Result { + let requested_start = requested_start.max(self.clock_started_at); + { + let lease = self + .active_leases + .get_mut(lease_id) + .ok_or_else(|| CaptureGraphError::UnknownLease(lease_id.to_owned()))?; + if lease.retrospective_extension.is_some() { + return Err(CaptureGraphError::RetrospectiveExtensionAlreadySet( + lease_id.to_owned(), + )); + } + let extension = lease.started_at.saturating_duration_since(requested_start); + lease.retrospective_extension = Some(extension); + } + let lease = self + .active_leases + .get(lease_id) + .expect("lease just updated"); + self.open_lease_snapshot(lease, lease.started_at) + } + + pub(crate) fn end_consumer( + &mut self, + lease_id: &str, + ) -> Result { + self.end_consumer_at(lease_id, Instant::now()) + } + + /// Ends a lease and returns its closed interval. This is the sole path that + /// releases its ref-counted producer demand. + pub(crate) fn end_consumer_at( + &mut self, + lease_id: &str, + now: Instant, + ) -> Result { + self.require_graph_instant(now)?; + let lease = self + .active_leases + .remove(lease_id) + .ok_or_else(|| CaptureGraphError::UnknownLease(lease_id.to_owned()))?; + if now < lease.started_at { + self.active_leases.insert(lease.id.clone(), lease); + return Err(CaptureGraphError::InvalidInterval); + } + for source in &lease.sources { + let no_remaining_demand = { + let demand = self + .source_demand + .get_mut(source) + .expect("lease demand exists"); + *demand -= 1; + *demand == 0 + }; + if no_remaining_demand { + self.source_demand.remove(source); + } + } + Ok(ClosedConsumerLease { + lease: self.closed_lease_snapshot(&lease, now)?, + }) + } + + pub(crate) fn record_coverage_gap_at( + &mut self, + source: CaptureSource, + reason: CoverageGapReason, + started_at: Instant, + ended_at: Instant, + ) -> Result { + let interval = self.interval(started_at, ended_at)?; + let gap = CoverageGap { + id: format!("gap-{}", self.next_gap_id), + source, + reason, + interval, + }; + self.next_gap_id += 1; + self.coverage_gaps.push(gap.clone()); + Ok(gap) + } + + pub(crate) fn pin_interval_at( + &mut self, + started_at: Instant, + ended_at: Instant, + label: Option, + ) -> Result { + let pin = PinRecord { + id: format!("pin-{}", self.next_pin_id), + interval: self.interval(started_at, ended_at)?, + label, + }; + self.next_pin_id += 1; + self.pins.push(pin.clone()); + Ok(pin) + } + + /// Pins an already-serialized graph interval without reconstructing it + /// through wall-clock time. This is the normal bridge from a closed + /// consumer lease to retention-aware media materialization. + pub(crate) fn pin_interval( + &mut self, + interval: &CaptureInterval, + label: Option, + ) -> Result { + let started_at = self + .clock_started_at + .checked_add(Duration::from_millis(interval.started_elapsed_ms)) + .ok_or(CaptureGraphError::InvalidInterval)?; + let ended_at = self + .clock_started_at + .checked_add(Duration::from_millis(interval.ended_elapsed_ms)) + .ok_or(CaptureGraphError::InvalidInterval)?; + self.pin_interval_at(started_at, ended_at, label) + } + + /// Releases a retention pin after its bounded artifact has been copied to + /// durable pending storage. Releasing an unknown pin is deliberately a + /// no-op so cancellation and error cleanup can be idempotent. + pub(crate) fn release_pin(&mut self, pin_id: &str) -> bool { + let before = self.pins.len(); + self.pins.retain(|pin| pin.id != pin_id); + self.pins.len() != before + } + + pub(crate) fn status_at(&self, now: Instant) -> Result { + self.require_graph_instant(now)?; + Ok(CaptureGraphStatus { + graph_started_at: format_rfc3339_millis(self.epoch_started_at), + graph_elapsed_ms: self.elapsed_ms(now)?, + active_sources: self + .source_demand + .iter() + .map(|(source, consumer_count)| ActiveSourceStatus { + source: *source, + consumer_count: *consumer_count, + }) + .collect(), + active_consumers: self + .active_leases + .values() + .map(|lease| self.open_lease_snapshot(lease, now)) + .collect::, _>>()?, + coverage_gaps: self.coverage_gaps.clone(), + pins: self.pins.clone(), + }) + } + + fn open_lease_snapshot( + &self, + lease: &Lease, + now: Instant, + ) -> Result { + let started_at = self.effective_start(lease)?; + Ok(ConsumerLeaseSnapshot { + id: lease.id.clone(), + consumer: lease.consumer, + sources: lease.sources.iter().copied().collect(), + interval: self.interval(started_at, now)?, + retrospective_extension_ms: lease + .retrospective_extension + .map(|duration| duration.as_millis() as u64) + .unwrap_or_default(), + }) + } + + fn closed_lease_snapshot( + &self, + lease: &Lease, + ended_at: Instant, + ) -> Result { + let started_at = self.effective_start(lease)?; + Ok(ConsumerLeaseSnapshot { + id: lease.id.clone(), + consumer: lease.consumer, + sources: lease.sources.iter().copied().collect(), + interval: self.interval(started_at, ended_at)?, + retrospective_extension_ms: lease + .retrospective_extension + .map(|duration| duration.as_millis() as u64) + .unwrap_or_default(), + }) + } + + fn effective_start(&self, lease: &Lease) -> Result { + Ok(lease.started_at - lease.retrospective_extension.unwrap_or_default()) + } + + fn interval( + &self, + started_at: Instant, + ended_at: Instant, + ) -> Result { + self.require_graph_instant(started_at)?; + self.require_graph_instant(ended_at)?; + if ended_at < started_at { + return Err(CaptureGraphError::InvalidInterval); + } + Ok(CaptureInterval { + started_at: self.timestamp(started_at)?, + ended_at: self.timestamp(ended_at)?, + started_elapsed_ms: self.elapsed_ms(started_at)?, + ended_elapsed_ms: self.elapsed_ms(ended_at)?, + }) + } + + fn require_graph_instant(&self, instant: Instant) -> Result<(), CaptureGraphError> { + if instant < self.clock_started_at { + Err(CaptureGraphError::BeforeGraphStart) + } else { + Ok(()) + } + } + + fn elapsed_ms(&self, instant: Instant) -> Result { + self.require_graph_instant(instant)?; + Ok(instant.duration_since(self.clock_started_at).as_millis() as u64) + } + + fn timestamp(&self, instant: Instant) -> Result { + let elapsed = self.elapsed_ms(instant)?; + Ok(format_rfc3339_millis( + self.epoch_started_at + ChronoDuration::milliseconds(elapsed as i64), + )) + } +} + +fn format_rfc3339_millis(timestamp: DateTime) -> String { + timestamp.to_rfc3339_opts(chrono::SecondsFormat::Millis, true) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn graph() -> (CaptureGraph, Instant) { + let start = Instant::now(); + ( + CaptureGraph::new_at( + start, + DateTime::parse_from_rfc3339("2026-07-14T12:00:00Z") + .unwrap() + .with_timezone(&Utc), + ), + start, + ) + } + + #[test] + fn refcounts_shared_source_demand() { + let (mut graph, start) = graph(); + let first = graph + .start_consumer_at(CaptureConsumer::Rewind, [CaptureSource::Screen], start) + .unwrap(); + graph + .start_consumer_at( + CaptureConsumer::VisualIndex, + [CaptureSource::Screen, CaptureSource::SystemAudio], + start + Duration::from_millis(10), + ) + .unwrap(); + let status = graph.status_at(start + Duration::from_millis(10)).unwrap(); + assert_eq!(status.active_sources[0].consumer_count, 2); + assert_eq!(status.active_sources[1].consumer_count, 1); + graph + .end_consumer_at(&first.id, start + Duration::from_millis(20)) + .unwrap(); + let status = graph.status_at(start + Duration::from_millis(20)).unwrap(); + assert_eq!( + status.active_sources, + vec![ + ActiveSourceStatus { + source: CaptureSource::Screen, + consumer_count: 1 + }, + ActiveSourceStatus { + source: CaptureSource::SystemAudio, + consumer_count: 1 + } + ] + ); + } + + #[test] + fn countdown_completion_starts_without_pre_roll() { + let (mut graph, start) = graph(); + let countdown_completed = start + Duration::from_secs(3); + let lease = graph + .start_consumer_at( + CaptureConsumer::Clip, + [CaptureSource::Screen], + countdown_completed, + ) + .unwrap(); + assert_eq!(lease.interval.started_elapsed_ms, 3_000); + assert_eq!(lease.retrospective_extension_ms, 0); + } + + #[test] + fn retrospective_extension_is_explicit_and_bounded() { + let (mut graph, start) = graph(); + let lease = graph + .start_consumer_at( + CaptureConsumer::Rewind, + [CaptureSource::Screen], + start + Duration::from_secs(10), + ) + .unwrap(); + let extended = graph + .extend_retrospectively(&lease.id, Duration::from_secs(4)) + .unwrap(); + assert_eq!(extended.interval.started_elapsed_ms, 6_000); + assert_eq!(extended.retrospective_extension_ms, 4_000); + assert!(matches!( + graph.extend_retrospectively( + &lease.id, + MAX_RETROSPECTIVE_EXTENSION + Duration::from_millis(1) + ), + Err(CaptureGraphError::RetrospectiveExtensionTooLarge { .. }) + )); + } + + #[test] + fn absolute_retrospective_start_clamps_only_to_graph_coverage() { + let (mut graph, start) = graph(); + let lease = graph + .start_consumer_at( + CaptureConsumer::Meeting, + [CaptureSource::Screen], + start + Duration::from_secs(600), + ) + .unwrap(); + let extended = graph + .extend_retrospectively_to_start(&lease.id, start - Duration::from_secs(1)) + .unwrap(); + assert_eq!(extended.interval.started_elapsed_ms, 0); + assert_eq!(extended.retrospective_extension_ms, 600_000); + } + + #[test] + fn replacement_lease_changes_rewind_source_demand_with_capture_mode() { + let (mut graph, start) = graph(); + let visuals = graph + .start_consumer_at(CaptureConsumer::Rewind, [CaptureSource::Screen], start) + .unwrap(); + assert_eq!( + graph.status_at(start).unwrap().active_sources, + vec![ActiveSourceStatus { + source: CaptureSource::Screen, + consumer_count: 1, + }] + ); + + graph + .end_consumer_at(&visuals.id, start + Duration::from_millis(1)) + .unwrap(); + graph + .start_consumer_at( + CaptureConsumer::Rewind, + [ + CaptureSource::Screen, + CaptureSource::SystemAudio, + CaptureSource::Microphone, + ], + start + Duration::from_millis(1), + ) + .unwrap(); + + assert_eq!( + graph + .status_at(start + Duration::from_millis(1)) + .unwrap() + .active_sources, + vec![ + ActiveSourceStatus { + source: CaptureSource::Screen, + consumer_count: 1, + }, + ActiveSourceStatus { + source: CaptureSource::SystemAudio, + consumer_count: 1, + }, + ActiveSourceStatus { + source: CaptureSource::Microphone, + consumer_count: 1, + }, + ] + ); + } + + #[test] + fn closing_a_lease_returns_a_closed_interval_and_releases_demand() { + let (mut graph, start) = graph(); + let lease = graph + .start_consumer_at(CaptureConsumer::Meeting, [CaptureSource::Microphone], start) + .unwrap(); + let closed = graph + .end_consumer_at(&lease.id, start + Duration::from_secs(2)) + .unwrap(); + assert_eq!(closed.lease.interval.ended_elapsed_ms, 2_000); + assert!(graph + .status_at(start + Duration::from_secs(2)) + .unwrap() + .active_sources + .is_empty()); + } + + #[test] + fn coverage_gaps_preserve_source_reason_and_interval() { + let (mut graph, start) = graph(); + let gap = graph + .record_coverage_gap_at( + CaptureSource::Screen, + CoverageGapReason::PrivacyExclusion, + start + Duration::from_secs(1), + start + Duration::from_secs(2), + ) + .unwrap(); + assert_eq!(gap.id, "gap-1"); + assert_eq!(gap.source, CaptureSource::Screen); + assert_eq!(gap.reason, CoverageGapReason::PrivacyExclusion); + assert_eq!(gap.interval.started_elapsed_ms, 1_000); + assert_eq!( + graph + .status_at(start + Duration::from_secs(2)) + .unwrap() + .coverage_gaps, + vec![gap] + ); + } + + #[test] + fn pins_have_stable_ids_and_closed_intervals() { + let (mut graph, start) = graph(); + let first = graph + .pin_interval_at(start, start + Duration::from_millis(1), Some("keep".into())) + .unwrap(); + let second = graph + .pin_interval_at( + start + Duration::from_millis(2), + start + Duration::from_millis(3), + None, + ) + .unwrap(); + assert_eq!(first.id, "pin-1"); + assert_eq!(second.id, "pin-2"); + assert_eq!(second.interval.ended_elapsed_ms, 3); + } + + #[test] + fn closed_lease_intervals_can_be_pinned_and_released_idempotently() { + let (mut graph, start) = graph(); + let lease = graph + .start_consumer_at(CaptureConsumer::Clip, [CaptureSource::Screen], start) + .unwrap(); + let closed = graph + .end_consumer_at(&lease.id, start + Duration::from_secs(2)) + .unwrap(); + let pin = graph + .pin_interval(&closed.lease.interval, Some("clip materialization".into())) + .unwrap(); + + assert_eq!(pin.interval, closed.lease.interval); + assert!(graph.release_pin(&pin.id)); + assert!(!graph.release_pin(&pin.id)); + assert!(graph + .status_at(start + Duration::from_secs(2)) + .unwrap() + .pins + .is_empty()); + } +} diff --git a/templates/clips/desktop/src-tauri/src/clips/mod.rs b/templates/clips/desktop/src-tauri/src/clips/mod.rs index aa6d04ddfd..ad0eabc8d1 100644 --- a/templates/clips/desktop/src-tauri/src/clips/mod.rs +++ b/templates/clips/desktop/src-tauri/src/clips/mod.rs @@ -1,3 +1,4 @@ +use serde::Serialize; #[cfg(target_os = "macos")] use std::io::Write; use std::path::PathBuf; @@ -40,6 +41,7 @@ const COUNTDOWN_CONTROL_HIT_PAD: f64 = 8.0; // countdown overlay so only the button zones are interactive. static COUNTDOWN_CONTROL_TRACKING: AtomicBool = AtomicBool::new(false); const BUBBLE_LABEL: &str = "bubble"; +const PREPARING_LABEL: &str = "preparing"; const FINALIZING_LABEL: &str = "finalizing"; const FLOW_BAR_LABEL: &str = "flow-bar"; const REGION_GUIDES_LABEL: &str = "region-guides"; @@ -50,6 +52,7 @@ const OVERLAY_LABELS: &[&str] = &[ COUNTDOWN_LABEL, TOOLBAR_LABEL, BUBBLE_LABEL, + PREPARING_LABEL, FINALIZING_LABEL, FLOW_BAR_LABEL, REGION_GUIDES_LABEL, @@ -355,12 +358,11 @@ fn load_bubble_position(app: &AppHandle) -> Option<(i32, i32)> { /// cursor events so the user can still click into whatever they're about to /// record, and closes itself when the countdown finishes. #[tauri::command] -pub async fn show_countdown(app: AppHandle) -> Result<(), String> { +pub async fn show_countdown(app: AppHandle) -> Result { dlog!("[clips-tray] show_countdown invoked"); mark_popover_shown(&app); if let Some(existing) = app.get_webview_window(COUNTDOWN_LABEL) { stop_countdown_control_tracking(); - let _ = app.emit("clips:countdown-shortcuts-active", false); let _ = existing.close(); } let (mx, my, mw, mh) = tray_monitor_physical_rect(&app); @@ -379,11 +381,10 @@ pub async fn show_countdown(app: AppHandle) -> Result<(), String> { .skip_taskbar(true) .shadow(false) .visible(false) - // Don't steal focus from the popover when the overlay opens — - // otherwise macOS fires Focused(false) on the popover, which - // kicks off a cascade of blur-related React re-renders and - // eventually (past the 1500ms guard) auto-hides the popover. - .focused(false) + // The countdown is intentionally modal for three seconds so its + // advertised Return/Escape controls work without system-wide key + // monitoring. The recorder has already parked the popover. + .focused(true) .build() .map_err(|e| { eprintln!("[clips-tray] countdown build failed: {}", e); @@ -394,11 +395,26 @@ pub async fn show_countdown(app: AppHandle) -> Result<(), String> { let _ = win.set_ignore_cursor_events(true); set_capture_excluded(&win); configure_overlay_behavior(&win); - let _ = win.show(); - let _ = app.emit("clips:countdown-shortcuts-active", true); + let generation = match crate::shortcuts::prepare_countdown_shortcuts(app.clone()).await { + Ok(generation) => generation, + Err(error) => { + let _ = win.close(); + return Err(error); + } + }; + // This is a short, explicit modal moment. Making the overlay the actual + // key window lets its ordinary DOM handler receive Return/Escape without + // broad, system-wide Input Monitoring. Closing it restores the user's + // prior application before capture begins. + present_interactive_window(&win); start_countdown_control_tracking(&app); dlog!("[clips-tray] countdown shown"); - Ok(()) + Ok(generation) +} + +#[tauri::command] +pub async fn finish_countdown_shortcuts(app: AppHandle, generation: u64) -> Result<(), String> { + crate::shortcuts::finish_countdown_shortcuts(app, generation).await } /// True when the global cursor sits inside either circular cancel/skip button @@ -458,6 +474,65 @@ fn stop_countdown_control_tracking() { COUNTDOWN_CONTROL_TRACKING.store(false, Ordering::SeqCst); } +/// Compact visible readiness state for the slow work that intentionally runs +/// before the numeric countdown. This capture-excluded card briefly takes +/// focus as the first stage of the explicit modal start flow, so the user sees +/// that Start was accepted without changing the exact countdown-zero boundary. +#[tauri::command] +pub async fn show_preparing(app: AppHandle) -> Result<(), String> { + if let Some(existing) = app.get_webview_window(PREPARING_LABEL) { + let _ = existing.close(); + } + let (mx, my, mw, mh) = tray_monitor_physical_rect(&app); + let scale = overlay_scale_factor(&app); + let content_w: u32 = (260.0 * scale).round() as u32; + let content_h: u32 = (58.0 * scale).round() as u32; + let margin: i32 = (14.0 * scale).round() as i32; + let gutter = overlay_shadow_gutter_physical(&app); + let w = content_w + gutter * 2; + let h = content_h + gutter * 2; + let x = (mx + (mw.saturating_sub(w) / 2) as i32).max(mx); + let y = (my + mh as i32 - h as i32 - margin).max(my); + let win = WebviewWindowBuilder::new(&app, PREPARING_LABEL, build_overlay_url("preparing")) + .title("Preparing recording") + .decorations(false) + .transparent(true) + .always_on_top(true) + .skip_taskbar(true) + .resizable(false) + .shadow(false) + .visible(false) + .focused(true) + .build() + .map_err(|error| format!("preparing window build failed: {error}"))?; + let _ = win.set_size(tauri::Size::Physical(PhysicalSize::new(w, h))); + let _ = win.set_position(PhysicalPosition::new(x, y)); + let _ = win.set_ignore_cursor_events(true); + set_capture_excluded(&win); + configure_overlay_behavior(&win); + // Preparation and countdown are one explicit modal start flow. Making the + // compact readiness card key ensures it gets a real first paint and is + // announced before the full-screen countdown replaces it. + present_interactive_window(&win); + let _ = app.emit("clips:toolbar-preparing", true); + // A newly-created webview needs one paint before the async preparation can + // race ahead and replace it with the countdown. Without this brief yield, + // fast machines can show only a disabled toolbar and never render the + // promised textual readiness state. + tokio::time::sleep(Duration::from_millis(120)).await; + dlog!("[clips-tray] preparing recording shown"); + Ok(()) +} + +#[tauri::command] +pub async fn hide_preparing(app: AppHandle) -> Result<(), String> { + let _ = app.emit("clips:toolbar-preparing", false); + if let Some(window) = app.get_webview_window(PREPARING_LABEL) { + let _ = window.close(); + } + Ok(()) +} + /// Compact bottom-left status window shown while the recorder flushes its /// final chunks and awaits the server finalize. Keeping the native window near /// the card's bounds makes its open/dismiss controls clickable without placing @@ -988,7 +1063,6 @@ pub async fn hide_overlays( preserve_finalizing: Option, ) -> Result<(), String> { stop_countdown_control_tracking(); - let _ = app.emit("clips:countdown-shortcuts-active", false); for label in overlay_labels_to_hide(preserve_finalizing.unwrap_or(false)) { if let Some(w) = app.get_webview_window(label) { let _ = w.close(); @@ -1011,7 +1085,6 @@ pub async fn hide_overlays( #[tauri::command] pub async fn hide_recording_chrome(app: AppHandle) -> Result<(), String> { stop_countdown_control_tracking(); - let _ = app.emit("clips:countdown-shortcuts-active", false); // The countdown + toolbar always tear down on recording stop. The region // guides only tear down when they aren't pinned on-screen via the always-on // toggle — otherwise we'd flicker close→reopen right after stop. @@ -1128,21 +1201,21 @@ pub fn open_macos_privacy_settings(pane: String) -> Result<(), String> { #[cfg(target_os = "macos")] { let url = match pane.as_str() { - "camera" => "x-apple.systempreferences:com.apple.preference.security?Privacy_Camera", + "camera" => "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_Camera", "microphone" => { - "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone" + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_Microphone" } "screen" => { - "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture" + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_ScreenCapture" } "speech" => { - "x-apple.systempreferences:com.apple.preference.security?Privacy_SpeechRecognition" + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_SpeechRecognition" } "accessibility" => { - "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility" + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_Accessibility" } "input-monitoring" => { - "x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent" + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_ListenEvent" } _ => return Err(format!("Unknown macOS privacy pane: {pane}")), }; @@ -1154,6 +1227,157 @@ pub fn open_macos_privacy_settings(pane: String) -> Result<(), String> { } } +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RewindExcludedApplication { + bundle_id: String, + name: String, + path: Option, + installed: bool, +} + +#[cfg(target_os = "macos")] +fn app_bundle_id(path: &str) -> Option { + let output = Command::new("/usr/bin/mdls") + .args(["-name", "kMDItemCFBundleIdentifier", "-raw", path]) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let bundle_id = String::from_utf8_lossy(&output.stdout).trim().to_string(); + (!bundle_id.is_empty() && bundle_id != "(null)").then_some(bundle_id) +} + +#[cfg(target_os = "macos")] +fn app_name_from_path(path: &str) -> String { + std::path::Path::new(path) + .file_stem() + .and_then(|name| name.to_str()) + .unwrap_or("Application") + .to_string() +} + +fn fallback_app_name(bundle_id: &str) -> String { + match bundle_id { + "com.1password.1password" | "com.agilebits.onepassword7" => "1Password".to_string(), + "com.bitwarden.desktop" => "Bitwarden".to_string(), + "com.dashlane.dashlane" => "Dashlane".to_string(), + "com.lastpass.lastpass" => "LastPass".to_string(), + _ => bundle_id + .rsplit('.') + .next() + .filter(|name| !name.is_empty()) + .unwrap_or("Application") + .to_string(), + } +} + +#[cfg(target_os = "macos")] +fn find_installed_app(bundle_id: &str) -> Option { + let query = format!("kMDItemCFBundleIdentifier == '{bundle_id}'"); + let output = Command::new("/usr/bin/mdfind").arg(query).output().ok()?; + if !output.status.success() { + return None; + } + String::from_utf8_lossy(&output.stdout) + .lines() + .find(|path| path.ends_with(".app")) + .map(str::to_string) +} + +#[tauri::command] +pub fn resolve_rewind_excluded_apps( + bundle_ids: Vec, +) -> Result, String> { + #[cfg(target_os = "macos")] + { + return Ok(bundle_ids + .into_iter() + .map(|bundle_id| { + let path = find_installed_app(&bundle_id); + let name = path + .as_deref() + .map(app_name_from_path) + .unwrap_or_else(|| fallback_app_name(&bundle_id)); + RewindExcludedApplication { + bundle_id, + name, + installed: path.is_some(), + path, + } + }) + .collect()); + } + #[cfg(not(target_os = "macos"))] + { + Ok(bundle_ids + .into_iter() + .map(|bundle_id| RewindExcludedApplication { + name: fallback_app_name(&bundle_id), + bundle_id, + path: None, + installed: false, + }) + .collect()) + } +} + +#[cfg(target_os = "macos")] +fn choose_rewind_excluded_apps_blocking() -> Result, String> { + let script = r#" +set chosenApps to choose application with prompt "Choose apps Rewind should never remember" with multiple selections allowed +set chosenPaths to {} +repeat with chosenApp in chosenApps + set end of chosenPaths to POSIX path of (chosenApp as alias) +end repeat +set AppleScript's text item delimiters to linefeed +return chosenPaths as text +"#; + let output = Command::new("/usr/bin/osascript") + .args(["-e", script]) + .output() + .map_err(|error| format!("Could not open the application picker: {error}"))?; + if !output.status.success() { + let error = String::from_utf8_lossy(&output.stderr); + if error.contains("-128") || error.to_ascii_lowercase().contains("canceled") { + return Ok(Vec::new()); + } + return Err(format!("Could not choose applications: {}", error.trim())); + } + let mut apps = Vec::new(); + for path in String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::trim) + .filter(|path| !path.is_empty()) + { + let Some(bundle_id) = app_bundle_id(path) else { + continue; + }; + apps.push(RewindExcludedApplication { + bundle_id, + name: app_name_from_path(path), + path: Some(path.to_string()), + installed: true, + }); + } + Ok(apps) +} + +#[tauri::command] +pub async fn choose_rewind_excluded_apps() -> Result, String> { + #[cfg(target_os = "macos")] + { + return tauri::async_runtime::spawn_blocking(choose_rewind_excluded_apps_blocking) + .await + .map_err(|error| format!("The application picker stopped unexpectedly: {error}"))?; + } + #[cfg(not(target_os = "macos"))] + { + Err("Choosing excluded applications is currently available on macOS only.".to_string()) + } +} + #[tauri::command] pub fn open_local_recording_folder(path: String) -> Result<(), String> { let trimmed = path.trim(); @@ -1902,9 +2126,9 @@ fn type_text_unicode(text: &str, target_bundle_id: Option) { #[cfg(not(target_os = "macos"))] fn type_text_unicode(_text: &str) {} -/// Record the popover's current recording state. When active, clicking the -/// tray icon emits a stop event instead of toggling the popover — so the -/// user can stop a recording from anywhere with one click. +/// Record the popover's current recording state. While active, ordinary app +/// and tray opens restore the parked popover; stopping remains an explicit +/// action in the popover, toolbar, or tray menu. #[tauri::command] pub async fn set_recording_state(app: AppHandle, active: bool) -> Result<(), String> { dlog!("[clips-tray] set_recording_state active={}", active); @@ -2254,7 +2478,10 @@ fn is_pinhole_popover(window: &WebviewWindow) -> bool { fn present_popover(app: &AppHandle, window: &WebviewWindow) { clear_voice_wake_state(app); - set_capture_included(window); + // Reopening Clips must not silently override the user's capture-visibility + // preference. `set_capture_excluded` keeps the window private by default + // and includes it only when "Show Clips in screen captures" is enabled. + set_capture_excluded(window); // Re-apply Space behavior — `orderOut:` resets it, so without this the // popover sticks to whichever Space it was first shown on. configure_overlay_behavior(window); diff --git a/templates/clips/desktop/src-tauri/src/config.rs b/templates/clips/desktop/src-tauri/src/config.rs index 3073de5112..75e32e480b 100644 --- a/templates/clips/desktop/src-tauri/src/config.rs +++ b/templates/clips/desktop/src-tauri/src/config.rs @@ -49,6 +49,18 @@ pub struct ScreenMemoryConfig { pub segment_seconds: u64, #[serde(default = "default_screen_memory_sample_interval_seconds")] pub sample_interval_seconds: u64, + #[serde(default)] + pub capture_mode: RewindCaptureMode, + #[serde(default = "default_rewind_review_before_sending")] + pub review_before_sending: bool, + #[serde(default = "default_rewind_auto_preview_before_sending")] + pub auto_preview_before_sending: bool, + #[serde(default)] + pub agent_clip_retention: RewindAgentClipRetention, + #[serde(default = "default_screen_memory_excluded_bundle_ids")] + pub excluded_bundle_ids: Vec, + #[serde(default = "default_screen_memory_exclude_private_windows")] + pub exclude_private_windows: bool, } impl Default for ScreenMemoryConfig { @@ -60,10 +72,39 @@ impl Default for ScreenMemoryConfig { max_bytes: default_screen_memory_max_bytes(), segment_seconds: default_screen_memory_segment_seconds(), sample_interval_seconds: default_screen_memory_sample_interval_seconds(), + capture_mode: RewindCaptureMode::default(), + review_before_sending: default_rewind_review_before_sending(), + auto_preview_before_sending: default_rewind_auto_preview_before_sending(), + agent_clip_retention: RewindAgentClipRetention::default(), + excluded_bundle_ids: default_screen_memory_excluded_bundle_ids(), + exclude_private_windows: default_screen_memory_exclude_private_windows(), } } } +/// The local capture tracks Rewind is allowed to retain. Audio collection is +/// explicit so an existing local buffer never begins recording sound by default. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RewindCaptureMode { + #[default] + Visuals, + VisualsAudio, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RewindAgentClipRetention { + #[default] + Forever, + #[serde(rename = "24-hours")] + Hours24, + #[serde(rename = "7-days")] + Days7, + #[serde(rename = "30-days")] + Days30, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct FeatureConfig { @@ -130,7 +171,24 @@ fn default_whisper_model_enabled() -> bool { } fn default_screen_memory_retention_hours() -> u32 { - 24 + 8 +} + +fn default_screen_memory_excluded_bundle_ids() -> Vec { + [ + "com.1password.1password", + "com.agilebits.onepassword7", + "com.bitwarden.desktop", + "com.dashlane.dashlane", + "com.lastpass.lastpass", + ] + .into_iter() + .map(str::to_string) + .collect() +} + +fn default_screen_memory_exclude_private_windows() -> bool { + false } fn default_screen_memory_max_bytes() -> u64 { @@ -145,6 +203,14 @@ fn default_screen_memory_sample_interval_seconds() -> u64 { 10 } +fn default_rewind_review_before_sending() -> bool { + true +} + +fn default_rewind_auto_preview_before_sending() -> bool { + true +} + impl Default for FeatureConfig { fn default() -> Self { Self { @@ -268,6 +334,14 @@ pub async fn get_feature_config(app: AppHandle) -> Result #[tauri::command] pub async fn set_feature_config(app: AppHandle, config: FeatureConfig) -> Result<(), String> { let previous = load_config(&app); + if (crate::rewind_clip::is_active(&app) || crate::util::is_recording_active(&app)) + && rewind_capture_contract_changed(&previous.screen_memory, &config.screen_memory) + { + return Err( + "Rewind capture settings stay unchanged while a Clip is recording. Stop the Clip before turning Rewind on or off, pausing it, or changing what it remembers." + .to_string(), + ); + } if previous.launch_at_login_enabled != config.launch_at_login_enabled { if let Err(err) = apply_launch_at_login(&app, config.launch_at_login_enabled) { eprintln!("[clips-tray] launch-at-login apply failed: {err}"); @@ -296,3 +370,76 @@ pub async fn set_feature_config(app: AppHandle, config: FeatureConfig) -> Result let _ = app.emit("app:feature-config-changed", config); Ok(()) } + +fn rewind_capture_contract_changed( + previous: &ScreenMemoryConfig, + next: &ScreenMemoryConfig, +) -> bool { + previous.enabled != next.enabled + || previous.paused != next.paused + || previous.capture_mode != next.capture_mode +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn screen_memory_config_defaults_agent_handoff_fields_when_loading_legacy_json() { + let config: ScreenMemoryConfig = serde_json::from_value(serde_json::json!({ + "enabled": true, + "retentionHours": 24 + })) + .unwrap(); + + assert_eq!(config.retention_hours, 24); + assert_eq!(config.capture_mode, RewindCaptureMode::Visuals); + assert!(config.review_before_sending); + assert!(config.auto_preview_before_sending); + assert_eq!( + config.agent_clip_retention, + RewindAgentClipRetention::Forever + ); + assert!(!config.exclude_private_windows); + assert!(config + .excluded_bundle_ids + .contains(&"com.1password.1password".to_string())); + } + + #[test] + fn rewind_enums_use_stable_kebab_case_values() { + assert_eq!( + serde_json::to_string(&RewindCaptureMode::VisualsAudio).unwrap(), + "\"visuals-audio\"" + ); + assert_eq!( + serde_json::to_string(&RewindAgentClipRetention::Days7).unwrap(), + "\"7-days\"" + ); + } + + #[test] + fn active_clip_interlock_only_blocks_capture_contract_changes() { + let previous = ScreenMemoryConfig { + enabled: true, + paused: false, + capture_mode: RewindCaptureMode::VisualsAudio, + ..ScreenMemoryConfig::default() + }; + let mut retention_only = previous.clone(); + retention_only.retention_hours = 24; + assert!(!rewind_capture_contract_changed(&previous, &retention_only)); + + let mut paused = previous.clone(); + paused.paused = true; + assert!(rewind_capture_contract_changed(&previous, &paused)); + + let mut disabled = previous.clone(); + disabled.enabled = false; + assert!(rewind_capture_contract_changed(&previous, &disabled)); + + let mut mode_changed = previous.clone(); + mode_changed.capture_mode = RewindCaptureMode::Visuals; + assert!(rewind_capture_contract_changed(&previous, &mode_changed)); + } +} diff --git a/templates/clips/desktop/src-tauri/src/lib.rs b/templates/clips/desktop/src-tauri/src/lib.rs index 921037b6df..f8da20cb61 100644 --- a/templates/clips/desktop/src-tauri/src/lib.rs +++ b/templates/clips/desktop/src-tauri/src/lib.rs @@ -6,6 +6,8 @@ mod accessibility; mod adhoc_meetings_watcher; +mod capture_audio_bus; +mod capture_graph; mod clips; mod config; mod debug; @@ -18,7 +20,15 @@ mod notifications; mod permission_status; mod recording_indicator; mod remote_flags; +mod rewind_capture_suspension; +mod rewind_chapters; +mod rewind_clip; +mod rewind_egress; +mod rewind_local_ask; +mod rewind_meeting_history; mod screen_memory; +mod screen_memory_ocr; +mod screen_memory_transcript; mod sentry_report; mod shortcuts; mod silence_detector; @@ -39,7 +49,7 @@ use state::{ }; use util::{ configure_overlay_behavior, is_recording_active, present_interactive_window, - set_capture_included, + set_capture_excluded, }; // Embedded fallback icon — a tiny 16x16 solid purple PNG so the binary always @@ -50,7 +60,7 @@ pub(crate) const TRAY_PNG: &[u8] = include_bytes!("../icons/tray.png"); fn present_popover(app: &tauri::AppHandle) { if let Some(window) = app.get_webview_window("popover") { - set_capture_included(&window); + set_capture_excluded(&window); configure_overlay_behavior(&window); position_popover(app, &window); present_interactive_window(&window); @@ -71,6 +81,9 @@ pub fn run() { .invoke_handler(tauri::generate_handler![ // clips commands clips::show_countdown, + clips::finish_countdown_shortcuts, + clips::show_preparing, + clips::hide_preparing, clips::show_finalizing, clips::hide_finalizing, clips::show_toolbar, @@ -88,6 +101,8 @@ pub fn run() { clips::show_popover, clips::park_popover_offscreen, clips::open_macos_privacy_settings, + clips::choose_rewind_excluded_apps, + clips::resolve_rewind_excluded_apps, clips::open_local_recording_folder, clips::request_macos_screen_recording_access, clips::resize_popover, @@ -142,6 +157,7 @@ pub fn run() { native_screen::native_fullscreen_pending_uploads, native_screen::native_fullscreen_recording_retry_upload, native_screen::native_fullscreen_recording_mark_upload_error, + native_screen::native_fullscreen_recording_clear_upload, native_screen::native_fullscreen_recording_dismiss_upload, native_screen::native_fullscreen_open_drafts_folder, // local-only always-on screen memory compatibility helpers @@ -149,6 +165,36 @@ pub fn run() { screen_memory::screen_memory_delete_all, screen_memory::screen_memory_export_recent, screen_memory::screen_memory_open_folder, + screen_memory::screen_memory_install_agent_connection, + screen_memory::screen_memory_next_agent_handoff, + screen_memory::screen_memory_update_agent_handoff, + screen_memory::screen_memory_due_agent_handoffs, + screen_memory::screen_memory_mark_agent_handoff_deleted, + screen_memory::screen_memory_cancel_agent_handoff_cleanup, + // local-only Rewind evidence policy gate and egress audit trail + rewind_egress::rewind_prepare_evidence_egress, + rewind_egress::rewind_complete_evidence_egress, + rewind_egress::rewind_fail_evidence_egress, + rewind_egress::rewind_list_evidence_egress, + rewind_local_ask::rewind_local_ask, + rewind_local_ask::rewind_replay_moment, + rewind_meeting_history::rewind_meeting_history_status, + rewind_meeting_history::rewind_meeting_history_prepare, + rewind_meeting_history::rewind_meeting_history_collect, + rewind_meeting_history::rewind_meeting_history_cancel, + rewind_clip::rewind_clip_status, + rewind_clip::rewind_clip_prepare, + rewind_clip::rewind_clip_start, + rewind_clip::rewind_clip_extend, + rewind_clip::rewind_clip_pause, + rewind_clip::rewind_clip_resume, + rewind_clip::rewind_clip_stop_and_upload, + rewind_clip::rewind_clip_stop_and_save, + rewind_clip::rewind_clip_cancel, + rewind_clip::rewind_agent_handoff_upload, + rewind_clip::rewind_agent_handoff_preview, + rewind_capture_suspension::rewind_capture_suspension_acquire, + rewind_capture_suspension::rewind_capture_suspension_release, // recording indicator pill recording_indicator::recording_pill_show, recording_indicator::recording_pill_expand, @@ -221,6 +267,10 @@ pub fn run() { .manage(LastTranscript::default()) .manage(native_screen::NativeFullscreenRecordingState::default()) .manage(screen_memory::ScreenMemoryState::default()) + .manage(capture_graph::CaptureGraphState::default()) + .manage(rewind_clip::RewindClipState::default()) + .manage(rewind_meeting_history::RewindMeetingHistoryState::default()) + .manage(rewind_capture_suspension::RewindCaptureSuspensionState::default()) .manage(meetings_watcher::MeetingsWatcherState::default()) .manage(adhoc_meetings_watcher::AdhocMeetingsWatcherState::default()) .manage(notifications::MeetingNotificationState::default()) @@ -292,8 +342,8 @@ pub fn run() { // toggle is off). clips::reconcile_region_guides(app.handle()); shortcuts::register_shortcuts(app)?; + shortcuts::install_countdown_local_key_monitor(app); shortcuts::install_popover_dismiss_handler(app); - shortcuts::install_countdown_shortcut_handler(app); // Spawn the upcoming-meetings poller. Idempotent — gated by a // OnceLock inside `spawn_watcher`. The frontend wires the @@ -417,7 +467,11 @@ pub fn run() { // Reopen is macOS-only — gated behind cfg so Windows compiles. #[cfg(target_os = "macos")] if let tauri::RunEvent::Reopen { .. } = _event { - toggle_popover(_app_handle); + if is_recording_active(_app_handle) { + clips::force_show_popover(_app_handle); + } else { + toggle_popover(_app_handle); + } } // `app.exit()` (tray Quit, Cmd+Q, OS shutdown) delivers // `ExitRequested` first, then `Exit` unless prevented. A live diff --git a/templates/clips/desktop/src-tauri/src/logfile.rs b/templates/clips/desktop/src-tauri/src/logfile.rs index bb7baa0c98..85ae3f6896 100644 --- a/templates/clips/desktop/src-tauri/src/logfile.rs +++ b/templates/clips/desktop/src-tauri/src/logfile.rs @@ -114,7 +114,19 @@ fn rotate_if_needed(path: &Path) { fn append_line(path: &Path, line: &str) { if let Ok(mut file) = OpenOptions::new().create(true).append(true).open(path) { - let _ = writeln!(file, "{} {line}", timestamp()); + // One append syscall keeps direct diagnostics from interleaving with + // the webview/stdout log pump at the character level. + let record = format!("{} {line}\n", timestamp()); + let _ = file.write_all(record.as_bytes()); + } +} + +/// Write one native diagnostic directly to the persistent log. Unlike +/// `eprintln!`, this also works in debug-signed Alpha builds where stdout and +/// stderr intentionally remain attached to the launching terminal. +pub(crate) fn diagnostic(line: &str) { + if let Some(path) = log_path() { + append_line(&path, line); } } diff --git a/templates/clips/desktop/src-tauri/src/native_screen.rs b/templates/clips/desktop/src-tauri/src/native_screen.rs index ea475ddd5d..d311636258 100644 --- a/templates/clips/desktop/src-tauri/src/native_screen.rs +++ b/templates/clips/desktop/src-tauri/src/native_screen.rs @@ -105,6 +105,20 @@ fn audio_filter_chain(downmix: bool, denoise: bool, mic_pregain: bool) -> String filters.push(AUDIO_LOUDNESS_FILTER); filters.join(",") } + +#[cfg(all(test, target_os = "macos"))] +mod fragment_fence_backend_tests { + use super::*; + + #[test] + fn fragment_fence_rejects_unsupported_backend() { + let child = Command::new("/usr/bin/true").spawn().unwrap(); + let backend = NativeFullscreenBackend::Screencapture { child }; + assert!(backend + .request_fragment_fence(PathBuf::from("/tmp/next.mp4")) + .is_err()); + } +} const NATIVE_CAPTURE_MAX_LONG_EDGE: u32 = 1280; const NATIVE_CAPTURE_FPS: u32 = 24; @@ -115,14 +129,20 @@ const NATIVE_CAPTURE_FPS: u32 = 24; mod custom_capture; #[cfg(target_os = "macos")] use custom_capture::{ - start_custom_screencapturekit_backend_at, CustomCaptureResume, CustomScreenCaptureWriter, + prepare_clip_sink, start_custom_screencapturekit_backend_at, ClosedSegmentFile, + CustomCaptureResume, CustomScreenCaptureWriter, PreparedClipSink, SegmentFence, }; // Live chunk uploader: tails the fragmented MP4 and streams it during // recording; attached/finalized/abandoned from the session logic here. #[cfg(target_os = "macos")] mod live_upload; #[cfg(target_os = "macos")] -use live_upload::{attach_live_uploader_to_session, LiveUpload}; +pub(crate) use live_upload::ClipLiveUploadConfig; +#[cfg(target_os = "macos")] +use live_upload::{ + attach_live_uploader_to_session, cancel_clip_live_upload, finalize_clip_live_upload, + start_clip_live_uploader, LiveUpload, +}; const AVCONVERT_PATH: &str = "/usr/bin/avconvert"; const AVCONVERT_TIMEOUT: Duration = Duration::from_secs(5 * 60); const FFMPEG_TIMEOUT: Duration = Duration::from_secs(8 * 60); @@ -151,13 +171,13 @@ const DISK_MONITOR_CRITICAL_BYTES: u64 = 250 * 1024 * 1024; const DISK_MONITOR_INTERVAL_SECS: u64 = 30; #[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum NativeUploadMode { +pub(crate) enum NativeUploadMode { Buffered, Streaming, } impl NativeUploadMode { - fn from_option(value: Option) -> Self { + pub(crate) fn from_option(value: Option) -> Self { match value.as_deref() { Some("streaming") => Self::Streaming, _ => Self::Buffered, @@ -323,9 +343,307 @@ pub(crate) enum NativeFullscreenBackend { /// resume, keeping one continuous append-only file so live upload is /// never interrupted. resume: CustomCaptureResume, + /// Single temporary Clip writer slot sharing this producer's sample + /// callbacks. It is intentionally absent from the audio bus. + clip_sink: Arc>>, }, } +/// Metadata and explicit remote-upload authority for a temporary Clip sink. +/// Passing `upload.server_url: None` creates a local-only artifact. +#[cfg(target_os = "macos")] +pub(crate) struct SharedClipSinkRequest { + pub(crate) path: PathBuf, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) include_mic: bool, + pub(crate) include_system_audio: bool, + pub(crate) has_camera: bool, + pub(crate) upload: ClipLiveUploadConfig, +} + +#[cfg(target_os = "macos")] +pub(crate) struct SharedClipSinkResult { + pub(crate) path: PathBuf, + pub(crate) bytes: u64, + pub(crate) verification_pending: bool, + pub(crate) duration_ms: u64, + pub(crate) width: u32, + pub(crate) height: u32, +} + +#[cfg(target_os = "macos")] +impl SharedClipSinkResult { + pub(crate) fn into_native_upload_result( + self, + recording_id: String, + ) -> NativeFullscreenUploadResult { + NativeFullscreenUploadResult { + recording_id, + duration_ms: self.duration_ms as u128, + width: Some(self.width), + height: Some(self.height), + bytes: self.bytes, + verification_pending: self.verification_pending, + } + } +} + +/// Narrow handle exposed to Rewind orchestration. It mirrors the callbacks of +/// an already-running physical producer; it never owns an SCStream or audio +/// bus producer of its own. +#[cfg(target_os = "macos")] +pub(crate) struct SharedClipSink { + sink: PreparedClipSink, + path: PathBuf, + width: u32, + height: u32, + live_upload: Option, + upload_reset: Option, +} + +#[cfg(target_os = "macos")] +impl SharedClipSink { + pub(crate) fn path(&self) -> &Path { + &self.path + } + + pub(crate) fn dimensions(&self) -> (u32, u32) { + (self.width, self.height) + } + + pub(crate) fn duration_ms(&self) -> u64 { + self.sink.duration_ms() + } + + pub(crate) fn activate(&mut self) -> Result<(), String> { + self.sink.activate()?; + Ok(()) + } + + pub(crate) fn pause(&self) -> Result<(), String> { + self.sink.pause() + } + + pub(crate) fn resume(&self) -> Result<(), String> { + self.sink.resume() + } + + /// Close callback admission now; callers may defer `finalize` to a worker + /// without extending the logical capture interval. + pub(crate) fn deactivate(&self) { + self.sink.deactivate(); + } + + /// Return only bytes the server has acknowledged so far. + pub(crate) fn uploaded_bytes_now(&self) -> Option { + self.live_upload + .as_ref() + .map(|live| live.ctrl.uploaded_bytes.load(Ordering::SeqCst)) + } + + /// Finalize and verify the local artifact before any server finalization. + /// The caller can persist retry metadata at this durable boundary, so an + /// app exit or network failure while awaiting the server never strands an + /// unindexed file. + pub(crate) fn finalize_writer(&mut self) -> Result { + if let Err(error) = self.sink.finalize() { + if let Some(live) = self.live_upload.as_ref() { + cancel_clip_live_upload(live); + } + return Err(error); + } + let bytes = std::fs::metadata(&self.path) + .map(|metadata| metadata.len()) + .unwrap_or(0); + if bytes == 0 { + if let Some(live) = self.live_upload.as_ref() { + cancel_clip_live_upload(live); + } + return Err("shared Clip writer produced an empty local artifact".into()); + } + let logical_duration_ms = self.sink.duration_ms() as u128; + let measured_duration_ms = match probe_local_media_duration_ms(&self.path) { + Ok(duration_ms) => duration_ms, + Err(error) => { + if let Some(live) = self.live_upload.as_ref() { + cancel_clip_live_upload(live); + } + return Err(error); + } + }; + if !media_durations_materially_match(logical_duration_ms, measured_duration_ms) { + if let Some(live) = self.live_upload.as_ref() { + cancel_clip_live_upload(live); + } + return Err(format!( + "Clip may be incomplete. The local media duration ({measured_duration_ms} ms) did not match the recorded duration ({logical_duration_ms} ms)." + )); + } + Ok(SharedClipSinkResult { + path: self.path.clone(), + bytes, + verification_pending: false, + duration_ms: measured_duration_ms as u64, + width: self.width, + height: self.height, + }) + } + + pub(crate) async fn finalize_upload(&mut self, duration_ms: u64) -> Result { + match self.live_upload.take() { + Some(live) => { + let result = finalize_clip_live_upload(live, duration_ms).await?; + Ok(result.verification_pending) + } + None => Ok(false), + } + } + + pub(crate) fn cancel_upload(&mut self) { + if let Some(live) = self.live_upload.take() { + cancel_clip_live_upload(&live); + } + } + + pub(crate) fn cancel(mut self) { + if let Some(live) = self.live_upload.take() { + cancel_clip_live_upload(&live); + } + self.sink.cancel(); + } + + /// Use when Add previous 30s/5m switches to Rewind's exact local + /// materializer. A partially streamed live file must never be combined + /// with pre-roll; cancel it and reset the server sequence before the + /// materialized whole artifact takes over upload. + pub(crate) async fn abandon_for_rewind_materialization(mut self) -> Result<(), String> { + if let Some(live) = self.live_upload.take() { + cancel_clip_live_upload(&live); + } + self.sink.cancel(); + if let Some(config) = self.upload_reset.take() { + let Some((server_url, recording_id, auth_token, cookie)) = config.validated()? else { + return Ok(()); + }; + reset_upload_chunks( + &server_url, + &recording_id, + MP4_RECORDING_MIME_TYPE, + &auth_token, + &cookie, + ) + .await?; + } + Ok(()) + } +} + +#[cfg(target_os = "macos")] +pub(crate) fn prepare_shared_clip_sink( + app: &AppHandle, + backend: &NativeFullscreenBackend, + request: SharedClipSinkRequest, +) -> Result { + let NativeFullscreenBackend::CustomScreenCaptureKit { clip_sink, .. } = backend else { + return Err("shared Clip sinks require the active custom ScreenCaptureKit producer".into()); + }; + let has_audio = request.include_mic || request.include_system_audio; + // Validate remote authority before allocating/installing the sink, so a + // malformed remote request cannot leave a prepared local writer attached. + let upload_reset = request.upload.clone(); + let live_upload = start_clip_live_uploader( + app, + request.path.clone(), + MP4_RECORDING_MIME_TYPE.to_string(), + Some(request.width), + Some(request.height), + has_audio, + request.has_camera, + request.upload, + )?; + let sink = match prepare_clip_sink( + Arc::clone(clip_sink), + &request.path, + request.width, + request.height, + request.include_mic, + request.include_system_audio, + ) { + Ok(sink) => sink, + Err(error) => { + if let Some(live) = live_upload.as_ref() { + cancel_clip_live_upload(live); + } + return Err(error); + } + }; + Ok(SharedClipSink { + sink, + path: request.path, + width: request.width, + height: request.height, + live_upload, + upload_reset: Some(upload_reset), + }) +} + +#[cfg(target_os = "macos")] +impl NativeFullscreenBackend { + /// Queue a local fMP4 fence without stopping the stream or recreating its + /// writer. Only the custom segmented backend supports this; callers must + /// treat unsupported capture backends as a hard local-buffer failure. + pub(crate) fn request_fragment_fence( + &self, + next_path: PathBuf, + ) -> Result { + match self { + Self::CustomScreenCaptureKit { writer, .. } => writer.request_fragment_fence(next_path), + _ => Err("fragment fences are unsupported by this capture backend".into()), + } + } + + /// Await a queued fence from a worker thread, never from an AVFoundation + /// delegate callback. A timeout fails closed: callers must not consume the + /// next path as if a file had been finalized. + pub(crate) fn await_fragment_fence( + fence: SegmentFence, + timeout: Duration, + ) -> Result { + fence.wait(timeout) + } +} + +/// Start the custom ScreenCaptureKit path in local rolling-buffer mode. +/// Unlike the ordinary recorder, this always selects delegate-fed fMP4 output +/// and preserves microphone/system audio as separate tracks, regardless of +/// the remote live-upload feature flag. +#[cfg(target_os = "macos")] +pub(crate) fn start_segmented_custom_screencapturekit_backend_at( + app: &AppHandle, + output_path: &Path, + include_audio: bool, + capture_system_audio: bool, + mic_device_id: Option<&str>, + mic_device_label: Option<&str>, + target_display_id: Option, + capture_region: Option, + defer_recording_output: bool, +) -> Result<(NativeFullscreenBackend, Option, Option), String> { + start_custom_screencapturekit_backend_at( + app, + output_path, + include_audio, + capture_system_audio, + mic_device_id, + mic_device_label, + target_display_id, + capture_region, + defer_recording_output, + true, + ) +} + /// Safety net for the `screencapture` fallback: if a session carrying a live /// `screencapture` child is ever dropped without going through /// `stop_native_recording`/`stop_screencapture` (app quit, crash unwind, or an @@ -596,7 +914,7 @@ pub fn native_fullscreen_claim_upload_open(recording_id: String) -> bool { true } -fn emit_native_upload_finished( +pub(crate) fn emit_native_upload_finished( app: &AppHandle, server_url: &str, recording_id: &str, @@ -687,12 +1005,12 @@ pub struct NativeFullscreenStartInfo { #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct NativeFullscreenUploadResult { - recording_id: String, - duration_ms: u128, - width: Option, - height: Option, - bytes: u64, - verification_pending: bool, + pub(crate) recording_id: String, + pub(crate) duration_ms: u128, + pub(crate) width: Option, + pub(crate) height: Option, + pub(crate) bytes: u64, + pub(crate) verification_pending: bool, } #[derive(Serialize, Clone)] @@ -726,6 +1044,120 @@ pub struct NativeFullscreenSaveResult { file: NativeLocalRecordingFile, } +/// One exact, source-local interval from a finalized native MP4. `start_ms` +/// and `end_ms` are half-open (`[start_ms, end_ms)`) and are never expanded to +/// an enclosing rolling-buffer segment by this layer. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct NativeMediaSlice { + pub path: PathBuf, + pub system_audio_path: Option, + pub microphone_path: Option, + pub start_ms: u64, + pub end_ms: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum NativeAudioSelection { + None, + System, + Microphone, + MixBoth, +} + +/// Final media facts after capture/segment consolidation. This is deliberately +/// independent from a live session so Rewind can hand a materialized artifact +/// to the same local-save/upload continuations later. +#[derive(Clone, Debug)] +pub(crate) struct FinalizedNativeArtifact { + pub path: PathBuf, + pub mime_type: &'static str, + pub duration_ms: u128, + pub width: Option, + pub height: Option, + pub mic_captured: bool, + pub system_audio_captured: bool, + pub has_camera: bool, + pub custom_pipeline: bool, +} + +impl FinalizedNativeArtifact { + fn from_session(session: &NativeFullscreenSession, duration_ms: u128) -> Self { + Self { + path: session.path.clone(), + mime_type: session.mime_type, + duration_ms, + width: session.width, + height: session.height, + mic_captured: session.restart.mic_captured_in_file, + system_audio_captured: session.restart.capture_system_audio, + has_camera: false, + custom_pipeline: session.custom_pipeline, + } + } + + pub(crate) fn rewind_mp4( + path: PathBuf, + duration_ms: u128, + width: Option, + height: Option, + mic_captured: bool, + system_audio_captured: bool, + ) -> Self { + Self { + path, + mime_type: MP4_RECORDING_MIME_TYPE, + duration_ms, + width, + height, + mic_captured, + system_audio_captured, + has_camera: false, + custom_pipeline: true, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PlannedNativeSlice { + path: PathBuf, + start_ms: u64, + end_ms: u64, +} + +/// Validate exact source-local ranges against independently inspected asset +/// durations. A gap is rejected rather than silently bridging it: materialized +/// Rewind output must not imply continuous coverage it does not have. +pub(crate) fn plan_native_media_slices( + slices: &[NativeMediaSlice], + source_durations_ms: &[u64], +) -> Result, String> { + if slices.is_empty() || slices.len() != source_durations_ms.len() { + return Err("media slice plan requires one non-empty duration per slice".into()); + } + let mut planned = Vec::with_capacity(slices.len()); + let mut expected_start = None; + for (slice, duration_ms) in slices.iter().zip(source_durations_ms) { + if slice.path.as_os_str().is_empty() || *duration_ms == 0 { + return Err("media slice has a missing or empty source".into()); + } + if slice.start_ms >= slice.end_ms || slice.end_ms > *duration_ms { + return Err("media slice range is invalid or exceeds its source".into()); + } + if let Some(expected) = expected_start { + if slice.start_ms != expected { + return Err("media slice plan contains a coverage gap or overlap".into()); + } + } + expected_start = Some(slice.end_ms); + planned.push(PlannedNativeSlice { + path: slice.path.clone(), + start_ms: slice.start_ms, + end_ms: slice.end_ms, + }); + } + Ok(planned) +} + impl From<&SavedNativeRecording> for PendingNativeRecording { fn from(saved: &SavedNativeRecording) -> Self { Self { @@ -1469,15 +1901,15 @@ pub async fn native_fullscreen_recording_stop_and_upload( } } - let result = upload_recording_file( + let artifact = FinalizedNativeArtifact::from_session(&session, duration_ms); + let result = upload_finalized_native_artifact( &app, - &session, + &artifact, server_url.clone(), recording_id.clone(), auth_token, cookie, upload_mode, - duration_ms, has_audio, has_camera, ) @@ -1582,7 +2014,8 @@ pub async fn native_fullscreen_recording_stop_and_save( ); } - save_native_recording_to_local_export(&app, &session, &folder_name, &file_role, duration_ms) + let artifact = FinalizedNativeArtifact::from_session(&session, duration_ms); + save_finalized_native_artifact_to_local_export(&app, &artifact, &folder_name, &file_role) } #[tauri::command] @@ -2180,6 +2613,7 @@ fn start_segment_backend( target_display_id, capture_region, false, + false, ) } else { start_screencapturekit_backend_at( @@ -2640,6 +3074,17 @@ pub async fn native_fullscreen_recording_mark_upload_error( Ok(()) } +#[tauri::command] +pub async fn native_fullscreen_recording_clear_upload( + app: AppHandle, + recording_id: String, +) -> Result<(), String> { + let saved = read_saved_recording_metadata(&app, &recording_id)?; + clear_saved_recording(&app, &saved)?; + let _ = app.emit("clips:pending-uploads-changed", ()); + Ok(()) +} + #[tauri::command] pub async fn native_fullscreen_recording_dismiss_upload( app: AppHandle, @@ -2876,37 +3321,6 @@ fn names_match(a: &str, b: &str) -> bool { short.iter().all(|token| long.contains(token)) } -#[cfg(target_os = "macos")] -fn is_built_in_input_name(value: &str) -> bool { - let value = normalize_audio_device_name(value); - value.contains("macbook") - || value.contains("built in") - || value.contains("builtin") - || value.contains("internal microphone") -} - -#[cfg(target_os = "macos")] -fn is_phone_input_name(value: &str) -> bool { - let value = normalize_audio_device_name(value); - value.contains("iphone") - || value.contains("ipad") - || value.contains("continuity") - || value.contains("phone microphone") -} - -#[cfg(target_os = "macos")] -fn preferred_default_microphone_device(devices: &[AudioInputDevice]) -> Option { - devices - .iter() - .find(|device| is_built_in_input_name(&device.name)) - .or_else(|| { - devices - .iter() - .find(|device| !is_phone_input_name(&device.name)) - }) - .cloned() -} - #[cfg(target_os = "macos")] pub(crate) fn resolve_microphone_capture_device( device_id: Option<&str>, @@ -2926,15 +3340,13 @@ pub(crate) fn resolve_microphone_capture_device( }); if device_id.is_none() && device_label.is_none() { - let resolved = preferred_default_microphone_device(&devices); + // "Default mic" must remain the actual macOS default. Do not pin a + // convenient-looking built-in device: studio/USB users may be speaking + // into the system-selected input while that laptop mic records silence. eprintln!( - "[clips-tray] mic resolve: no explicit input provided -> {}", - match &resolved { - Some(device) => format!("using {} ({})", device.name, device.id), - None => "using macOS default input".to_string(), - } + "[clips-tray] mic resolve: no explicit input provided -> using macOS default input" ); - return Ok(resolved); + return Ok(None); } let resolved = device_id @@ -2988,12 +3400,11 @@ fn move_or_copy_file(from: &Path, to: &Path) -> Result<(), String> { Ok(()) } -fn save_native_recording_to_local_export( +pub(crate) fn save_finalized_native_artifact_to_local_export( app: &AppHandle, - session: &NativeFullscreenSession, + artifact: &FinalizedNativeArtifact, folder_name: &str, file_role: &str, - duration_ms: u128, ) -> Result { let safe_folder_name = sanitize_path_component(folder_name, "clip"); let safe_role = match file_role { @@ -3009,11 +3420,11 @@ fn save_native_recording_to_local_export( std::fs::create_dir_all(&folder) .map_err(|e| format!("local recording folder unavailable: {e}"))?; - let extension = native_extension_for_mime_type(session.mime_type); + let extension = native_extension_for_mime_type(artifact.mime_type); let file_name = format!("{}.{}", local_role_file_stem(safe_role), extension); let destination = folder.join(&file_name); let _ = std::fs::remove_file(&destination); - move_or_copy_file(&session.path, &destination)?; + move_or_copy_file(&artifact.path, &destination)?; let bytes = std::fs::metadata(&destination) .map_err(|e| format!("local recording metadata unavailable: {e}"))? @@ -3030,11 +3441,11 @@ fn save_native_recording_to_local_export( role: safe_role.to_string(), path: destination.to_string_lossy().to_string(), file_name, - mime_type: session.mime_type.to_string(), + mime_type: artifact.mime_type.to_string(), bytes, - duration_ms, - width: session.width, - height: session.height, + duration_ms: artifact.duration_ms, + width: artifact.width, + height: artifact.height, }, }) } @@ -3331,6 +3742,67 @@ fn persist_saved_recording_error(app: &AppHandle, saved: &mut SavedNativeRecordi let _ = write_saved_recording_metadata(app, saved); } +/// Index a finalized shared-producer Clip in the ordinary pending-upload +/// store. The file may live in Rewind's temporary artifact directory; the +/// metadata is the durable pointer that makes restart/retry and user-visible +/// recovery use the same flow as every other native recording. +pub(crate) fn persist_shared_clip_recording( + app: &AppHandle, + path: &Path, + server_url: &str, + recording_id: &str, + duration_ms: u128, + width: u32, + height: u32, + include_mic: bool, + include_system_audio: bool, + has_camera: bool, + error: Option<&str>, +) -> Result<(), String> { + let bytes = std::fs::metadata(path) + .map_err(|source| format!("shared Clip local artifact unavailable: {source}"))? + .len(); + if bytes == 0 { + return Err("shared Clip local artifact is empty".into()); + } + let saved = SavedNativeRecording { + recording_id: recording_id.to_string(), + server_url: server_url.trim_end_matches('/').to_string(), + file_path: path.to_path_buf(), + segment_paths: Vec::new(), + mime_type: MP4_RECORDING_MIME_TYPE.to_string(), + duration_ms, + width: Some(width), + height: Some(height), + bytes, + has_audio: include_mic || include_system_audio, + mic_captured: include_mic, + system_audio_captured: include_system_audio, + has_camera, + saved_at: now_iso(), + last_attempt_at: error.map(|_| now_iso()), + last_error: error.map(ToString::to_string), + retry_count: u32::from(error.is_some()), + custom_pipeline: true, + corrupt: false, + }; + write_saved_recording_metadata(app, &saved)?; + let _ = app.emit("clips:pending-uploads-changed", ()); + Ok(()) +} + +/// Remove the shared Clip's temporary artifact and pending metadata after a +/// conclusive server success (or after a local export has its own copy). +pub(crate) fn clear_shared_clip_recording(app: &AppHandle, recording_id: &str, path: &Path) { + let saved = read_saved_recording_metadata(app, recording_id).ok(); + if let Some(saved) = saved { + clear_saved_recording_after_success(app, &saved); + } else if let Err(error) = remove_saved_file(path, "shared Clip temporary artifact") { + eprintln!("[clips-tray] shared Clip cleanup failed: {error}"); + } + let _ = app.emit("clips:pending-uploads-changed", ()); +} + #[cfg(target_os = "macos")] /// Return the `CGDirectDisplayID` of the display containing the centre of /// the last-clicked tray icon. Uses the Tauri monitor list to locate the @@ -3433,6 +3905,7 @@ fn start_screencapturekit_recording( target_display_id, capture_region, defer_recording_output, + false, )? } else { start_screencapturekit_backend_at( @@ -3654,6 +4127,66 @@ fn region_source_rect( #[cfg(target_os = "macos")] const SCK_FINALIZE_TIMEOUT: Duration = Duration::from_secs(10); +/// `SCStream::stop_capture()` occasionally stops the underlying capture but +/// never returns from ScreenCaptureKit's synchronous completion wait. Keep +/// teardown bounded so the writer can still close its inputs, flush the final +/// fragment, and let the upload path validate the playable file on disk. +#[cfg(target_os = "macos")] +const CUSTOM_SCK_STOP_TIMEOUT: Duration = Duration::from_secs(3); + +#[cfg(target_os = "macos")] +fn run_bounded_capture_stop(stop: F, timeout: Duration) -> Result<(), String> +where + F: FnOnce() -> Result<(), String> + Send + 'static, +{ + let (tx, rx) = std::sync::mpsc::sync_channel(1); + std::thread::spawn(move || { + let _ = tx.send(stop()); + }); + match rx.recv_timeout(timeout) { + Ok(result) => result, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => Err(format!( + "custom ScreenCaptureKit stop timed out after {}s; continuing finalization", + timeout.as_secs() + )), + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + Err("custom ScreenCaptureKit stop worker disconnected".into()) + } + } +} + +#[cfg(all(test, target_os = "macos"))] +mod bounded_capture_stop_tests { + use super::run_bounded_capture_stop; + use std::time::{Duration, Instant}; + + #[test] + fn returns_the_capture_stop_result() { + assert_eq!( + run_bounded_capture_stop(|| Ok(()), Duration::from_millis(50)), + Ok(()) + ); + assert_eq!( + run_bounded_capture_stop(|| Err("stop failed".to_string()), Duration::from_millis(50)), + Err("stop failed".to_string()) + ); + } + + #[test] + fn releases_the_caller_when_capture_stop_hangs() { + let started = Instant::now(); + let result = run_bounded_capture_stop( + || { + std::thread::sleep(Duration::from_secs(1)); + Ok(()) + }, + Duration::from_millis(20), + ); + assert!(result.unwrap_err().contains("timed out")); + assert!(started.elapsed() < Duration::from_millis(250)); + } +} + /// Stop the active recording. When `wait_for_finalize` is set (save/upload /// paths — the file is about to be moved) this blocks until ScreenCaptureKit /// signals the recording finished, so the caller never moves a half-written @@ -3678,14 +4211,20 @@ pub(crate) fn stop_native_recording( // Stop the watchdog first so it can't rebuild the stream out from // under us mid-teardown. watchdog_shutdown.store(true, Ordering::SeqCst); - let stop_result = stream - .lock() - .map_err(|e| format!("custom ScreenCaptureKit stop lock poisoned: {e}")) - .and_then(|guard| { - guard - .stop_capture() - .map_err(|e| format!("custom ScreenCaptureKit stop failed: {e:?}")) - }); + let stream_for_stop = Arc::clone(stream); + let stop_result = run_bounded_capture_stop( + move || { + stream_for_stop + .lock() + .map_err(|e| format!("custom ScreenCaptureKit stop lock poisoned: {e}")) + .and_then(|guard| { + guard + .stop_capture() + .map_err(|e| format!("custom ScreenCaptureKit stop failed: {e:?}")) + }) + }, + CUSTOM_SCK_STOP_TIMEOUT, + ); if let Err(err) = &stop_result { eprintln!("[clips-tray] custom capture stop_capture error: {err}"); } @@ -3796,29 +4335,31 @@ fn stop_screencapture(child: &mut Child) -> Result<(), String> { } } -async fn upload_recording_file( +/// Upload a finalized local artifact through the ordinary native preparation +/// and response path. Rewind materializers can call this after they have +/// produced a bounded local MP4; it owns no capture session or live uploader. +pub(crate) async fn upload_finalized_native_artifact( app: &AppHandle, - session: &NativeFullscreenSession, + artifact: &FinalizedNativeArtifact, server_url: String, recording_id: String, auth_token: String, cookie: String, upload_mode: NativeUploadMode, - duration_ms: u128, has_audio: bool, has_camera: bool, ) -> Result { let prepared = prepare_recording_file( app, - &session.path, - session.mime_type, - session.width, - session.height, - Some(duration_ms), + &artifact.path, + artifact.mime_type, + artifact.width, + artifact.height, + Some(artifact.duration_ms), has_audio, - session.restart.mic_captured_in_file, - session.restart.capture_system_audio, - session.custom_pipeline, + artifact.mic_captured, + artifact.system_audio_captured, + artifact.custom_pipeline, )?; let upload_result = upload_prepared_recording_file( app, @@ -3828,9 +4369,9 @@ async fn upload_recording_file( auth_token, cookie, upload_mode, - duration_ms, - session.width, - session.height, + artifact.duration_ms, + artifact.width, + artifact.height, has_audio, has_camera, ) @@ -5646,8 +6187,225 @@ fn validate_recording_segment_file(path: &Path) -> Result<(), String> { Ok(()) } +/// Compose exact source-local MP4 ranges. Passthrough exports can begin video +/// on the preceding sync sample; callers requiring byte/sample-exact exclusion +/// at a trimmed first boundary must request a transcode materialization rather +/// than treating this passthrough composition as an isolation boundary. +#[cfg(target_os = "macos")] +pub(crate) fn compose_mp4_slices(slices: &[NativeMediaSlice], output: &Path) -> Result<(), String> { + compose_mp4_slices_impl(slices, output, false) +} + +/// Materialize bounded Rewind media through a fresh encode. Unlike the +/// passthrough compositor, this is the required path when no pre-start frame +/// may survive a trimmed boundary. It is intentionally separate from ordinary +/// recorder segment concat, which remains fast passthrough. +#[cfg(target_os = "macos")] +pub(crate) fn materialize_mp4_slices_exact( + slices: &[NativeMediaSlice], + output: &Path, + audio: NativeAudioSelection, +) -> Result<(), String> { + let temporary = output.with_extension("bounded-compose.mp4"); + let _ = std::fs::remove_file(&temporary); + compose_mp4_slices(slices, &temporary)?; + let ffmpeg = resolve_ffmpeg_path().ok_or_else(|| { + "exact bounded materialization requires ffmpeg; passthrough composition is not private enough for this range".to_string() + })?; + let mut command = Command::new(ffmpeg); + command + .arg("-y") + .arg("-i") + .arg(&temporary) + .arg("-map") + .arg("0:v:0?") + .arg("-c:v") + .arg("libx264") + // Exact Rewind exports must be freshly encoded to guarantee that no + // pre-boundary keyframe survives, but the default x264 preset makes + // Stop look hung for roughly the full duration of the Clip. This path + // favors interactive finalization speed; the ordinary upload + // transcode still owns the product's size/quality policy. + .arg("-preset") + .arg("ultrafast"); + match audio { + NativeAudioSelection::None => { + command.arg("-an"); + } + NativeAudioSelection::System => { + command.args(["-map", "0:a:0", "-c:a", "aac"]); + } + NativeAudioSelection::Microphone => { + command.args(["-map", "0:a:1", "-c:a", "aac"]); + } + NativeAudioSelection::MixBoth => { + command.args([ + "-filter_complex", + "[0:a:0][0:a:1]amix=inputs=2:normalize=0[a]", + "-map", + "[a]", + "-c:a", + "aac", + ]); + } + } + let child = command + .arg("-movflags") + .arg("+faststart") + .arg(output) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("exact bounded ffmpeg spawn failed: {e}"))?; + let result = wait_for_transcode_child( + child, + FFMPEG_TIMEOUT, + "ffmpeg exact bounded materialization", + ); + let _ = std::fs::remove_file(&temporary); + result +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn materialize_mp4_slices_exact( + _slices: &[NativeMediaSlice], + _output: &Path, + _audio: NativeAudioSelection, +) -> Result<(), String> { + Err("exact bounded Rewind materialization is available only on macOS".into()) +} + #[cfg(target_os = "macos")] fn concat_mp4_segments(segments: &[PathBuf], output: &Path) -> Result<(), String> { + let slices = segments + .iter() + .cloned() + .map(|path| NativeMediaSlice { + path, + system_audio_path: None, + microphone_path: None, + start_ms: 0, + end_ms: u64::MAX, + }) + .collect::>(); + compose_mp4_slices_impl(&slices, output, true) +} + +#[cfg(target_os = "macos")] +struct PreparedSliceFiles(Vec); + +#[cfg(target_os = "macos")] +impl Drop for PreparedSliceFiles { + fn drop(&mut self) { + for path in &self.0 { + let _ = std::fs::remove_file(path); + } + } +} + +#[cfg(target_os = "macos")] +fn prepare_audio_sidecar_slices( + slices: &[NativeMediaSlice], + output: &Path, +) -> Result<(Vec, PreparedSliceFiles), String> { + let ffmpeg = resolve_ffmpeg_path() + .ok_or_else(|| "Rewind audio materialization requires ffmpeg".to_string())?; + let mut prepared = Vec::with_capacity(slices.len()); + let mut temporary = Vec::new(); + for (index, slice) in slices.iter().enumerate() { + if slice.system_audio_path.is_none() && slice.microphone_path.is_none() { + prepared.push(slice.clone()); + continue; + } + let system = slice + .system_audio_path + .as_ref() + .ok_or_else(|| "Rewind system-audio sidecar is missing".to_string())?; + let microphone = slice + .microphone_path + .as_ref() + .ok_or_else(|| "Rewind microphone sidecar is missing".to_string())?; + for path in [&slice.path, system, microphone] { + if !path.exists() { + return Err(format!( + "Rewind media unit is incomplete: {}", + path.display() + )); + } + } + let muxed = output.with_extension(format!("sidecar-{index}.mp4")); + let _ = std::fs::remove_file(&muxed); + let child = Command::new(&ffmpeg) + .args(["-y", "-v", "error", "-nostdin", "-i"]) + .arg(&slice.path) + .arg("-i") + .arg(system) + .arg("-i") + .arg(microphone) + .args([ + "-map", + "0:v:0", + "-map", + "1:a:0", + "-map", + "2:a:0", + "-c:v", + "copy", + "-c:a", + "aac", + "-af", + "apad", + "-shortest", + "-movflags", + "+faststart", + ]) + .arg(&muxed) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| format!("Rewind sidecar mux spawn failed: {error}"))?; + wait_for_transcode_child(child, FFMPEG_TIMEOUT, "Rewind sidecar mux")?; + let mut prepared_slice = slice.clone(); + prepared_slice.path = muxed.clone(); + prepared_slice.system_audio_path = None; + prepared_slice.microphone_path = None; + prepared.push(prepared_slice); + temporary.push(muxed); + } + Ok((prepared, PreparedSliceFiles(temporary))) +} + +const MAX_PREPARED_SLICE_DURATION_DRIFT_MS: u64 = 1_000; + +fn bounded_slice_range_ms( + start_ms: u64, + end_ms: u64, + asset_duration_ms: u64, +) -> Result<(u64, u64), String> { + if start_ms >= end_ms || start_ms >= asset_duration_ms { + return Err("media slice range is invalid or exceeds source".to_string()); + } + if end_ms <= asset_duration_ms { + return Ok((start_ms, end_ms)); + } + let drift_ms = end_ms.saturating_sub(asset_duration_ms); + if drift_ms > MAX_PREPARED_SLICE_DURATION_DRIFT_MS { + return Err("media slice range is invalid or exceeds source".to_string()); + } + // Muxing the local PCM sidecars to AAC can shorten the container, while + // graph-clock segment boundaries can also lead the encoded media PTS by a + // fraction of a second during writer startup/finalization. Clamp only that + // bounded trailing drift; larger mismatches still fail closed so a + // genuinely wrong source/range cannot be materialized. + Ok((start_ms, asset_duration_ms)) +} + +#[cfg(target_os = "macos")] +fn compose_mp4_slices_impl( + slices: &[NativeMediaSlice], + output: &Path, + allow_full_source_wrapper: bool, +) -> Result<(), String> { use std::ffi::CString; use std::sync::mpsc; use std::time::Duration as StdDuration; @@ -5765,6 +6523,23 @@ fn concat_mp4_segments(segments: &[PathBuf], output: &Path) -> Result<(), String } } + unsafe fn track_at( + asset: &AnyObject, + media_type: *const AnyObject, + index: usize, + ) -> Option<*mut AnyObject> { + let tracks: *mut AnyObject = msg_send![asset, tracksWithMediaType: media_type]; + if tracks.is_null() { + return None; + } + let count: usize = msg_send![tracks, count]; + if index >= count { + return None; + } + let track: *mut AnyObject = msg_send![tracks, objectAtIndex: index]; + (!track.is_null()).then_some(track) + } + unsafe fn cmtime_add(a: CMTime, b: CMTime) -> CMTime { #[link(name = "CoreMedia", kind = "framework")] extern "C" { @@ -5773,10 +6548,12 @@ fn concat_mp4_segments(segments: &[PathBuf], output: &Path) -> Result<(), String CMTimeAdd(a, b) } - if segments.is_empty() { - return Err("concat called with no segments".into()); + if slices.is_empty() { + return Err("compose called with no media slices".into()); } + let (slices, _prepared_slice_files) = prepare_audio_sidecar_slices(slices, output)?; + unsafe { let composition_cls = class_named("AVMutableComposition") .ok_or_else(|| "AVMutableComposition missing".to_string())?; @@ -5797,6 +6574,11 @@ fn concat_mp4_segments(segments: &[PathBuf], output: &Path) -> Result<(), String addMutableTrackWithMediaType: AVMediaTypeAudio, preferredTrackID: KCM_PERSISTENT_TRACK_ID_INVALID ]; + let audio_track_2: *mut AnyObject = msg_send![ + &*composition, + addMutableTrackWithMediaType: AVMediaTypeAudio, + preferredTrackID: KCM_PERSISTENT_TRACK_ID_INVALID + ]; if video_track.is_null() && audio_track.is_null() { return Err("composition has no tracks to write into".into()); } @@ -5805,8 +6587,10 @@ fn concat_mp4_segments(segments: &[PathBuf], output: &Path) -> Result<(), String let asset_cls = class_named("AVURLAsset").ok_or_else(|| "AVURLAsset missing".to_string())?; let mut appended_any = false; + let mut expected_track_shape: Option<(bool, bool, bool)> = None; - for path in segments { + for slice in &slices { + let path = &slice.path; validate_recording_segment_file(path)?; let url = file_url(path) .ok_or_else(|| format!("could not build NSURL for {}", path.display()))?; @@ -5824,13 +6608,71 @@ fn concat_mp4_segments(segments: &[PathBuf], output: &Path) -> Result<(), String path.display() )); } - let range = CMTimeRange { - start: CM_TIME_ZERO, - duration, + let range = if allow_full_source_wrapper && slice.end_ms == u64::MAX { + CMTimeRange { + start: CM_TIME_ZERO, + duration, + } + } else { + let asset_duration_ms = (duration.value as i128) + .saturating_mul(1000) + .checked_div(duration.timescale as i128) + .unwrap_or(0) + .max(0) as u64; + let (start_ms, end_ms) = + bounded_slice_range_ms(slice.start_ms, slice.end_ms, asset_duration_ms) + .map_err(|message| { + format!( + "{message}: {} (requested={}..{}ms source={}ms)", + path.display(), + slice.start_ms, + slice.end_ms, + asset_duration_ms + ) + })?; + CMTimeRange { + start: CMTime { + value: start_ms as i64, + timescale: 1000, + flags: 1, + epoch: 0, + }, + duration: CMTime { + value: end_ms.saturating_sub(start_ms) as i64, + timescale: 1000, + flags: 1, + epoch: 0, + }, + } }; + let seg_video = first_track(&*asset, AVMediaTypeVideo); + let seg_audio = first_track(&*asset, AVMediaTypeAudio); + let seg_audio_2 = track_at(&*asset, AVMediaTypeAudio, 1); + let shape = ( + seg_video.is_some(), + seg_audio.is_some(), + seg_audio_2.is_some(), + ); + if shape == (false, false, false) { + return Err(format!( + "media slice has no compatible tracks: {}", + path.display() + )); + } + if let Some(expected) = expected_track_shape { + if shape != expected { + return Err(format!( + "media slices have incompatible audio/video tracks: {}", + path.display() + )); + } + } else { + expected_track_shape = Some(shape); + } + if !video_track.is_null() { - if let Some(seg_video) = first_track(&*asset, AVMediaTypeVideo) { + if let Some(seg_video) = seg_video { let mut err_ptr: *mut AnyObject = std::ptr::null_mut(); let ok: bool = msg_send![ video_track, @@ -5848,7 +6690,7 @@ fn concat_mp4_segments(segments: &[PathBuf], output: &Path) -> Result<(), String } } if !audio_track.is_null() { - if let Some(seg_audio) = first_track(&*asset, AVMediaTypeAudio) { + if let Some(seg_audio) = seg_audio { let mut err_ptr: *mut AnyObject = std::ptr::null_mut(); let ok: bool = msg_send![ audio_track, @@ -5865,7 +6707,16 @@ fn concat_mp4_segments(segments: &[PathBuf], output: &Path) -> Result<(), String } } } - cursor = cmtime_add(cursor, duration); + if !audio_track_2.is_null() { + if let Some(seg_audio) = seg_audio_2 { + let mut err_ptr: *mut AnyObject = std::ptr::null_mut(); + let ok: bool = msg_send![audio_track_2, insertTimeRange: range, ofTrack: seg_audio, atTime: cursor, error: &mut err_ptr]; + if !ok { + return Err(format!("AVMutableCompositionTrack insertTimeRange (second audio) failed for {}", path.display())); + } + } + } + cursor = cmtime_add(cursor, range.duration); appended_any = true; } @@ -5928,6 +6779,64 @@ fn concat_mp4_segments(segments: &[PathBuf], output: &Path) -> Result<(), String Ok(()) } +#[cfg(test)] +mod native_media_slice_tests { + use super::*; + + fn slice(path: &str, start_ms: u64, end_ms: u64) -> NativeMediaSlice { + NativeMediaSlice { + path: PathBuf::from(path), + system_audio_path: None, + microphone_path: None, + start_ms, + end_ms, + } + } + + #[test] + fn plans_adjacent_exact_ranges_and_duration() { + let plan = plan_native_media_slices( + &[slice("one.mp4", 0, 125), slice("two.mp4", 125, 300)], + &[125, 300], + ) + .unwrap(); + assert_eq!( + plan.iter() + .map(|slice| slice.end_ms - slice.start_ms) + .sum::(), + 300 + ); + } + + #[test] + fn clamps_only_small_trailing_mux_duration_drift() { + assert_eq!(bounded_slice_range_ms(100, 900, 900).unwrap(), (100, 900)); + assert_eq!(bounded_slice_range_ms(100, 900, 867).unwrap(), (100, 867)); + assert_eq!( + bounded_slice_range_ms(100, 1_172, 1_000).unwrap(), + (100, 1_000) + ); + assert!(bounded_slice_range_ms(100, 1_200, 199).is_err()); + assert!(bounded_slice_range_ms(900, 950, 900).is_err()); + } + + #[test] + fn rejects_gap_overlap_missing_and_out_of_bounds_ranges() { + assert!(plan_native_media_slices( + &[slice("one.mp4", 0, 100), slice("two.mp4", 125, 200)], + &[100, 200] + ) + .is_err()); + assert!(plan_native_media_slices( + &[slice("one.mp4", 0, 100), slice("two.mp4", 90, 200)], + &[100, 200] + ) + .is_err()); + assert!(plan_native_media_slices(&[slice("", 0, 10)], &[10]).is_err()); + assert!(plan_native_media_slices(&[slice("one.mp4", 0, 11)], &[10]).is_err()); + } +} + #[cfg(test)] mod audio_track_probe_tests { use super::{ @@ -5959,14 +6868,16 @@ mod audio_track_probe_tests { } fn write_temp_mp4(bytes: &[u8]) -> std::path::PathBuf { + static TEMP_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let mut path = std::env::temp_dir(); path.push(format!( - "clips-audio-probe-test-{}-{}.mp4", + "clips-audio-probe-test-{}-{}-{}.mp4", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + TEMP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed), )); let mut f = std::fs::File::create(&path).unwrap(); f.write_all(bytes).unwrap(); diff --git a/templates/clips/desktop/src-tauri/src/native_screen/custom_capture.rs b/templates/clips/desktop/src-tauri/src/native_screen/custom_capture.rs index a9474ee151..852fd96cc3 100644 --- a/templates/clips/desktop/src-tauri/src/native_screen/custom_capture.rs +++ b/templates/clips/desktop/src-tauri/src/native_screen/custom_capture.rs @@ -11,6 +11,7 @@ use super::*; use screencapturekit::error::SCError; use screencapturekit::stream::delegate_trait::SCStreamDelegateTrait; +use std::io::{Seek, SeekFrom, Write}; /// `AVAssetWriter.status` raw value for `.completed`. const AV_WRITER_STATUS_COMPLETED: i64 = 2; @@ -38,6 +39,13 @@ const CAPTURE_VIDEO_BPP: f64 = 0.15; const CAPTURE_STALL_TIMEOUT: Duration = Duration::from_secs(4); // Watchdog poll cadence. const CAPTURE_WATCHDOG_POLL: Duration = Duration::from_millis(1000); +/// A stream is not healthy merely because `start_capture` returned. Rewind's +/// fragmented writer must receive a usable video frame and emit real media. +const CAPTURE_FIRST_SAMPLE_TIMEOUT: Duration = Duration::from_secs(4); +const CAPTURE_FIRST_FRAGMENT_TIMEOUT: Duration = Duration::from_secs(3); +/// Keep a small PCM tail unwritten so a fence can still divide callbacks that +/// arrived ahead of the corresponding encoded video fragment report. +const AUDIO_FENCE_LOOKBEHIND_SECONDS: f64 = 2.0; // Consecutive failed restarts (rebuild or start error, or an immediate // re-stall) before giving up and finalizing whatever was captured. A single // successful stretch of frames resets the counter. @@ -62,6 +70,10 @@ pub(crate) struct CaptureWatch { /// the watchdog must not read the silence as a stall and rebuild — resume /// brings a fresh stream back and clears this. paused: AtomicBool, + screen_samples: AtomicU64, + usable_screen_samples: AtomicU64, + system_audio_samples: AtomicU64, + microphone_samples: AtomicU64, } impl CaptureWatch { @@ -71,6 +83,10 @@ impl CaptureWatch { stream_stopped: Mutex::new(None), user_stopped: AtomicBool::new(false), paused: AtomicBool::new(false), + screen_samples: AtomicU64::new(0), + usable_screen_samples: AtomicU64::new(0), + system_audio_samples: AtomicU64::new(0), + microphone_samples: AtomicU64::new(0), } } @@ -80,6 +96,33 @@ impl CaptureWatch { } } + fn note_sample(&self, of_type: SCStreamOutputType, usable_screen: bool) { + let (counter, label) = match of_type { + SCStreamOutputType::Screen => (&self.screen_samples, "screen"), + SCStreamOutputType::Audio => (&self.system_audio_samples, "system-audio"), + SCStreamOutputType::Microphone => (&self.microphone_samples, "microphone"), + }; + let count = counter.fetch_add(1, Ordering::Relaxed) + 1; + if count == 1 { + crate::logfile::diagnostic(&format!("[capture-health] first {label} sample callback")); + } + if usable_screen { + let usable = self.usable_screen_samples.fetch_add(1, Ordering::Relaxed) + 1; + if usable == 1 { + crate::logfile::diagnostic("[capture-health] first usable screen sample"); + } + } + } + + fn sample_counts(&self) -> (u64, u64, u64, u64) { + ( + self.screen_samples.load(Ordering::Relaxed), + self.usable_screen_samples.load(Ordering::Relaxed), + self.system_audio_samples.load(Ordering::Relaxed), + self.microphone_samples.load(Ordering::Relaxed), + ) + } + fn since_activity(&self) -> Duration { self.last_activity .lock() @@ -130,15 +173,460 @@ impl CaptureWatch { // can safely tail it forever. // --------------------------------------------------------------------------- -/// Owns the local recording file in segmented mode. The ObjC delegate appends -/// every segment it receives; append order matches delivery order (delegate -/// callbacks are serial, and the file mutex serializes any stragglers). +/// Apple's `AVAssetWriterSegmentType` values. Apple documents initialization +/// data as `1` and separable media data as `2`; keeping this distinction is +/// what lets each logical output remain a standalone fMP4. +const AV_ASSET_WRITER_SEGMENT_TYPE_INITIALIZATION: isize = 1; +const AV_ASSET_WRITER_SEGMENT_TYPE_SEPARABLE: isize = 2; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum CustomWriterOutput { + Standard, + RewindCmaf, + ClipHls, +} + +impl CustomWriterOutput { + fn segmented(self) -> bool { + !matches!(self, Self::Standard) + } + + fn preserves_separate_audio(self) -> bool { + matches!(self, Self::RewindCmaf) + } +} + +fn segmented_output_enabled(output: CustomWriterOutput, live_upload_enabled: bool) -> bool { + output.segmented() || live_upload_enabled +} + +fn live_audio_mixing_enabled( + output: CustomWriterOutput, + include_audio: bool, + capture_system_audio: bool, +) -> bool { + include_audio && capture_system_audio && !output.preserves_separate_audio() +} + +pub(crate) fn audio_sidecar_path(video_path: &Path, source: &str) -> PathBuf { + let stem = video_path + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or("segment"); + video_path.with_file_name(format!("{stem}.{source}.wav")) +} + +struct FloatWavWriter { + file: std::fs::File, + data_bytes: u32, + sample_rate: u32, +} + +impl FloatWavWriter { + fn create(path: &Path, sample_rate: u32) -> Result { + let mut file = std::fs::File::create(path) + .map_err(|error| format!("audio sidecar create failed: {error}"))?; + file.write_all(&[0u8; 44]) + .map_err(|error| format!("audio sidecar header reserve failed: {error}"))?; + Ok(Self { + file, + data_bytes: 0, + sample_rate, + }) + } + + fn append_at(&mut self, samples: &[f32], start_seconds: f64) -> Result<(), String> { + let target = (start_seconds.max(0.0) * self.sample_rate as f64).round() as usize; + let current = self.data_bytes as usize / 4; + if target > current { + for _ in 0..target.saturating_sub(current) { + self.file + .write_all(&0.0f32.to_le_bytes()) + .map_err(|error| format!("audio sidecar silence write failed: {error}"))?; + } + self.data_bytes = self + .data_bytes + .saturating_add((target.saturating_sub(current) as u32).saturating_mul(4)); + } + let current = self.data_bytes as usize / 4; + let skip = current.saturating_sub(target).min(samples.len()); + for sample in &samples[skip..] { + self.file + .write_all(&sample.to_le_bytes()) + .map_err(|error| format!("audio sidecar write failed: {error}"))?; + } + self.data_bytes = self + .data_bytes + .saturating_add(((samples.len() - skip) as u32).saturating_mul(4)); + Ok(()) + } + + fn finish(mut self) -> Result<(), String> { + let riff_size = 36u32.saturating_add(self.data_bytes); + let byte_rate = self.sample_rate.saturating_mul(4); + let mut header = Vec::with_capacity(44); + header.extend_from_slice(b"RIFF"); + header.extend_from_slice(&riff_size.to_le_bytes()); + header.extend_from_slice(b"WAVEfmt "); + header.extend_from_slice(&16u32.to_le_bytes()); + header.extend_from_slice(&3u16.to_le_bytes()); + header.extend_from_slice(&1u16.to_le_bytes()); + header.extend_from_slice(&self.sample_rate.to_le_bytes()); + header.extend_from_slice(&byte_rate.to_le_bytes()); + header.extend_from_slice(&4u16.to_le_bytes()); + header.extend_from_slice(&32u16.to_le_bytes()); + header.extend_from_slice(b"data"); + header.extend_from_slice(&self.data_bytes.to_le_bytes()); + self.file + .seek(SeekFrom::Start(0)) + .and_then(|_| self.file.write_all(&header)) + .and_then(|_| self.file.flush()) + .and_then(|_| self.file.sync_all()) + .map_err(|error| format!("audio sidecar finalize failed: {error}")) + } +} + +struct AudioSidecarState { + system: Option, + microphone: Option, + failed: Option, + segment_base_pts: Option, + latest_end_pts: Option, + session_start_pts: Option, + fence_pending: bool, + queued: std::collections::VecDeque, +} + +struct QueuedAudio { + microphone: bool, + samples: Vec, + sample_rate: f64, + pts_seconds: f64, +} + +struct AudioSidecarManager { + state: Mutex, + sources: crate::capture_audio_bus::AudioSources, +} + +impl AudioSidecarManager { + fn create( + video_path: &Path, + sources: crate::capture_audio_bus::AudioSources, + ) -> Result, String> { + let system = sources + .system + .then(|| FloatWavWriter::create(&audio_sidecar_path(video_path, "system"), 48_000)) + .transpose()?; + let microphone = sources + .microphone + .then(|| FloatWavWriter::create(&audio_sidecar_path(video_path, "microphone"), 48_000)) + .transpose()?; + Ok(Arc::new(Self { + state: Mutex::new(AudioSidecarState { + system, + microphone, + failed: None, + segment_base_pts: None, + latest_end_pts: None, + session_start_pts: None, + fence_pending: false, + queued: std::collections::VecDeque::new(), + }), + sources, + })) + } + + fn append( + &self, + microphone: bool, + samples: &[f32], + sample_rate: f64, + pts_seconds: f64, + session_start_seconds: f64, + ) { + let Ok(mut state) = self.state.lock() else { + return; + }; + if state.failed.is_some() { + return; + } + if (sample_rate - 48_000.0).abs() > 1.0 { + state.failed = Some(format!( + "audio sidecar received unsupported sample rate {sample_rate}" + )); + return; + } + state.session_start_pts.get_or_insert(session_start_seconds); + state.queued.push_back(QueuedAudio { + microphone, + samples: samples.to_vec(), + sample_rate, + pts_seconds, + }); + if !state.fence_pending { + let cutoff = pts_seconds - AUDIO_FENCE_LOOKBEHIND_SECONDS; + Self::flush_complete_buffers_before(&mut state, cutoff, session_start_seconds); + } + } + + fn flush_complete_buffers_before( + state: &mut AudioSidecarState, + cutoff_pts: f64, + session_start_seconds: f64, + ) { + while state.queued.front().is_some_and(|buffer| { + buffer.pts_seconds + buffer.samples.len() as f64 / buffer.sample_rate <= cutoff_pts + }) { + let buffer = state.queued.pop_front().expect("front checked above"); + Self::append_locked( + state, + buffer.microphone, + &buffer.samples, + buffer.sample_rate, + buffer.pts_seconds, + session_start_seconds, + ); + } + } + + fn flush_all(state: &mut AudioSidecarState, session_start_seconds: f64) { + let queued = std::mem::take(&mut state.queued); + for buffer in queued { + Self::append_locked( + state, + buffer.microphone, + &buffer.samples, + buffer.sample_rate, + buffer.pts_seconds, + session_start_seconds, + ); + } + } + + fn append_locked( + state: &mut AudioSidecarState, + microphone: bool, + samples: &[f32], + sample_rate: f64, + pts_seconds: f64, + session_start_seconds: f64, + ) { + let base = *state.segment_base_pts.get_or_insert(session_start_seconds); + // ScreenCaptureKit callbacks from different output queues can arrive + // after a fragment fence even when their PTS belongs to the preceding + // fragment. Discard only the portion before this segment's video + // boundary instead of incorrectly prepending it to the new sidecar. + let skip = (((base - pts_seconds) * sample_rate).ceil() as isize) + .clamp(0, samples.len() as isize) as usize; + let samples = &samples[skip..]; + if samples.is_empty() { + return; + } + let pts_seconds = pts_seconds + skip as f64 / sample_rate; + let start_seconds = (pts_seconds - base).max(0.0); + state.latest_end_pts = Some( + state + .latest_end_pts + .unwrap_or(pts_seconds) + .max(pts_seconds + samples.len() as f64 / sample_rate), + ); + let writer = if microphone { + state.microphone.as_mut() + } else { + state.system.as_mut() + }; + if let Some(writer) = writer { + if let Err(error) = writer.append_at(samples, start_seconds) { + state.failed = Some(error); + } + } + } + + fn begin_fence(&self) -> Result<(), String> { + let mut state = self.state.lock().map_err(|error| error.to_string())?; + if let Some(error) = state.failed.clone() { + return Err(error); + } + if state.fence_pending { + return Err("an audio sidecar fence is already pending".into()); + } + state.fence_pending = true; + Ok(()) + } + + fn complete_fence(&self, next_video_path: &Path, boundary_seconds: f64) -> Result<(), String> { + let mut state = self.state.lock().map_err(|error| error.to_string())?; + if let Some(error) = state.failed.clone() { + return Err(error); + } + if !state.fence_pending { + return Err("audio sidecar fence completed without a pending fence".into()); + } + let session_start = state.session_start_pts.unwrap_or(0.0); + let boundary_pts = session_start + boundary_seconds; + let queued = std::mem::take(&mut state.queued); + for buffer in &queued { + let before = (((boundary_pts - buffer.pts_seconds) * buffer.sample_rate).round() + as isize) + .clamp(0, buffer.samples.len() as isize) as usize; + if before > 0 { + Self::append_locked( + &mut state, + buffer.microphone, + &buffer.samples[..before], + buffer.sample_rate, + buffer.pts_seconds, + session_start, + ); + } + } + if let Some(writer) = state.system.take() { + writer.finish()?; + } + if let Some(writer) = state.microphone.take() { + writer.finish()?; + } + state.system = self + .sources + .system + .then(|| FloatWavWriter::create(&audio_sidecar_path(next_video_path, "system"), 48_000)) + .transpose()?; + state.microphone = self + .sources + .microphone + .then(|| { + FloatWavWriter::create(&audio_sidecar_path(next_video_path, "microphone"), 48_000) + }) + .transpose()?; + state.segment_base_pts = Some(boundary_pts); + state.fence_pending = false; + for buffer in queued { + let after = (((boundary_pts - buffer.pts_seconds) * buffer.sample_rate).round() + as isize) + .clamp(0, buffer.samples.len() as isize) as usize; + if after < buffer.samples.len() { + Self::append_locked( + &mut state, + buffer.microphone, + &buffer.samples[after..], + buffer.sample_rate, + buffer.pts_seconds + after as f64 / buffer.sample_rate, + boundary_pts, + ); + } + } + crate::logfile::diagnostic(&format!( + "[capture-health] audio sidecars fenced at writer PTS {boundary_seconds:.6}s" + )); + Ok(()) + } + + fn cancel_fence(&self) { + let Ok(mut state) = self.state.lock() else { + return; + }; + if !state.fence_pending { + return; + } + let session_start = state.session_start_pts.unwrap_or(0.0); + state.fence_pending = false; + Self::flush_all(&mut state, session_start); + } + + fn finish(&self) -> Result<(), String> { + self.cancel_fence(); + let mut state = self.state.lock().map_err(|error| error.to_string())?; + let session_start = state.session_start_pts.unwrap_or(0.0); + Self::flush_all(&mut state, session_start); + if let Some(writer) = state.system.take() { + writer.finish()?; + } + if let Some(writer) = state.microphone.take() { + writer.finish()?; + } + state.failed.clone().map_or(Ok(()), Err) + } +} + +/// Metadata for a logical fMP4 closed at a media-fragment boundary. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct ClosedSegmentFile { + pub path: PathBuf, + pub sequence: u64, + pub media_fragments: u64, + pub bytes_written: u64, + pub boundary_seconds: Option, +} + +/// A non-blocking fence request. Await it off AVFoundation's delegate queue. +pub(crate) struct SegmentFence { + result: std::sync::mpsc::Receiver>, + audio_sidecars: Option<(Arc, PathBuf)>, +} + +impl SegmentFence { + pub(crate) fn wait(self, timeout: Duration) -> Result { + match self.result.recv_timeout(timeout) { + Ok(Ok(closed)) => { + if let Some((sidecars, next_path)) = self.audio_sidecars { + let boundary = closed.boundary_seconds.ok_or_else(|| { + sidecars.cancel_fence(); + "AVAssetWriter did not report the video fragment boundary".to_string() + })?; + sidecars.complete_fence(&next_path, boundary)?; + } + Ok(closed) + } + Ok(Err(error)) => { + if let Some((sidecars, _)) = self.audio_sidecars { + sidecars.cancel_fence(); + } + Err(error) + } + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + if let Some((sidecars, _)) = self.audio_sidecars { + sidecars.cancel_fence(); + } + Err("fragment fence timed out".into()) + } + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + if let Some((sidecars, _)) = self.audio_sidecars { + sidecars.cancel_fence(); + } + Err("fragment fence was cancelled before completion".into()) + } + } + } +} + +struct PendingFence { + next_path: PathBuf, + completion: std::sync::mpsc::Sender>, +} + +struct SegmentSinkState { + file: std::fs::File, + path: PathBuf, + sequence: u64, + bytes_written: u64, + media_fragments: u64, + init_segment: Option>, + pending_fences: std::collections::VecDeque, + failed: Option, +} + +#[derive(Clone, Copy, Debug, Default)] +struct SegmentProgress { + bytes_written: u64, + media_fragments: u64, + has_initialization: bool, +} + +/// Delegate callbacks and fence requests share one mutex, making every media +/// fragment an indivisible routing unit. No fragment can cross two files. pub(super) struct SegmentSink { - file: Mutex, - /// First write error, if any; surfaced by `finish()` so a failing disk - /// turns into a reported error instead of a silently truncated file. - failed: Mutex>, - segments: AtomicU64, + state: Mutex, } impl SegmentSink { @@ -146,37 +634,182 @@ impl SegmentSink { let file = std::fs::File::create(path) .map_err(|e| format!("could not create recording file {}: {e}", path.display()))?; Ok(Arc::new(Self { - file: Mutex::new(file), - failed: Mutex::new(None), - segments: AtomicU64::new(0), + state: Mutex::new(SegmentSinkState { + file, + path: path.to_path_buf(), + sequence: 0, + bytes_written: 0, + media_fragments: 0, + init_segment: None, + pending_fences: std::collections::VecDeque::new(), + failed: None, + }), })) } - fn append(&self, bytes: &[u8]) { + fn fail(state: &mut SegmentSinkState, error: String) { + if state.failed.is_none() { + state.failed = Some(error.clone()); + } + while let Some(fence) = state.pending_fences.pop_front() { + let _ = fence.completion.send(Err(error.clone())); + } + } + + fn write_current(state: &mut SegmentSinkState, bytes: &[u8]) -> Result<(), String> { use std::io::Write; - let Ok(mut file) = self.file.lock() else { - return; - }; - if let Err(err) = file.write_all(bytes) { - if let Ok(mut failed) = self.failed.lock() { - if failed.is_none() { - eprintln!("[mixer] segment write failed: {err}"); - *failed = Some(format!("segment write failed: {err}")); - } + state + .file + .write_all(bytes) + .map_err(|err| format!("segment write failed: {err}"))?; + state.bytes_written = state.bytes_written.saturating_add(bytes.len() as u64); + Ok(()) + } + + fn failure(&self) -> Option { + self.state + .lock() + .ok() + .and_then(|guard| guard.failed.clone()) + } + + fn progress(&self) -> SegmentProgress { + self.state + .lock() + .map(|state| SegmentProgress { + bytes_written: state.bytes_written, + media_fragments: state.media_fragments, + has_initialization: state.init_segment.is_some(), + }) + .unwrap_or_default() + } + + fn cancel_pending(&self, error: &str) { + if let Ok(mut state) = self.state.lock() { + while let Some(fence) = state.pending_fences.pop_front() { + let _ = fence.completion.send(Err(error.to_owned())); } - return; } - let n = self.segments.fetch_add(1, Ordering::Relaxed) + 1; - if n == 1 { - eprintln!( - "[mixer] first output segment written ({} bytes)", - bytes.len() - ); + } + + pub(crate) fn fence(&self, next_path: PathBuf) -> Result { + let mut state = self.state.lock().map_err(|e| e.to_string())?; + if let Some(error) = state.failed.clone() { + return Err(error); } + if next_path == state.path + || state + .pending_fences + .iter() + .any(|f| f.next_path == next_path) + { + return Err("fragment fence path is already active or pending".into()); + } + let (completion, result) = std::sync::mpsc::channel(); + state.pending_fences.push_back(PendingFence { + next_path, + completion, + }); + Ok(SegmentFence { + result, + audio_sidecars: None, + }) } - fn failure(&self) -> Option { - self.failed.lock().ok().and_then(|guard| guard.clone()) + fn append(&self, bytes: &[u8], segment_type: isize, boundary_seconds: Option) { + let Ok(mut state) = self.state.lock() else { + return; + }; + if state.failed.is_some() { + return; + } + match segment_type { + AV_ASSET_WRITER_SEGMENT_TYPE_INITIALIZATION => { + state.init_segment = Some(bytes.to_vec()); + crate::logfile::diagnostic(&format!( + "[capture-health] writer initialization segment: {} bytes", + bytes.len() + )); + if state.media_fragments == 0 && state.bytes_written == 0 { + if let Err(error) = Self::write_current(&mut state, bytes) { + Self::fail(&mut state, error); + } + } + } + AV_ASSET_WRITER_SEGMENT_TYPE_SEPARABLE => { + let Some(init) = state.init_segment.clone() else { + Self::fail( + &mut state, + "media fragment arrived before initialization segment".into(), + ); + return; + }; + // Do not close an init-only file. Once the current logical + // file has media, service exactly one request at each later + // boundary; repeated fences then make consecutive non-empty + // files without splitting a fragment. + if state.media_fragments > 0 && !state.pending_fences.is_empty() { + let fence = state.pending_fences.pop_front().expect("checked above"); + use std::io::Write; + let closed = ClosedSegmentFile { + path: state.path.clone(), + sequence: state.sequence, + media_fragments: state.media_fragments, + bytes_written: state.bytes_written, + boundary_seconds, + }; + let rotate = (|| -> Result<(), String> { + state + .file + .flush() + .map_err(|e| format!("segment flush failed: {e}"))?; + state + .file + .sync_all() + .map_err(|e| format!("segment sync failed: {e}"))?; + let mut next = std::fs::File::create(&fence.next_path).map_err(|e| { + format!( + "could not create fenced recording file {}: {e}", + fence.next_path.display() + ) + })?; + next.write_all(&init) + .map_err(|e| format!("segment init write failed: {e}"))?; + state.file = next; + state.path = fence.next_path.clone(); + state.sequence = state.sequence.saturating_add(1); + state.bytes_written = init.len() as u64; + state.media_fragments = 0; + Self::write_current(&mut state, bytes)?; + state.media_fragments = 1; + Ok(()) + })(); + match rotate { + Ok(()) => { + let _ = fence.completion.send(Ok(closed)); + } + Err(error) => { + let _ = fence.completion.send(Err(error.clone())); + Self::fail(&mut state, error); + } + } + } else if let Err(error) = Self::write_current(&mut state, bytes) { + Self::fail(&mut state, error); + } else { + state.media_fragments = state.media_fragments.saturating_add(1); + if state.media_fragments == 1 { + crate::logfile::diagnostic(&format!( + "[capture-health] first writer media fragment: {} bytes", + bytes.len() + )); + } + } + } + _ => Self::fail( + &mut state, + format!("unsupported AVAssetWriter segment type {segment_type}"), + ), + } } } @@ -196,12 +829,13 @@ objc2::define_class!( impl SegmentWriterDelegate { /// `AVAssetWriterDelegate` — receives each fMP4 segment (type 1 = /// initialization, 2 = separable media) as it is produced. - #[unsafe(method(assetWriter:didOutputSegmentData:segmentType:))] + #[unsafe(method(assetWriter:didOutputSegmentData:segmentType:segmentReport:))] fn did_output_segment( &self, _writer: *mut objc2::runtime::AnyObject, data: *mut objc2::runtime::AnyObject, - _segment_type: isize, + segment_type: isize, + segment_report: *mut objc2::runtime::AnyObject, ) { // Crossing the ObjC boundary: a Rust panic here would abort the // whole process, so contain it. @@ -219,8 +853,24 @@ objc2::define_class!( return; } let bytes = unsafe { std::slice::from_raw_parts(ptr as *const u8, len) }; + let boundary_seconds = if segment_report.is_null() { + None + } else { + let reports: *mut objc2::runtime::AnyObject = unsafe { + objc2::msg_send![&*segment_report, trackReports] + }; + let count: usize = if reports.is_null() { 0 } else { unsafe { objc2::msg_send![&*reports, count] } }; + if count == 0 { + None + } else { + let track: *mut objc2::runtime::AnyObject = unsafe { objc2::msg_send![&*reports, objectAtIndex: 0usize] }; + let earliest: ObjcCMTime = unsafe { objc2::msg_send![&*track, earliestPresentationTimeStamp] }; + ((earliest.flags & 1) != 0 && earliest.timescale > 0) + .then_some(earliest.value as f64 / earliest.timescale as f64) + } + }; use objc2::DefinedClass; - self.ivars().sink.append(bytes); + self.ivars().sink.append(bytes, segment_type, boundary_seconds); })); } } @@ -265,6 +915,15 @@ pub(crate) struct CustomScreenCaptureWriter { /// append-only file. Zero until the first resume. Read by the video /// retime path and the live audio mixer. pause_offset_bits: Arc, + /// Present for Rewind's one physical ScreenCaptureKit producer. Clones + /// share one registration across watchdog rebuilds; `finish` deactivates + /// it before capture teardown so new consumers cannot attach to a dying + /// stream. + audio_producer: Option, + /// Rewind persists independently selectable microphone and system PCM + /// beside the video-only fMP4 because AVAssetWriter's segmented profiles + /// reject a writer graph containing two AAC inputs. + audio_sidecars: Option>, } /// The lock-guarded half of the writer: the retained AVFoundation objects @@ -320,11 +979,260 @@ unsafe impl Send for CustomScreenCaptureWriterState {} /// keeps receiving samples over the whole recording. struct CustomScreenCaptureOutputHandler { writer: CustomScreenCaptureWriter, + /// At most one temporary Clips writer may mirror this physical producer. + /// It deliberately lives beside the callback handler rather than the + /// audio bus: the bus has one producer contract and must never publish a + /// second copy just because a Clip starts. + clip_sink: Arc>>, recording_enabled: Arc, mic_ready: Option>, watch: Arc, } +pub(crate) struct ClipSinkSlot { + writer: CustomScreenCaptureWriter, + gate: Arc, +} + +fn install_only_slot(slot: &mut Option, value: T) -> Result<(), String> { + if slot.is_some() { + return Err("a Clip sink is already attached to this capture producer".into()); + } + *slot = Some(value); + Ok(()) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +enum ClipSinkState { + Prepared = 0, + Active = 1, + Paused = 2, + Closed = 3, +} + +struct ClipSinkGate { + state: AtomicU64, + activated_at: Mutex>, + paused_at: Mutex>, + paused_total: Mutex, + logical_end: Mutex>, +} + +impl ClipSinkGate { + fn new() -> Self { + Self { + state: AtomicU64::new(ClipSinkState::Prepared as u64), + activated_at: Mutex::new(None), + paused_at: Mutex::new(None), + paused_total: Mutex::new(Duration::ZERO), + logical_end: Mutex::new(None), + } + } + + fn state(&self) -> ClipSinkState { + match self.state.load(Ordering::SeqCst) { + 1 => ClipSinkState::Active, + 2 => ClipSinkState::Paused, + 3 => ClipSinkState::Closed, + _ => ClipSinkState::Prepared, + } + } + + fn accepts(&self) -> bool { + self.state() == ClipSinkState::Active + } + + fn media_duration_ms(&self) -> u64 { + let started = self.activated_at.lock().ok().and_then(|value| *value); + let Some(started) = started else { + return 0; + }; + let paused_total = self + .paused_total + .lock() + .map(|value| *value) + .unwrap_or_default(); + let current_pause = self + .paused_at + .lock() + .ok() + .and_then(|value| *value) + .map(|value| value.elapsed()) + .unwrap_or_default(); + started + .elapsed() + .saturating_sub(paused_total) + .saturating_sub(current_pause) + .as_millis() as u64 + } +} + +/// Handle for the one temporary Clip writer attached to an existing custom +/// ScreenCaptureKit producer. Preparation allocates the writer but does not +/// admit callbacks; activation is a single in-memory state transition. +pub(crate) struct PreparedClipSink { + slot: Arc>>, + writer: CustomScreenCaptureWriter, + gate: Arc, +} + +impl PreparedClipSink { + pub(crate) fn activate(&self) -> Result<(), String> { + let slot = self.slot.lock().map_err(|error| error.to_string())?; + let installed = slot + .as_ref() + .is_some_and(|installed| Arc::ptr_eq(&installed.gate, &self.gate)); + if !installed || self.gate.state() != ClipSinkState::Prepared { + return Err("Clip sink is not prepared".to_string()); + } + *self + .gate + .activated_at + .lock() + .map_err(|error| error.to_string())? = Some(Instant::now()); + // Publish Active last, while holding the same slot lock callbacks use. + self.gate + .state + .store(ClipSinkState::Active as u64, Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn pause(&self) -> Result<(), String> { + let _slot = self.slot.lock().map_err(|error| error.to_string())?; + if self.gate.state() != ClipSinkState::Active { + return Err("Clip sink is not active".into()); + } + *self + .gate + .paused_at + .lock() + .map_err(|error| error.to_string())? = Some(Instant::now()); + // Close callback admission only after the pause boundary is durable. + self.gate + .state + .store(ClipSinkState::Paused as u64, Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn resume(&self) -> Result<(), String> { + let _slot = self.slot.lock().map_err(|error| error.to_string())?; + if self.gate.state() != ClipSinkState::Paused { + return Err("Clip sink is no longer paused".into()); + } + let paused_at = self + .gate + .paused_at + .lock() + .map_err(|e| e.to_string())? + .take() + .ok_or_else(|| "Clip sink is not paused".to_string())?; + let paused_for = paused_at.elapsed(); + let mut paused_total = self + .gate + .paused_total + .lock() + .map_err(|error| error.to_string())?; + *paused_total = paused_total.saturating_add(paused_for); + self.writer + .set_pause_offset(self.writer.pause_offset() + paused_for.as_secs_f64()); + // Re-open callback admission only after every timestamp offset is in + // place. The slot lock prevents an in-flight append from observing a + // half-resumed writer. + self.gate + .state + .store(ClipSinkState::Active as u64, Ordering::SeqCst); + Ok(()) + } + + /// Logical close is synchronous and precedes the potentially slow writer + /// finalization. This makes Stop's accepted-sample boundary exact. + pub(crate) fn deactivate(&self) { + // The callback holds this same lock across its accepted append. Taking + // it here makes the return from deactivate the exact no-more-samples + // boundary: a callback either completed before logical end or sees + // Closed after it. No disk/network/finalize work occurs under it. + if let Ok(_slot) = self.slot.lock() { + self.gate + .state + .store(ClipSinkState::Closed as u64, Ordering::SeqCst); + } + if let Ok(mut end) = self.gate.logical_end.lock() { + *end = Some(Instant::now()); + } + } + + pub(crate) fn duration_ms(&self) -> u64 { + self.gate.media_duration_ms() + } + + pub(crate) fn finalize(&self) -> Result<(), String> { + self.deactivate(); + let result = self.writer.finish(true); + self.remove(); + result + } + + pub(crate) fn cancel(&self) { + self.deactivate(); + let _ = self.writer.finish(false); + self.remove(); + } + + fn remove(&self) { + if let Ok(mut slot) = self.slot.lock() { + let same_sink = slot + .as_ref() + .is_some_and(|installed| Arc::ptr_eq(&installed.gate, &self.gate)); + if same_sink { + *slot = None; + } + } + } +} + +/// Allocate an Apple-HLS, append-only Clip writer and install it as the only +/// secondary sink on a running physical producer. This does not start any +/// ScreenCaptureKit or audio input and is deliberately independent of the +/// ordinary remote-live-upload feature flag. +pub(crate) fn prepare_clip_sink( + slot: Arc>>, + output_path: &Path, + width: u32, + height: u32, + include_mic: bool, + include_system_audio: bool, +) -> Result { + let mut installed = slot.lock().map_err(|e| e.to_string())?; + if installed.is_some() { + return Err("a Clip sink is already attached to this capture producer".into()); + } + let writer = CustomScreenCaptureWriter::new( + output_path, + width, + height, + include_system_audio, + include_mic, + live_audio_mixing_enabled( + CustomWriterOutput::ClipHls, + include_mic, + include_system_audio, + ), + CustomWriterOutput::ClipHls, + None, + )?; + let gate = Arc::new(ClipSinkGate::new()); + install_only_slot( + &mut installed, + ClipSinkSlot { + writer: writer.clone(), + gate: Arc::clone(&gate), + }, + )?; + drop(installed); + Ok(PreparedClipSink { slot, writer, gate }) +} + impl Clone for CustomScreenCaptureWriter { fn clone(&self) -> Self { Self { @@ -335,6 +1243,8 @@ impl Clone for CustomScreenCaptureWriter { appends_closed: Arc::clone(&self.appends_closed), dropped_samples: Arc::clone(&self.dropped_samples), pause_offset_bits: Arc::clone(&self.pause_offset_bits), + audio_producer: self.audio_producer.clone(), + audio_sidecars: self.audio_sidecars.clone(), } } } @@ -350,6 +1260,22 @@ impl SCStreamOutputTrait for CustomScreenCaptureOutputHandler { mic_ready.store(true, Ordering::Relaxed); } } + // Rewind may already own both physical audio inputs before a meeting + // begins. Publish decoded PCM even while recording output is deferred; + // the bus is about source ownership, not writer attachment. + if matches!( + of_type, + SCStreamOutputType::Audio | SCStreamOutputType::Microphone + ) { + let panic_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = objc2::exception::catch(std::panic::AssertUnwindSafe(|| { + self.writer.publish_audio_sample(&sample_buffer, of_type); + })); + })); + if panic_result.is_err() { + eprintln!("[mixer] panic while publishing shared {of_type:?} PCM"); + } + } if !self.recording_enabled.load(Ordering::SeqCst) { return; } @@ -357,6 +1283,10 @@ impl SCStreamOutputTrait for CustomScreenCaptureOutputHandler { // is still alive; record it so the watchdog can tell a genuinely stalled // stream apart from a quiet one. self.watch.note_activity(); + self.watch.note_sample( + of_type, + matches!(of_type, SCStreamOutputType::Screen) && sample_buffer.image_buffer().is_some(), + ); if self.writer.appends_closed.load(Ordering::SeqCst) { return; } @@ -391,6 +1321,16 @@ impl SCStreamOutputTrait for CustomScreenCaptureOutputHandler { ); } } + // Mirror into the temporary Clip writer only after its atomic + // activation. Do not call `publish_audio_sample` here: this is a + // second consumer of existing callbacks, not another audio producer. + if let Ok(slot) = self.clip_sink.lock() { + if let Some(clip) = slot.as_ref().filter(|clip| clip.gate.accepts()) { + let _ = objc2::exception::catch(std::panic::AssertUnwindSafe(|| { + clip.writer.append_sample(&sample_buffer, of_type); + })); + } + } // ScreenCaptureKit invokes this from its own dispatch queues through an // Objective-C boundary. Two failure modes can abort the whole process: // - an Objective-C exception (e.g. AVFoundation), which `catch_unwind` @@ -430,6 +1370,8 @@ impl CustomScreenCaptureWriter { capture_system_audio: bool, include_audio: bool, mix_live: bool, + output: CustomWriterOutput, + audio_producer: Option, ) -> Result { use objc2::msg_send; use objc2::runtime::AnyObject; @@ -439,13 +1381,19 @@ impl CustomScreenCaptureWriter { static AVFileTypeMPEG4: *const AnyObject; static AVMediaTypeVideo: *const AnyObject; static AVFileTypeProfileMPEG4AppleHLS: *const AnyObject; + static AVFileTypeProfileMPEG4CMAFCompliant: *const AnyObject; } // Force-load UniformTypeIdentifiers so the runtime UTType lookup in // segmented mode resolves. #[link(name = "UniformTypeIdentifiers", kind = "framework")] extern "C" {} - let segmented = crate::remote_flags::current().custom_sck_pipeline_live_upload_enabled; + // Rewind's local rolling buffer always needs append-only fMP4 output, + // independent of whether the remote live uploader is enabled. + let segmented = segmented_output_enabled( + output, + crate::remote_flags::current().custom_sck_pipeline_live_upload_enabled, + ); unsafe { let writer_cls = av_class_named("AVAssetWriter") .ok_or_else(|| "AVAssetWriter missing".to_string())?; @@ -482,8 +1430,21 @@ impl CustomScreenCaptureWriter { epoch: 0, }; let _: () = msg_send![&*writer, setPreferredOutputSegmentInterval: interval]; - let _: () = - msg_send![&*writer, setOutputFileTypeProfile: AVFileTypeProfileMPEG4AppleHLS]; + // Rewind's video-only rolling artifact uses CMAF, with its + // independent audio persisted in adjacent PCM sidecars below. + // Ordinary uploaded Clips must retain Apple HLS: those files + // contain a live-mixed AAC track, and applying Rewind's CMAF + // profile to that writer graph makes AVAssetWriter reject + // startWriting before the first byte reaches disk. + let output_profile = if output.preserves_separate_audio() { + AVFileTypeProfileMPEG4CMAFCompliant + } else { + AVFileTypeProfileMPEG4AppleHLS + }; + let _: () = msg_send![ + &*writer, + setOutputFileTypeProfile: output_profile + ]; let sink = SegmentSink::create(output_path)?; let delegate = SegmentWriterDelegate::new(Arc::clone(&sink)); @@ -537,9 +1498,17 @@ impl CustomScreenCaptureWriter { } let _: () = msg_send![&*writer, addInput: &*video_input]; - // Live-mixing mode writes one combined audio track; otherwise each - // captured source gets its own track (mixed later by ffmpeg, or - // left as the single captured source). + let sidecar_sources = crate::capture_audio_bus::AudioSources::new( + include_audio && output.preserves_separate_audio(), + capture_system_audio && output.preserves_separate_audio(), + ); + let audio_sidecars = (sidecar_sources.microphone || sidecar_sources.system) + .then(|| AudioSidecarManager::create(output_path, sidecar_sources)) + .transpose()?; + + // AVAssetWriter's fragmented profiles reject two independent AAC + // inputs. Rewind therefore writes video-only fMP4 plus local PCM + // sidecars; ordinary recordings retain their existing inputs. let (system_audio_input, mic_audio_input, mixed_audio_input, mixer) = if mix_live { let mixed = av_make_audio_writer_input(input_cls, &writer)?; ( @@ -549,12 +1518,13 @@ impl CustomScreenCaptureWriter { Some(LiveAudioMixer::new(segmented)?), ) } else { - let system_audio_input = if capture_system_audio { - Some(av_make_audio_writer_input(input_cls, &writer)?) - } else { - None - }; - let mic_audio_input = if include_audio { + let system_audio_input = + if capture_system_audio && !output.preserves_separate_audio() { + Some(av_make_audio_writer_input(input_cls, &writer)?) + } else { + None + }; + let mic_audio_input = if include_audio && !output.preserves_separate_audio() { Some(av_make_audio_writer_input(input_cls, &writer)?) } else { None @@ -582,10 +1552,59 @@ impl CustomScreenCaptureWriter { appends_closed: Arc::new(AtomicBool::new(false)), dropped_samples: Arc::new(AtomicU64::new(0)), pause_offset_bits: Arc::new(AtomicU64::new(0.0_f64.to_bits())), + audio_producer, + audio_sidecars, }) } } + fn publish_audio_sample( + &self, + sample: &screencapturekit::cm::CMSampleBuffer, + of_type: SCStreamOutputType, + ) { + if self.audio_producer.is_none() && self.audio_sidecars.is_none() { + return; + } + let label = match of_type { + SCStreamOutputType::Audio => "shared-system", + SCStreamOutputType::Microphone => "shared-mic", + _ => return, + }; + let Some((interleaved, pts_seconds)) = extract_interleaved_stereo(sample, label) else { + return; + }; + let mono: Vec = interleaved + .chunks_exact(2) + .map(|channels| (channels[0] + channels[1]) * 0.5) + .collect(); + if let Some(sidecars) = self.audio_sidecars.as_ref() { + let session_start_seconds = + f64::from_bits(self.session_start_bits.load(Ordering::SeqCst)); + if !session_start_seconds.is_nan() { + sidecars.append( + matches!(of_type, SCStreamOutputType::Microphone), + &mono, + AUDIO_OUTPUT_SAMPLE_RATE as f64, + pts_seconds, + session_start_seconds, + ); + } + } + let Some(producer) = self.audio_producer.as_ref() else { + return; + }; + match of_type { + SCStreamOutputType::Audio => { + producer.publish_system(&mono, AUDIO_OUTPUT_SAMPLE_RATE as f64) + } + SCStreamOutputType::Microphone => { + producer.publish_microphone(&mono, AUDIO_OUTPUT_SAMPLE_RATE as f64) + } + _ => {} + } + } + /// Whether the writer runs the segmented (zero-based, append-only) output /// used by live upload. Only that mode rebases sample timestamps, so it's /// the only mode where a fresh SCStream can be spliced onto the same file. @@ -593,11 +1612,56 @@ impl CustomScreenCaptureWriter { self.inner.lock().map(|g| g.segmented).unwrap_or(false) } + pub(crate) fn request_fragment_fence( + &self, + next_path: PathBuf, + ) -> Result { + let guard = self.inner.lock().map_err(|e| e.to_string())?; + if guard.finished || guard.failed.is_some() { + return Err("fragment fences require an active custom capture writer".into()); + } + let sink = guard + .segment_sink + .clone() + .ok_or_else(|| "fragment fences require segmented custom capture".to_string())?; + if let Some(sidecars) = self.audio_sidecars.as_ref() { + sidecars.begin_fence()?; + } + let mut fence = match sink.fence(next_path.clone()) { + Ok(fence) => fence, + Err(error) => { + if let Some(sidecars) = self.audio_sidecars.as_ref() { + sidecars.cancel_fence(); + } + return Err(error); + } + }; + if let Some(sidecars) = self.audio_sidecars.as_ref() { + fence.audio_sidecars = Some((Arc::clone(sidecars), next_path)); + } + Ok(fence) + } + /// Whether the writer session has begun (first video frame appended). pub(super) fn is_started(&self) -> bool { self.started.load(Ordering::SeqCst) } + fn segment_progress(&self) -> SegmentProgress { + self.inner + .lock() + .ok() + .and_then(|guard| guard.segment_sink.as_ref().map(|sink| sink.progress())) + .unwrap_or_default() + } + + fn failure(&self) -> Option { + self.inner + .lock() + .ok() + .and_then(|guard| guard.failed.clone()) + } + /// Accumulated pause offset in seconds. Every appended sample skips this /// much wall-clock time so a pause/resume leaves no gap in the file. pub(super) fn pause_offset(&self) -> f64 { @@ -821,21 +1885,30 @@ impl CustomScreenCaptureWriter { .store(secs.to_bits(), Ordering::SeqCst); } self.started.store(true, Ordering::SeqCst); + crate::logfile::diagnostic("[capture-health] AVAssetWriter session started"); true } Ok(false) => { self.appends_closed.store(true, Ordering::SeqCst); - guard.failed = Some(format!( + let error = format!( "AVAssetWriter startWriting failed{}", av_writer_error_suffix(&guard.writer) + ); + crate::logfile::diagnostic(&format!( + "[capture-health] {error} (segmented={segmented})" )); + guard.failed = Some(error); false } Err(exc) => { self.appends_closed.store(true, Ordering::SeqCst); let detail = describe_objc_exception(exc); eprintln!("[mixer] startWriting raised Objective-C exception: {detail}"); - guard.failed = Some(format!("AVAssetWriter startWriting raised: {detail}")); + let error = format!("AVAssetWriter startWriting raised: {detail}"); + crate::logfile::diagnostic(&format!( + "[capture-health] {error} (segmented={segmented})" + )); + guard.failed = Some(error); false } } @@ -895,6 +1968,15 @@ impl CustomScreenCaptureWriter { /// writer status; without it, finalization completes in the background /// (the fragmented file is already playable up to the last fragment). pub(super) fn finish(&self, wait_for_finalize: bool) -> Result<(), String> { + // Close shared subscriptions before stopping SCK. A meeting start that + // races teardown will now see no producer and may safely use the legacy + // physical path, instead of attaching to a stream that is going away. + if let Some(producer) = self.audio_producer.as_ref() { + producer.deactivate(); + } + if let Some(sidecars) = self.audio_sidecars.as_ref() { + sidecars.finish()?; + } self.appends_closed.store(true, Ordering::SeqCst); let dropped = self.dropped_samples.load(Ordering::Relaxed); eprintln!( @@ -961,6 +2043,9 @@ impl CustomScreenCaptureWriter { ) }; if let Some(err) = failed { + if let Some(sink) = segment_sink.as_ref() { + sink.cancel_pending("fragment fence cancelled because writer finalization failed"); + } return Err(err); } unsafe { @@ -971,6 +2056,11 @@ impl CustomScreenCaptureWriter { if !started { let _: () = msg_send![&*writer, cancelWriting]; + if let Some(sink) = segment_sink.as_ref() { + sink.cancel_pending( + "fragment fence cancelled because writer received no samples", + ); + } return Err("AVAssetWriter received no samples".into()); } let _: () = msg_send![&*video_input, markAsFinished]; @@ -991,11 +2081,21 @@ impl CustomScreenCaptureWriter { let _: () = msg_send![&*writer, finishWritingWithCompletionHandler: &*block]; if wait_for_finalize && rx.recv_timeout(StdDuration::from_secs(15)).is_err() { let _: () = msg_send![&*writer, cancelWriting]; + if let Some(sink) = segment_sink.as_ref() { + sink.cancel_pending( + "fragment fence cancelled because writer finalization timed out", + ); + } return Err("AVAssetWriter finalize timed out".into()); } if wait_for_finalize { let status: i64 = msg_send![&*writer, status]; if status != AV_WRITER_STATUS_COMPLETED { + if let Some(sink) = segment_sink.as_ref() { + sink.cancel_pending( + "fragment fence cancelled because writer finalization failed", + ); + } return Err(format!( "AVAssetWriter finalize failed (status={status}{})", av_writer_error_suffix(&writer) @@ -1008,8 +2108,12 @@ impl CustomScreenCaptureWriter { // unless a disk write failed along the way. if let Some(sink) = segment_sink.as_ref() { if let Some(err) = sink.failure() { + sink.cancel_pending("fragment fence cancelled because segment output failed"); return Err(err); } + sink.cancel_pending( + "fragment fence reached writer finalization before a media boundary", + ); } eprintln!("[mixer] writer finish completed (wait={wait_for_finalize})"); Ok(()) @@ -1615,9 +2719,52 @@ fn extract_interleaved_stereo( // Normalize every source to the mixer's output rate so contiguous placement // on the timeline matches real time (otherwise the mic is pitch-shifted). out = resample_interleaved_stereo(&out, src_rate, AUDIO_OUTPUT_SAMPLE_RATE as f64); + log_decoded_audio_signal_once(label, &out); Some((out, pts_seconds)) } +/// Record whether decoded source PCM contains a real signal before it reaches +/// the timeline mixer. This distinguishes capture/format failures from mixer +/// or writer failures without persisting any audio content. +fn log_decoded_audio_signal_once(label: &str, samples: &[f32]) { + use std::sync::atomic::AtomicBool; + static SYS_SIGNAL_LOGGED: AtomicBool = AtomicBool::new(false); + static MIC_SIGNAL_LOGGED: AtomicBool = AtomicBool::new(false); + let peak = samples + .iter() + .copied() + .map(f32::abs) + .fold(0.0_f32, f32::max); + // Initial SCK audio callbacks are normally zero-filled while the device + // warms. Wait for the first real signal so this diagnostic proves source + // health instead of permanently recording that uninteresting pre-roll. + if peak <= 0.000_01 { + return; + } + let flag = if label.contains("mic") { + &MIC_SIGNAL_LOGGED + } else { + &SYS_SIGNAL_LOGGED + }; + if flag.swap(true, Ordering::SeqCst) { + return; + } + let rms = if samples.is_empty() { + 0.0 + } else { + (samples + .iter() + .map(|sample| (*sample as f64) * (*sample as f64)) + .sum::() + / samples.len() as f64) + .sqrt() + }; + crate::logfile::diagnostic(&format!( + "[capture-health] decoded {label} PCM: samples={} peak={peak:.6} rms={rms:.6}", + samples.len() + )); +} + /// Logs the decoded audio format the first time each source is seen, so format /// mismatches (rate / channels / int-vs-float / interleaving) are diagnosable. fn log_audio_format_once( @@ -2297,6 +3444,10 @@ fn build_custom_scstream( config.set_source_rect(rect); } if let Some(device) = selected_mic.as_ref() { + crate::logfile::diagnostic(&format!( + "[capture-health] microphone selected: {} ({})", + device.name, device.id + )); config.set_microphone_capture_device_id(&device.id); } config.set_stream_name(Some("Clips custom full-screen recording")); @@ -2315,6 +3466,75 @@ fn build_custom_scstream( Ok(stream) } +fn await_segmented_capture_readiness( + stream: &SCStream, + writer: &CustomScreenCaptureWriter, + watch: &CaptureWatch, + requested_system_audio: bool, + requested_microphone: bool, +) -> Result<(), String> { + let sample_deadline = Instant::now() + CAPTURE_FIRST_SAMPLE_TIMEOUT; + while Instant::now() < sample_deadline { + if let Some(error) = watch.take_stream_stopped() { + return Err(error); + } + if let Some(error) = writer.failure() { + return Err(error); + } + let (_, usable_screen, _, _) = watch.sample_counts(); + if usable_screen > 0 && writer.is_started() { + break; + } + std::thread::sleep(Duration::from_millis(25)); + } + + let (screen, usable_screen, system, mic) = watch.sample_counts(); + if usable_screen == 0 || !writer.is_started() { + let _ = stream.stop_capture(); + let finish_error = writer.finish(true).err(); + return Err(format!( + "capture startup timed out before the first usable video sample/writer session (screen={screen}, usable_screen={usable_screen}, system_audio={system}, microphone={mic}){}", + finish_error + .map(|error| format!("; writer: {error}")) + .unwrap_or_default() + )); + } + + let fragment_deadline = Instant::now() + CAPTURE_FIRST_FRAGMENT_TIMEOUT; + while Instant::now() < fragment_deadline { + if let Some(error) = watch.take_stream_stopped() { + return Err(error); + } + if let Some(error) = writer.failure() { + return Err(error); + } + let progress = writer.segment_progress(); + if progress.has_initialization && progress.media_fragments > 0 && progress.bytes_written > 0 + { + crate::logfile::diagnostic(&format!( + "[capture-health] segmented capture ready: {} bytes, {} media fragment(s)", + progress.bytes_written, progress.media_fragments + )); + return Ok(()); + } + std::thread::sleep(Duration::from_millis(25)); + } + + let (screen, usable_screen, system, mic) = watch.sample_counts(); + let progress = writer.segment_progress(); + let _ = stream.stop_capture(); + let finish_error = writer.finish(true).err(); + Err(format!( + "capture startup timed out waiting for the first fragmented-media bytes (screen={screen}, usable_screen={usable_screen}, system_audio={system}, microphone={mic}, requested_system_audio={requested_system_audio}, requested_microphone={requested_microphone}, initialization={}, media_fragments={}, bytes={}){}", + progress.has_initialization, + progress.media_fragments, + progress.bytes_written, + finish_error + .map(|error| format!("; writer: {error}")) + .unwrap_or_default() + )) +} + /// Supervise a running custom capture and rebuild the SCStream when it stops or /// goes silent (the Spaces/full-screen interruption). Runs on its own thread; /// exits when recording is torn down (`shutdown`) or the writer is closed. @@ -2355,6 +3575,12 @@ fn spawn_capture_watchdog( eprintln!( "[mixer] appends closed without a stop request; treating as fatal capture failure and finalizing partial recording" ); + let writer_error = writer + .failure() + .unwrap_or_else(|| "unknown writer failure".to_string()); + crate::logfile::diagnostic(&format!( + "[capture-health] writer closed unexpectedly; finalizing partial recording: {writer_error}" + )); if let Ok(guard) = stream.lock() { let _ = guard.stop_capture(); } @@ -2483,7 +3709,7 @@ fn spawn_capture_watchdog( /// build + start the SCStream from rebuildable params, and spawn the /// capture watchdog that supervises it. Returns the backend handle plus /// the output dimensions. -pub(super) fn start_custom_screencapturekit_backend_at( +pub(crate) fn start_custom_screencapturekit_backend_at( app: &AppHandle, output_path: &Path, include_audio: bool, @@ -2493,6 +3719,7 @@ pub(super) fn start_custom_screencapturekit_backend_at( target_display_id: Option, capture_region: Option, defer_recording_output: bool, + force_segmented_output: bool, ) -> Result<(NativeFullscreenBackend, Option, Option), String> { eprintln!("[clips-tray] starting custom screen capture backend"); let content = @@ -2526,9 +3753,26 @@ pub(super) fn start_custom_screencapturekit_backend_at( height, }; - // Live-mix only when both audio sources are present; a single source needs - // no mixing and goes straight to its own track. - let mix_live = include_audio && capture_system_audio; + // Rewind keeps microphone and system audio on separate writer tracks for + // later local transcription. The ordinary recorder retains its existing + // live-mix behavior (needed for its remote upload path). + let output = if force_segmented_output { + CustomWriterOutput::RewindCmaf + } else { + CustomWriterOutput::Standard + }; + let mix_live = live_audio_mixing_enabled(output, include_audio, capture_system_audio); + // The active custom capture is the single physical owner of mic/system + // audio for both Rewind and ordinary Clips. Live transcription subscribes + // to this producer instead of opening a competing SCK/AVAudioEngine input, + // which can make every microphone consumer receive digital silence. + let audio_producer = (include_audio || capture_system_audio) + .then(|| { + crate::capture_audio_bus::AudioProducer::register( + crate::capture_audio_bus::AudioSources::new(include_audio, capture_system_audio), + ) + }) + .transpose()?; let writer = CustomScreenCaptureWriter::new( output_path, width, @@ -2536,12 +3780,16 @@ pub(super) fn start_custom_screencapturekit_backend_at( capture_system_audio, include_audio, mix_live, + output, + audio_producer, )?; let recording_enabled = Arc::new(AtomicBool::new(!defer_recording_output)); + let clip_sink = Arc::new(Mutex::new(None)); let mic_ready = include_audio.then(|| Arc::new(AtomicBool::new(false))); let watch = Arc::new(CaptureWatch::new()); let handler = CustomScreenCaptureOutputHandler { writer: writer.clone(), + clip_sink: Arc::clone(&clip_sink), recording_enabled: Arc::clone(&recording_enabled), mic_ready: mic_ready.clone(), watch: Arc::clone(&watch), @@ -2552,6 +3800,26 @@ pub(super) fn start_custom_screencapturekit_backend_at( let _ = std::fs::remove_file(output_path); return Err(format!("custom capture start failed: {err:?}")); } + crate::logfile::diagnostic(&format!( + "[capture-health] ScreenCaptureKit start_capture returned success: system_audio={capture_system_audio} microphone={include_audio} segmented={force_segmented_output} deferred={defer_recording_output}" + )); + if force_segmented_output && !defer_recording_output { + if let Err(error) = await_segmented_capture_readiness( + &stream, + &writer, + &watch, + capture_system_audio, + include_audio, + ) { + let _ = std::fs::remove_file(output_path); + let _ = std::fs::remove_file(audio_sidecar_path(output_path, "system")); + let _ = std::fs::remove_file(audio_sidecar_path(output_path, "microphone")); + crate::logfile::diagnostic(&format!( + "[capture-health] segmented capture readiness failed: {error}" + )); + return Err(error); + } + } eprintln!( "[clips-tray] custom ScreenCaptureKit recording started: {width}x{height} @ {NATIVE_CAPTURE_FPS}fps from {capture_width}x{capture_height} (display {source_width}x{source_height}), mic={include_audio} system_audio={capture_system_audio} deferred_output={defer_recording_output}" ); @@ -2586,8 +3854,194 @@ pub(super) fn start_custom_screencapturekit_backend_at( recording_enabled, watchdog_shutdown, resume, + clip_sink, }, Some(width), Some(height), )) } + +#[cfg(test)] +mod fragment_fence_tests { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + + fn path(label: &str) -> PathBuf { + static NEXT: AtomicU64 = AtomicU64::new(0); + std::env::temp_dir().join(format!( + "clips-fragment-fence-{label}-{}-{}.mp4", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )) + } + + fn remove(paths: &[&Path]) { + for path in paths { + let _ = std::fs::remove_file(path); + } + } + + #[test] + fn caches_init_and_replays_it_for_each_fenced_file() { + let first = path("init-first"); + let second = path("init-second"); + let sink = SegmentSink::create(&first).unwrap(); + sink.append(b"INIT", AV_ASSET_WRITER_SEGMENT_TYPE_INITIALIZATION, None); + sink.append(b"M0", AV_ASSET_WRITER_SEGMENT_TYPE_SEPARABLE, Some(0.0)); + let fence = sink.fence(second.clone()).unwrap(); + sink.append(b"M1", AV_ASSET_WRITER_SEGMENT_TYPE_SEPARABLE, Some(1.0)); + let closed = fence.wait(Duration::from_secs(1)).unwrap(); + assert_eq!(closed.path, first); + sink.append(b"M2", AV_ASSET_WRITER_SEGMENT_TYPE_SEPARABLE, Some(2.0)); + assert_eq!(std::fs::read(&first).unwrap(), b"INITM0"); + assert_eq!(std::fs::read(&second).unwrap(), b"INITM1M2"); + remove(&[&first, &second]); + } + + #[test] + fn routes_each_media_fragment_exactly_once() { + let first = path("once-first"); + let second = path("once-second"); + let sink = SegmentSink::create(&first).unwrap(); + sink.append(b"I", AV_ASSET_WRITER_SEGMENT_TYPE_INITIALIZATION, None); + sink.append(b"A", AV_ASSET_WRITER_SEGMENT_TYPE_SEPARABLE, Some(0.0)); + let fence = sink.fence(second.clone()).unwrap(); + sink.append(b"B", AV_ASSET_WRITER_SEGMENT_TYPE_SEPARABLE, Some(1.0)); + sink.append(b"C", AV_ASSET_WRITER_SEGMENT_TYPE_SEPARABLE, Some(2.0)); + assert_eq!( + fence.wait(Duration::from_secs(1)).unwrap().media_fragments, + 1 + ); + assert_eq!(std::fs::read(&first).unwrap(), b"IA"); + assert_eq!(std::fs::read(&second).unwrap(), b"IBC"); + remove(&[&first, &second]); + } + + #[test] + fn repeated_fences_complete_in_fragment_order() { + let first = path("order-first"); + let second = path("order-second"); + let third = path("order-third"); + let sink = SegmentSink::create(&first).unwrap(); + sink.append(b"I", AV_ASSET_WRITER_SEGMENT_TYPE_INITIALIZATION, None); + sink.append(b"Z", AV_ASSET_WRITER_SEGMENT_TYPE_SEPARABLE, Some(0.0)); + let one = sink.fence(second.clone()).unwrap(); + let two = sink.fence(third.clone()).unwrap(); + sink.append(b"A", AV_ASSET_WRITER_SEGMENT_TYPE_SEPARABLE, Some(1.0)); + assert_eq!(one.wait(Duration::from_secs(1)).unwrap().sequence, 0); + sink.append(b"B", AV_ASSET_WRITER_SEGMENT_TYPE_SEPARABLE, Some(2.0)); + assert_eq!(two.wait(Duration::from_secs(1)).unwrap().sequence, 1); + assert_eq!(std::fs::read(&first).unwrap(), b"IZ"); + assert_eq!(std::fs::read(&second).unwrap(), b"IA"); + assert_eq!(std::fs::read(&third).unwrap(), b"IB"); + remove(&[&first, &second, &third]); + } + + #[test] + fn fence_creation_failure_fails_closed() { + let first = path("error-first"); + let directory = std::env::temp_dir(); + let sink = SegmentSink::create(&first).unwrap(); + sink.append(b"I", AV_ASSET_WRITER_SEGMENT_TYPE_INITIALIZATION, None); + sink.append(b"Z", AV_ASSET_WRITER_SEGMENT_TYPE_SEPARABLE, Some(0.0)); + let fence = sink.fence(directory).unwrap(); + sink.append(b"A", AV_ASSET_WRITER_SEGMENT_TYPE_SEPARABLE, Some(1.0)); + assert!(fence.wait(Duration::from_secs(1)).is_err()); + assert!(sink.failure().is_some()); + remove(&[&first]); + } + + #[test] + fn forced_segmented_rewind_disables_live_mix_for_selectable_sidecars() { + assert!(segmented_output_enabled( + CustomWriterOutput::RewindCmaf, + false + )); + assert!(!live_audio_mixing_enabled( + CustomWriterOutput::RewindCmaf, + true, + true + )); + assert!(live_audio_mixing_enabled( + CustomWriterOutput::Standard, + true, + true + )); + // Clip HLS is forced segmented regardless of the ordinary remote flag, + // but keeps a single live-mixed AAC track and no Rewind sidecars. + assert!(segmented_output_enabled(CustomWriterOutput::ClipHls, false)); + assert!(live_audio_mixing_enabled( + CustomWriterOutput::ClipHls, + true, + true + )); + } + + #[test] + fn clip_sink_gate_orders_prepare_activate_and_logical_close() { + let gate = ClipSinkGate::new(); + assert!(!gate.accepts(), "prepared sinks ignore callbacks"); + gate.state + .store(ClipSinkState::Active as u64, Ordering::SeqCst); + assert!(gate.accepts(), "activated sink accepts callbacks"); + gate.state + .store(ClipSinkState::Closed as u64, Ordering::SeqCst); + assert!(!gate.accepts(), "logical close wins before finalization"); + } + + #[test] + fn clip_sink_slot_rejects_second_install() { + let mut slot = None; + install_only_slot(&mut slot, 1_u8).unwrap(); + assert!(install_only_slot(&mut slot, 2_u8).is_err()); + assert_eq!(slot, Some(1)); + } + + #[test] + fn audio_sidecars_preserve_sources_and_rotate_as_valid_float_wav() { + let first = path("audio-first"); + let second = path("audio-second"); + let sidecars = AudioSidecarManager::create( + &first, + crate::capture_audio_bus::AudioSources::new(true, true), + ) + .unwrap(); + sidecars.append(false, &[0.25; 480], 48_000.0, 10.010, 10.0); + sidecars.append(true, &[0.5; 480], 48_000.0, 10.020, 10.0); + sidecars.begin_fence().unwrap(); + // These two buffers cross the exact 30 ms video-fragment boundary. + sidecars.append(false, &[0.1; 960], 48_000.0, 10.025, 10.0); + sidecars.append(true, &[0.2; 960], 48_000.0, 10.025, 10.0); + sidecars.complete_fence(&second, 0.030).unwrap(); + sidecars.append(false, &[0.1; 480], 48_000.0, 10.040, 10.0); + sidecars.append(true, &[0.2; 480], 48_000.0, 10.040, 10.0); + sidecars.finish().unwrap(); + + let first_system = audio_sidecar_path(&first, "system"); + let first_microphone = audio_sidecar_path(&first, "microphone"); + let second_system = audio_sidecar_path(&second, "system"); + let second_microphone = audio_sidecar_path(&second, "microphone"); + for wav in [ + &first_system, + &first_microphone, + &second_system, + &second_microphone, + ] { + let bytes = std::fs::read(wav).unwrap(); + assert_eq!(&bytes[..4], b"RIFF"); + assert_eq!(&bytes[8..12], b"WAVE"); + assert!(bytes.len() > 44); + } + let wav_samples = |path: &Path| (std::fs::metadata(path).unwrap().len() - 44) / 4; + assert_eq!(wav_samples(&first_system), 1_440); + assert_eq!(wav_samples(&first_microphone), 1_440); + assert_eq!(wav_samples(&second_system), 960); + assert_eq!(wav_samples(&second_microphone), 960); + remove(&[ + &first_system, + &first_microphone, + &second_system, + &second_microphone, + ]); + } +} diff --git a/templates/clips/desktop/src-tauri/src/native_screen/live_upload.rs b/templates/clips/desktop/src-tauri/src/native_screen/live_upload.rs index a5684b375a..3a323832a4 100644 --- a/templates/clips/desktop/src-tauri/src/native_screen/live_upload.rs +++ b/templates/clips/desktop/src-tauri/src/native_screen/live_upload.rs @@ -32,6 +32,9 @@ pub(super) struct LiveUploadCtrl { pub(super) cancelled: AtomicBool, /// Recorded media duration in ms; set just before `finalize`. pub(super) duration_ms: AtomicU64, + /// Bytes acknowledged by the server. Stop snapshots this before final-tail + /// drainage so telemetry measures genuine progressive upload. + pub(super) uploaded_bytes: AtomicU64, } pub(super) struct LiveUpload { @@ -57,6 +60,80 @@ struct LiveUploadParams { has_camera: bool, } +/// Explicit opt-in credentials for a secondary Clip sink. `server_url: None` +/// means local-only; a partial remote configuration is rejected rather than +/// broadening authentication or quietly changing the retention path. +#[derive(Clone)] +pub(crate) struct ClipLiveUploadConfig { + pub(crate) server_url: Option, + pub(crate) recording_id: Option, + pub(crate) auth_token: Option, + pub(crate) cookie: Option, +} + +impl ClipLiveUploadConfig { + pub(crate) fn validated(self) -> Result, String> { + let Some(server_url) = self.server_url.filter(|value| !value.trim().is_empty()) else { + return Ok(None); + }; + let recording_id = self + .recording_id + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| "Clip live upload requires a recording ID".to_string())?; + let auth_token = self.auth_token.unwrap_or_default(); + let cookie = self.cookie.unwrap_or_default(); + if auth_token.trim().is_empty() && cookie.trim().is_empty() { + return Err("Clip live upload requires auth token or cookie".into()); + } + Ok(Some((server_url, recording_id, auth_token, cookie))) + } +} + +pub(super) fn start_clip_live_uploader( + app: &AppHandle, + path: PathBuf, + mime_type: String, + width: Option, + height: Option, + has_audio: bool, + has_camera: bool, + upload: ClipLiveUploadConfig, +) -> Result, String> { + let Some((server_url, recording_id, auth_token, cookie)) = upload.validated()? else { + return Ok(None); + }; + Ok(Some(spawn_live_uploader( + app.clone(), + LiveUploadParams { + path, + server_url, + recording_id, + auth_token, + cookie, + mime_type, + width, + height, + has_audio, + has_camera, + }, + ))) +} + +pub(super) async fn finalize_clip_live_upload( + live: LiveUpload, + duration_ms: u64, +) -> Result { + live.ctrl.duration_ms.store(duration_ms, Ordering::SeqCst); + live.ctrl.finalize.store(true, Ordering::SeqCst); + live.result_rx + .await + .map_err(|_| "Clip live uploader ended before returning a result".to_string())? +} + +pub(super) fn cancel_clip_live_upload(live: &LiveUpload) { + live.ctrl.cancelled.store(true, Ordering::SeqCst); +} + /// Spawn the background uploader and return a handle the stop path uses to /// finalize (or cancel) it. fn spawn_live_uploader(app: AppHandle, params: LiveUploadParams) -> LiveUpload { @@ -64,6 +141,7 @@ fn spawn_live_uploader(app: AppHandle, params: LiveUploadParams) -> LiveUpload { finalize: AtomicBool::new(false), cancelled: AtomicBool::new(false), duration_ms: AtomicU64::new(0), + uploaded_bytes: AtomicU64::new(0), }); let (tx, rx) = tokio::sync::oneshot::channel(); let ctrl_task = ctrl.clone(); @@ -272,6 +350,10 @@ async fn live_upload_loop( return Err(e); } offset += chunk; + ctrl.uploaded_bytes.store(offset, Ordering::SeqCst); + crate::logfile::diagnostic(&format!( + "[live-upload] {rec}: acknowledged {offset} bytes before finalize={finalize}" + )); index += 1; emit_native_upload_progress(&app, "uploading", "Uploading clip", None, None); } @@ -330,6 +412,7 @@ async fn live_upload_loop( verification_pending = pending; } offset += take; + ctrl.uploaded_bytes.store(offset, Ordering::SeqCst); index += 1; final_sent = final_sent || is_last; } @@ -364,6 +447,9 @@ async fn live_upload_loop( } emit_native_upload_progress(&app, "opening", "Uploading clip", None, Some(1.0)); eprintln!("[live-upload] {rec}: done — {index} post(s), {final_len} bytes total"); + crate::logfile::diagnostic(&format!( + "[live-upload] {rec}: complete with {final_len} acknowledged bytes" + )); return Ok(LiveUploadResult { bytes: final_len, verification_pending, @@ -379,3 +465,41 @@ async fn live_upload_loop( tokio::time::sleep(Duration::from_millis(LIVE_UPLOAD_POLL_MS)).await; } } + +#[cfg(test)] +mod clip_upload_config_tests { + use super::*; + + #[test] + fn local_only_clip_upload_is_explicitly_allowed() { + assert!(ClipLiveUploadConfig { + server_url: None, + recording_id: None, + auth_token: None, + cookie: None, + } + .validated() + .unwrap() + .is_none()); + } + + #[test] + fn remote_clip_upload_rejects_partial_authority() { + assert!(ClipLiveUploadConfig { + server_url: Some("https://clips.example".into()), + recording_id: None, + auth_token: Some("token".into()), + cookie: None, + } + .validated() + .is_err()); + assert!(ClipLiveUploadConfig { + server_url: Some("https://clips.example".into()), + recording_id: Some("recording".into()), + auth_token: None, + cookie: None, + } + .validated() + .is_err()); + } +} diff --git a/templates/clips/desktop/src-tauri/src/rewind_capture_suspension.rs b/templates/clips/desktop/src-tauri/src/rewind_capture_suspension.rs new file mode 100644 index 0000000000..38158258ca --- /dev/null +++ b/templates/clips/desktop/src-tauri/src/rewind_capture_suspension.rs @@ -0,0 +1,165 @@ +use crate::capture_graph::CaptureSource; +use serde::Serialize; +use std::collections::BTreeSet; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Mutex; +use tauri::{AppHandle, Manager}; + +static LEASE_COUNTER: AtomicU64 = AtomicU64::new(0); +static SUSPENSION_ACTIVE: AtomicBool = AtomicBool::new(false); + +#[derive(Default)] +pub struct RewindCaptureSuspensionState { + inner: Mutex, +} + +#[derive(Default)] +struct RewindCaptureSuspensionRuntime { + leases: BTreeSet, + suspended_rewind: bool, +} + +impl RewindCaptureSuspensionRuntime { + /// True only when this release removed the final live owner. + fn release_lease(&mut self, lease_id: &str) -> bool { + self.leases.remove(lease_id) && self.leases.is_empty() + } +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RewindCaptureSuspensionLease { + lease_id: Option, + suspended_rewind: bool, +} + +fn next_lease_id() -> String { + let sequence = LEASE_COUNTER.fetch_add(1, Ordering::Relaxed); + format!("ordinary-capture-{sequence}") +} + +fn capture_conflicts( + requires_screen: bool, + requires_microphone: bool, + sources: &[CaptureSource], +) -> bool { + requires_screen + || (requires_microphone + && sources + .iter() + .any(|source| *source == CaptureSource::Microphone)) +} + +pub(crate) fn is_active(app: &AppHandle) -> bool { + let _ = app; + SUSPENSION_ACTIVE.load(Ordering::Acquire) +} + +/// Hands the one physical capture graph to an ordinary Clip that cannot reuse +/// Rewind. The first lease stops Rewind; nested leases keep it stopped until +/// the final owner releases. Persisted enable/pause/mode settings are untouched. +#[tauri::command] +pub async fn rewind_capture_suspension_acquire( + app: AppHandle, + requires_screen: bool, + requires_microphone: bool, +) -> Result { + let state = app.state::(); + let mut runtime = state.inner.lock().map_err(|error| error.to_string())?; + + if runtime.leases.is_empty() { + let sources = crate::screen_memory::rewind_clip_sources(&app); + let conflicts = capture_conflicts(requires_screen, requires_microphone, &sources); + let rewind_config = crate::config::feature_config(&app).screen_memory; + // Enabled, unpaused Rewind may currently be stopped for a privacy + // exclusion or transient producer transition. Lease the handoff based + // on its durable ownership intent so its worker cannot resume halfway + // through an ordinary recording. + if !conflicts || !rewind_config.enabled || rewind_config.paused { + return Ok(RewindCaptureSuspensionLease { + lease_id: None, + suspended_rewind: false, + }); + } + // Publish the reservation before entering the Screen Memory transition + // lock. Config sync and temporary audio demand can then observe the + // handoff without taking this module's mutex (avoiding lock inversion). + SUSPENSION_ACTIVE.store(true, Ordering::Release); + if let Err(error) = crate::screen_memory::suspend_physical_capture(&app) { + SUSPENSION_ACTIVE.store(false, Ordering::Release); + return Err(error); + } + runtime.suspended_rewind = true; + } + + let lease_id = next_lease_id(); + runtime.leases.insert(lease_id.clone()); + Ok(RewindCaptureSuspensionLease { + lease_id: Some(lease_id), + suspended_rewind: runtime.suspended_rewind, + }) +} + +/// Idempotent release: stale duplicate cleanup cannot resume Rewind while a +/// second logical owner still holds the physical capture handoff. +#[tauri::command] +pub async fn rewind_capture_suspension_release( + app: AppHandle, + lease_id: String, +) -> Result<(), String> { + let should_resume = { + let state = app.state::(); + let mut runtime = state.inner.lock().map_err(|error| error.to_string())?; + if !runtime.release_lease(lease_id.trim()) { + return Ok(()); + } + let should_resume = std::mem::take(&mut runtime.suspended_rewind); + SUSPENSION_ACTIVE.store(false, Ordering::Release); + should_resume + }; + if should_resume { + crate::screen_memory::resume_physical_capture(&app)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn screen_capture_always_conflicts_with_rewind() { + assert!(capture_conflicts(true, false, &[CaptureSource::Screen])); + } + + #[test] + fn camera_only_mic_conflicts_only_when_rewind_owns_the_mic() { + assert!(!capture_conflicts(false, true, &[CaptureSource::Screen])); + assert!(capture_conflicts( + false, + true, + &[CaptureSource::Screen, CaptureSource::Microphone] + )); + } + + #[test] + fn camera_without_mic_never_suspends_screen_memory() { + assert!(!capture_conflicts( + false, + false, + &[CaptureSource::Screen, CaptureSource::Microphone] + )); + } + + #[test] + fn nested_leases_resume_only_after_the_final_unique_release() { + let mut runtime = RewindCaptureSuspensionRuntime { + leases: BTreeSet::from(["first".to_string(), "second".to_string()]), + suspended_rewind: true, + }; + + assert!(!runtime.release_lease("first")); + assert!(!runtime.release_lease("first")); + assert!(runtime.release_lease("second")); + } +} diff --git a/templates/clips/desktop/src-tauri/src/rewind_chapters.rs b/templates/clips/desktop/src-tauri/src/rewind_chapters.rs new file mode 100644 index 0000000000..2a66b2d6b8 --- /dev/null +++ b/templates/clips/desktop/src-tauri/src/rewind_chapters.rs @@ -0,0 +1,2434 @@ +//! Retention-bound, local-only work chapters for the Rewind buffer. +//! +//! This deliberately stores references and bounded context only. Segments remain +//! the retention authority; a missing or invalid chapter manifest is disposable. + +#[cfg(test)] +use crate::config::RewindCaptureMode; +use crate::screen_memory::{ScreenMemoryEvent, ScreenMemorySegmentMetadata}; +use chrono::{DateTime, Duration, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, File}; +use std::io::{BufRead, BufReader}; +use std::path::Path; + +const CHAPTERS_FILE: &str = "chapters.json"; +const HARD_GAP_MS: i64 = 1_500; +const SCENE_SLICE_MS: i64 = 30_000; +// A safety bound, not a product boundary. Coherent work commonly lasts longer +// than the 15–20 minute acceptance samples and must not split merely by age. +const MAX_CHAPTER_SPAN_MS: i64 = 60 * 60 * 1_000; +const TRANSIENT_SCENE_MS: i64 = 75_000; +const MIN_SEMANTIC_TOPIC_MS: i64 = 120_000; +const SAME_SURFACE_CONTINUITY_OVERLAP: f32 = 0.40; +const MAX_KEYWORDS: usize = 8; +const MAX_EVIDENCE_REFS_PER_SOURCE: usize = 8; +const MAX_EVIDENCE_KEYWORDS: usize = 64; +const MAX_REPRESENTATIVE_MOMENTS: usize = 6; +const MAX_SIDECAR_ROWS: usize = 128; +const MAX_SIDECAR_LINE_BYTES: usize = 16 * 1024; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RewindChaptersManifest { + pub schema_version: u32, + pub generated_at: String, + pub state: ChapterIndexState, + pub coverage: ChapterCoverage, + #[serde(default)] + pub scenes: Vec, + pub chapters: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum ChapterIndexState { + Pending, + Partial, + Ready, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ChapterCoverage { + pub retained_segments: usize, + pub omitted_segments: usize, + pub gap_count: usize, + #[serde(default)] + pub gaps: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CoverageGap { + pub started_at: String, + pub ended_at: String, + pub duration_ms: i64, + pub reason: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RewindChapter { + pub id: String, + #[serde(default = "default_revision")] + pub revision: u32, + #[serde(default)] + pub aliases: Vec, + pub started_at: String, + pub ended_at: String, + pub duration_ms: i64, + pub label: String, + pub summary: String, + pub keywords: Vec, + pub confidence: f32, + #[serde(default)] + pub scene_refs: Vec, + pub segment_refs: Vec, + pub evidence_refs: Vec, + pub contexts: Vec, + pub representative_moments: Vec, + #[serde(default)] + pub representative_coverage: RepresentativeCoverage, + pub ambiguity_reasons: Vec, + pub index_state: ChapterIndexState, +} + +fn default_revision() -> u32 { + 1 +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RepresentativeCoverage { + pub covered_scenes: usize, + pub total_scenes: usize, + pub truncated: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RewindScene { + pub id: String, + pub started_at: String, + pub ended_at: String, + pub duration_ms: i64, + pub hard_boundary_before: bool, + pub segment_refs: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub context: Option, + pub keywords: Vec, + pub evidence_refs: Vec, + pub confidence: f32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SegmentRef { + pub id: String, + pub started_at: String, + pub ended_at: String, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct EvidenceRef { + pub source_type: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_kind: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub keywords: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub confidence: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub segment_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub offset_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub captured_at: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ChapterContext { + #[serde(skip_serializing_if = "Option::is_none")] + pub app_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub window_title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub bundle_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub document_title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub document_url: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RepresentativeMoment { + pub moment_id: String, + pub captured_at: String, + pub segment_id: String, + pub offset_ms: i64, + pub reason: String, +} + +#[derive(Clone)] +struct Item { + segment: ScreenMemorySegmentMetadata, + start: DateTime, + end: DateTime, + partial: bool, + content: ContentSignal, +} + +#[derive(Clone, Default)] +struct ContentSignal { + visual_keywords: BTreeMap, + system_audio_keywords: BTreeMap, + microphone_keywords: BTreeMap, + evidence_refs: Vec, +} + +#[derive(Clone)] +struct SceneItem { + scene: RewindScene, + start: DateTime, + end: DateTime, + visual_keywords: BTreeMap, + system_audio_keywords: BTreeMap, + microphone_keywords: BTreeMap, + partial: bool, +} + +pub(crate) fn rebuild( + dir: &Path, + segments: Vec, + events: Vec, +) -> Result<(), String> { + let content = content_signals(dir, &segments); + let previous = fs::read(dir.join(CHAPTERS_FILE)) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + let manifest = build_with_previous( + segments, + events, + pending_segment_ids(dir), + content, + previous.as_ref(), + Utc::now(), + ); + let bytes = serde_json::to_vec_pretty(&manifest) + .map_err(|e| format!("chapter manifest encode failed: {e}"))?; + write_atomic(&dir.join(CHAPTERS_FILE), &bytes) +} + +pub(crate) fn clear(dir: &Path) -> Result<(), String> { + let path = dir.join(CHAPTERS_FILE); + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(format!("chapter manifest remove failed: {e}")), + } +} + +fn build( + segments: Vec, + events: Vec, + generated_at: DateTime, +) -> RewindChaptersManifest { + build_with_pending( + segments, + events, + BTreeSet::new(), + BTreeMap::new(), + generated_at, + ) +} +fn build_with_pending( + segments: Vec, + events: Vec, + pending: BTreeSet, + content: BTreeMap, + generated_at: DateTime, +) -> RewindChaptersManifest { + build_with_previous(segments, events, pending, content, None, generated_at) +} + +fn build_with_previous( + segments: Vec, + mut events: Vec, + pending: BTreeSet, + content: BTreeMap, + previous: Option<&RewindChaptersManifest>, + generated_at: DateTime, +) -> RewindChaptersManifest { + events.sort_by(|a, b| a.captured_at.cmp(&b.captured_at)); + let mut omitted = 0usize; + let mut explicit_gaps = Vec::new(); + let mut items = segments + .into_iter() + .filter_map(|segment| { + if segment.exclusion_tainted || segment.corrupt || segment.error.is_some() { + omitted += 1; + if let (Ok(start), Ok(end)) = ( + DateTime::parse_from_rfc3339(&segment.started_at), + DateTime::parse_from_rfc3339(&segment.ended_at), + ) { + let start = start.with_timezone(&Utc); + let end = end.with_timezone(&Utc); + if end > start { + explicit_gaps.push(CoverageGap { + started_at: start.to_rfc3339(), + ended_at: end.to_rfc3339(), + duration_ms: (end - start).num_milliseconds(), + reason: if segment.exclusion_tainted { + "excluded capture interval" + } else if segment.corrupt { + "corrupt capture interval" + } else { + "failed capture interval" + } + .into(), + }); + } + } + return None; + } + let start = DateTime::parse_from_rfc3339(&segment.started_at) + .ok()? + .with_timezone(&Utc); + let end = DateTime::parse_from_rfc3339(&segment.ended_at) + .ok()? + .with_timezone(&Utc); + if end <= start { + omitted += 1; + return None; + } + let partial = pending.contains(&segment.id); + let content = content.get(&segment.id).cloned().unwrap_or_default(); + Some(Item { + segment, + start, + end, + partial, + content, + }) + }) + .collect::>(); + items.sort_by(|a, b| (a.start, &a.segment.id).cmp(&(b.start, &b.segment.id))); + let retained = items.len(); + let mut gaps = timeline_gaps(&items, &events); + gaps.append(&mut explicit_gaps); + gaps.sort_by(|a, b| a.started_at.cmp(&b.started_at)); + gaps.dedup_by(|a, b| a.started_at == b.started_at && a.ended_at == b.ended_at); + + let scene_items = build_scene_items(&items, &events); + let scenes = scene_items + .iter() + .map(|item| item.scene.clone()) + .collect::>(); + let groups = group_semantic_scenes(&scene_items); + let mut chapters = groups.into_iter().map(chapter).collect::>(); + apply_previous_identity(&mut chapters, previous); + let state = if retained == 0 { + ChapterIndexState::Pending + } else if chapters + .iter() + .any(|c| c.index_state == ChapterIndexState::Partial) + { + ChapterIndexState::Partial + } else { + ChapterIndexState::Ready + }; + RewindChaptersManifest { + schema_version: 2, + generated_at: generated_at.to_rfc3339(), + state, + coverage: ChapterCoverage { + retained_segments: retained, + omitted_segments: omitted, + gap_count: gaps.len(), + gaps, + }, + scenes, + chapters, + } +} + +fn timeline_gaps(items: &[Item], events: &[ScreenMemoryEvent]) -> Vec { + items + .windows(2) + .filter_map(|pair| { + let previous = &pair[0]; + let next = &pair[1]; + let duration_ms = (next.start - previous.end).num_milliseconds(); + (duration_ms > HARD_GAP_MS).then(|| { + let explicit_reason = events.iter().find_map(|event| { + (event.source == "coverage-gap" + && DateTime::parse_from_rfc3339(&event.captured_at) + .ok() + .map(|at| at.with_timezone(&Utc)) + .is_some_and(|at| at >= previous.end && at <= next.start)) + .then(|| event.coverage_gap_reason.clone()) + .flatten() + }); + CoverageGap { + started_at: previous.end.to_rfc3339(), + ended_at: next.start.to_rfc3339(), + duration_ms, + reason: explicit_reason.unwrap_or_else(|| { + if previous.segment.graph_epoch_id != next.segment.graph_epoch_id { + "capture restart interval" + } else { + "unretained capture interval" + } + .into() + }), + } + }) + }) + .collect() +} + +fn build_scene_items(items: &[Item], events: &[ScreenMemoryEvent]) -> Vec { + let mut scenes = Vec::new(); + for (item_index, item) in items.iter().enumerate() { + let mut boundaries = vec![item.start, item.end]; + let mut cursor = item.start + Duration::milliseconds(SCENE_SLICE_MS); + while cursor < item.end { + boundaries.push(cursor); + cursor += Duration::milliseconds(SCENE_SLICE_MS); + } + let mut last_semantic_key: Option = None; + for event in events { + let Some(at) = parse_utc(&event.captured_at) else { + continue; + }; + if at < item.start || at > item.end || event.source == "coverage-gap" { + continue; + } + let key = event_semantic_key(event); + if last_semantic_key.as_ref() != Some(&key) { + if at > item.start && at < item.end { + boundaries.push(at); + } + last_semantic_key = Some(key); + } + } + boundaries.sort(); + boundaries.dedup(); + for pair in boundaries.windows(2) { + let start = pair[0]; + let end = pair[1]; + if end <= start { + continue; + } + let context_event = context_event_at(events, start, end); + let context = context_event.map(chapter_context_from_event); + let mut visual_keywords = BTreeMap::new(); + let mut system_audio_keywords = BTreeMap::new(); + let mut microphone_keywords = BTreeMap::new(); + let mut evidence_refs = Vec::new(); + + if let Some(event) = context_event { + if let Some(title) = event.window_title.as_deref() { + let repeats_app_name = event + .app_name + .as_deref() + .is_some_and(|app| app.trim().eq_ignore_ascii_case(title.trim())); + if !generic_surface_title(title) && !repeats_app_name { + add_keywords(title, &mut visual_keywords); + } + } + let accessibility = accessibility_keywords(event); + merge_counts(&mut visual_keywords, &accessibility); + evidence_refs.push(EvidenceRef { + source_type: "app-context".into(), + source_kind: Some( + if event.accessibility.is_some() { + "accessibility" + } else { + "window" + } + .into(), + ), + keywords: top_keywords(&accessibility, 12), + confidence: Some(if event.accessibility.is_some() { + 0.9 + } else { + 0.55 + }), + segment_id: Some(item.segment.id.clone()), + offset_ms: Some((start - item.start).num_milliseconds()), + captured_at: Some(start.to_rfc3339()), + }); + } + + for reference in &item.content.evidence_refs { + let Some(at) = evidence_time(reference, item.start) else { + continue; + }; + if at < start || at >= end { + continue; + } + match reference.source_kind.as_deref() { + Some("visual" | "accessibility") => { + add_keyword_list(&reference.keywords, &mut visual_keywords) + } + Some("microphone") => { + add_keyword_list(&reference.keywords, &mut microphone_keywords) + } + _ => add_keyword_list(&reference.keywords, &mut system_audio_keywords), + } + evidence_refs.push(reference.clone()); + } + + let mut searchable = visual_keywords.clone(); + merge_counts(&mut searchable, &system_audio_keywords); + if searchable.is_empty() { + merge_counts(&mut searchable, µphone_keywords); + } + let hard_boundary_before = scenes.is_empty() + || (start == item.start + && item_index > 0 + && hard_boundary_between(&items[item_index - 1], item)); + let scene_id = format!("scene-{}-{}", item.segment.id, start.timestamp_millis()); + let confidence = if !visual_keywords.is_empty() { + 0.85 + } else if !system_audio_keywords.is_empty() { + 0.65 + } else if context.is_some() { + 0.5 + } else { + 0.3 + }; + let scene = RewindScene { + id: scene_id, + started_at: start.to_rfc3339(), + ended_at: end.to_rfc3339(), + duration_ms: (end - start).num_milliseconds(), + hard_boundary_before, + segment_refs: vec![SegmentRef { + id: item.segment.id.clone(), + started_at: item.segment.started_at.clone(), + ended_at: item.segment.ended_at.clone(), + }], + context, + keywords: top_keywords(&searchable, MAX_KEYWORDS), + evidence_refs: bounded_evidence_refs(evidence_refs), + confidence, + }; + scenes.push(SceneItem { + scene, + start, + end, + visual_keywords, + system_audio_keywords, + microphone_keywords, + partial: item.partial, + }); + } + } + scenes +} + +fn hard_boundary_between(previous: &Item, next: &Item) -> bool { + previous.segment.graph_epoch_id != next.segment.graph_epoch_id + || previous.segment.capture_mode != next.segment.capture_mode + || (next.start - previous.end).num_milliseconds() > HARD_GAP_MS +} + +fn context_event_at( + events: &[ScreenMemoryEvent], + start: DateTime, + end: DateTime, +) -> Option<&ScreenMemoryEvent> { + events + .iter() + .filter_map(|event| parse_utc(&event.captured_at).map(|at| (at, event))) + .filter(|(at, event)| *at <= end && event.source != "coverage-gap") + .filter(|(at, _)| *at <= start || (*at >= start && *at < end)) + .max_by_key(|(at, _)| *at) + .map(|(_, event)| event) +} + +fn chapter_context_from_event(event: &ScreenMemoryEvent) -> ChapterContext { + let document = event + .accessibility + .as_ref() + .and_then(|fingerprint| fingerprint.document.as_ref()); + ChapterContext { + app_name: event.app_name.clone(), + window_title: event.window_title.clone(), + bundle_id: event.bundle_id.clone(), + document_title: document.and_then(|node| { + node.title + .clone() + .or_else(|| node.document.clone()) + .filter(|value| !value.trim().is_empty()) + }), + document_url: document.and_then(|node| node.url.clone()), + } +} + +fn event_semantic_key(event: &ScreenMemoryEvent) -> String { + format!( + "{}|{}|{}|{}", + event.app_name.as_deref().unwrap_or_default(), + event.window_title.as_deref().unwrap_or_default(), + event.bundle_id.as_deref().unwrap_or_default(), + top_keywords(&accessibility_keywords(event), 16).join(",") + ) +} + +fn accessibility_keywords(event: &ScreenMemoryEvent) -> BTreeMap { + let mut keywords = BTreeMap::new(); + let Some(accessibility) = event.accessibility.as_ref() else { + return keywords; + }; + for node in [ + accessibility.document.as_ref(), + accessibility.focused.as_ref(), + ] + .into_iter() + .flatten() + { + // Roles describe the accessibility API, not the user's work. Ingesting + // them recursively produced labels such as "Working on AXTextArea". + for text in [ + node.title.as_deref(), + node.description.as_deref(), + node.document.as_deref(), + node.selected_text.as_deref(), + ] + .into_iter() + .flatten() + { + add_keywords(text, &mut keywords); + add_semantic_phrases(text, &mut keywords); + } + } + for label in &accessibility.visible_labels { + add_keywords(label, &mut keywords); + add_semantic_phrases(label, &mut keywords); + } + keywords +} + +fn add_keyword_list(keywords: &[String], counts: &mut BTreeMap) { + for keyword in keywords { + *counts.entry(keyword.clone()).or_default() += 1; + } +} + +fn evidence_time(reference: &EvidenceRef, segment_start: DateTime) -> Option> { + reference + .captured_at + .as_deref() + .and_then(parse_utc) + .or_else(|| { + reference + .offset_ms + .map(|offset| segment_start + Duration::milliseconds(offset.max(0))) + }) +} + +fn parse_utc(value: &str) -> Option> { + DateTime::parse_from_rfc3339(value) + .ok() + .map(|at| at.with_timezone(&Utc)) +} + +fn bounded_evidence_refs(mut references: Vec) -> Vec { + references.sort_by(|a, b| { + ( + &a.source_type, + &a.source_kind, + &a.segment_id, + &a.offset_ms, + &a.captured_at, + ) + .cmp(&( + &b.source_type, + &b.source_kind, + &b.segment_id, + &b.offset_ms, + &b.captured_at, + )) + }); + references.dedup_by(|a, b| { + a.source_type == b.source_type + && a.source_kind == b.source_kind + && a.segment_id == b.segment_id + && a.offset_ms == b.offset_ms + && a.captured_at == b.captured_at + }); + let mut by_source: BTreeMap> = BTreeMap::new(); + for reference in references { + by_source + .entry(format!( + "{}:{}", + reference.source_type, + reference.source_kind.as_deref().unwrap_or_default() + )) + .or_default() + .push(reference); + } + by_source + .into_values() + .flat_map(|references| evenly_bounded(references, MAX_EVIDENCE_REFS_PER_SOURCE)) + .collect() +} + +fn evenly_bounded(values: Vec, limit: usize) -> Vec { + if values.len() <= limit { + return values; + } + (0..limit) + .map(|index| { + let source_index = index * (values.len() - 1) / (limit - 1); + values[source_index].clone() + }) + .collect() +} +fn pending_segment_ids(dir: &Path) -> BTreeSet { + fs::read_dir(dir) + .ok() + .into_iter() + .flatten() + .filter_map(|entry| { + let entry = entry.ok()?; + let path = entry.path(); + let name = entry.file_name().to_string_lossy().to_string(); + let segment_id = name + .strip_suffix(".ocr-status.json") + .or_else(|| name.strip_suffix(".transcript-status.json"))?; + let bytes = fs::read(path).ok()?; + let state = serde_json::from_slice::(&bytes) + .ok()? + .get("state")? + .as_str()? + .to_owned(); + matches!( + state.as_str(), + "pending" | "indexing" | "transcribing" | "failed" + ) + .then_some(segment_id.to_owned()) + }) + .collect() +} + +fn content_signals( + dir: &Path, + segments: &[ScreenMemorySegmentMetadata], +) -> BTreeMap { + segments + .iter() + .filter(|segment| !segment.exclusion_tainted && !segment.corrupt && segment.error.is_none()) + .map(|segment| { + let mut signal = ContentSignal::default(); + read_sidecar_signal( + &dir.join(format!("{}.transcript.jsonl", segment.id)), + "transcript", + &segment.id, + &mut signal, + ); + read_sidecar_signal( + &dir.join(format!("{}.ocr.jsonl", segment.id)), + "ocr", + &segment.id, + &mut signal, + ); + (segment.id.clone(), signal) + }) + .collect() +} + +fn read_sidecar_signal( + path: &Path, + source_type: &str, + segment_id: &str, + signal: &mut ContentSignal, +) { + let Ok(file) = File::open(path) else { + return; + }; + for line in BufReader::new(file) + .lines() + .map_while(Result::ok) + .take(MAX_SIDECAR_ROWS) + { + if line.len() > MAX_SIDECAR_LINE_BYTES { + continue; + } + let Ok(row) = serde_json::from_str::(&line) else { + continue; + }; + let text = match source_type { + "transcript" => row.get("text"), + "ocr" => row.get("ocrText"), + _ => None, + } + .and_then(|value| value.as_str()); + let source_kind = if source_type == "transcript" { + row.get("source") + .and_then(|value| value.as_str()) + .unwrap_or("unknown-audio") + } else { + "visual" + }; + let mut row_keywords = BTreeMap::new(); + if let Some(text) = text { + add_keywords(text, &mut row_keywords); + if source_type == "ocr" { + add_semantic_phrases(text, &mut row_keywords); + } + let target = if source_type == "ocr" { + &mut signal.visual_keywords + } else if source_kind == "microphone" { + &mut signal.microphone_keywords + } else { + &mut signal.system_audio_keywords + }; + merge_counts(target, &row_keywords); + } + signal.evidence_refs.push(EvidenceRef { + source_type: source_type.into(), + source_kind: Some(source_kind.into()), + keywords: top_keywords(&row_keywords, MAX_EVIDENCE_KEYWORDS), + confidence: row + .get("confidence") + .and_then(|value| value.as_f64()) + .map(|value| value as f32), + segment_id: Some(segment_id.into()), + offset_ms: row + .get("startMs") + .or_else(|| row.get("offsetMs")) + .and_then(|value| value.as_i64()), + captured_at: row + .get("capturedAt") + .and_then(|value| value.as_str()) + .map(str::to_owned), + }); + } +} + +fn merge_counts(target: &mut BTreeMap, source: &BTreeMap) { + for (keyword, count) in source { + *target.entry(keyword.clone()).or_default() += count; + } +} + +fn top_keywords(counts: &BTreeMap, limit: usize) -> Vec { + let mut keywords = counts.iter().collect::>(); + keywords.sort_by(|(left_keyword, left_count), (right_keyword, right_count)| { + right_count + .cmp(left_count) + .then_with(|| left_keyword.cmp(right_keyword)) + }); + keywords + .into_iter() + .take(limit) + .map(|(keyword, _)| keyword.clone()) + .collect() +} + +fn add_keywords(text: &str, keywords: &mut BTreeMap) { + let lower = text.to_ascii_lowercase(); + // Skip only the credential-bearing line; a password field elsewhere in a + // frame must not erase unrelated, safe topic evidence from the whole OCR row. + for line in lower.lines().take(64) { + if credential_shaped_text(line) { + continue; + } + for word in line.split(|character: char| !character.is_ascii_alphabetic()) { + if word.len() < 4 || word.len() > 32 || STOP_WORDS.contains(&word) { + continue; + } + *keywords.entry(word.to_owned()).or_default() += 1; + } + } +} + +fn add_semantic_phrases(text: &str, keywords: &mut BTreeMap) { + let lower = text.to_ascii_lowercase(); + for line in lower.lines().take(64) { + if credential_shaped_text(line) { + continue; + } + let words = line + .split(|character: char| !character.is_ascii_alphabetic()) + .filter(|word| word.len() >= 4 && word.len() <= 24 && !STOP_WORDS.contains(word)) + .take(12) + .collect::>(); + for size in [3usize, 2] { + for phrase in words.windows(size).take(8) { + if phrase.iter().all(|word| LABEL_GENERIC_WORDS.contains(word)) { + continue; + } + *keywords.entry(phrase.join(" ")).or_default() += 1; + } + } + } +} + +fn credential_shaped_text(lower: &str) -> bool { + let marked = [ + "password", + "secret", + "api key", + "api_key", + "access token", + "bearer", + ] + .iter() + .any(|marker| lower.contains(marker)); + let email_shaped = lower.split_whitespace().any(|word| { + word.split_once('@') + .is_some_and(|(local, domain)| !local.is_empty() && domain.contains('.')) + }); + marked || email_shaped +} + +const STOP_WORDS: &[&str] = &[ + "about", "after", "again", "also", "been", "could", "from", "have", "into", "just", "like", + "more", "most", "only", "that", "their", "there", "these", "they", "this", "through", "using", + "were", "what", "when", "with", "would", "your", +]; + +fn group_semantic_scenes(scenes: &[SceneItem]) -> Vec> { + let mut groups: Vec> = Vec::new(); + for (index, scene) in scenes.iter().cloned().enumerate() { + let split = groups + .last() + .is_some_and(|group| should_split_semantically(group, &scene, &scenes[index + 1..])); + if split || groups.is_empty() { + groups.push(vec![scene]); + } else if let Some(group) = groups.last_mut() { + group.push(scene); + } + } + + // A bounded audit or retained archive can begin in the middle of a task. + // When the first observed foreground context changes almost immediately, + // there is no earlier evidence with which to prove a genuine chapter + // boundary. Keep that open-edge sliver with the first durable chapter and + // let the next rebuild recover the original boundary if more history exists. + if groups.len() > 1 { + let first_duration_ms = groups[0] + .last() + .map(|last| (last.end - groups[0][0].start).num_milliseconds()) + .unwrap_or_default(); + let next_has_hard_boundary = groups[1] + .first() + .is_some_and(|scene| scene.scene.hard_boundary_before); + if first_duration_ms < SCENE_SLICE_MS && !next_has_hard_boundary { + let first = groups.remove(0); + groups[0].splice(0..0, first); + } + } + groups +} + +fn should_split_semantically( + group: &[SceneItem], + next: &SceneItem, + following: &[SceneItem], +) -> bool { + let previous = group.last().expect("nonempty semantic group"); + if next.scene.hard_boundary_before { + return true; + } + if (next.end - group[0].start).num_milliseconds() > MAX_CHAPTER_SPAN_MS { + return true; + } + + // Window titles and accessibility summaries can change every few seconds + // inside one app. Treat the foreground application as the stable surface; + // semantic evidence below remains responsible for finding topic changes + // within that surface. + let context_changed = + !same_foreground_context(previous.scene.context.as_ref(), next.scene.context.as_ref()); + let current_group_ms = (previous.end - group[0].start).num_milliseconds(); + if current_group_ms < SCENE_SLICE_MS && (!context_changed || previous.scene.context.is_none()) { + return false; + } + let previous_semantics = if context_changed { + group + .iter() + .rev() + .map(semantic_keywords) + .find(|keywords| !keywords.is_empty()) + .unwrap_or_default() + } else { + let mut combined = BTreeMap::new(); + for scene in group.iter().rev().take(8) { + merge_counts(&mut combined, &semantic_keywords(scene)); + } + combined + }; + let next_semantics = semantic_keywords(next); + if previous.scene.context.is_none() && previous_semantics.is_empty() { + return false; + } + let overlap = keyword_overlap(&previous_semantics, &next_semantics); + + if context_changed && returns_after_transient_surface(group, next) { + return false; + } + + if context_changed + && following.iter().any(|later| { + (later.start - next.start).num_milliseconds() <= TRANSIENT_SCENE_MS + && group.iter().rev().any(|earlier| { + same_foreground_context( + later.scene.context.as_ref(), + earlier.scene.context.as_ref(), + ) + }) + }) + { + return false; + } + + if (context_changed && strong_cross_surface_continuity(&previous_semantics, &next_semantics)) + || (!context_changed && overlap >= SAME_SURFACE_CONTINUITY_OVERLAP) + { + return false; + } + + if !context_changed + && document_context_changed(previous.scene.context.as_ref(), next.scene.context.as_ref()) + && document_context_persists_for(next, following, MIN_SEMANTIC_TOPIC_MS) + { + return true; + } + + let persistent_new_context = + context_changed && context_persists_for(next, following, TRANSIENT_SCENE_MS); + if persistent_new_context { + return true; + } + + if !context_changed && !previous_semantics.is_empty() && !next_semantics.is_empty() { + return semantics_persist_for(next, following, MIN_SEMANTIC_TOPIC_MS); + } + + // A context seen only at the open edge of the retained/audited range is not + // enough evidence for a new chapter. The next rebuild can split at the + // original boundary once the foreground change has actually persisted. + false +} + +fn same_foreground_context(left: Option<&ChapterContext>, right: Option<&ChapterContext>) -> bool { + match (left, right) { + (None, None) => true, + (Some(left), Some(right)) => { + if let (Some(left_bundle), Some(right_bundle)) = + (left.bundle_id.as_deref(), right.bundle_id.as_deref()) + { + return left_bundle.eq_ignore_ascii_case(right_bundle); + } + if let (Some(left_app), Some(right_app)) = + (left.app_name.as_deref(), right.app_name.as_deref()) + { + return left_app.eq_ignore_ascii_case(right_app); + } + left.window_title == right.window_title + } + _ => false, + } +} + +fn document_context_key(context: Option<&ChapterContext>) -> Option { + let context = context?; + context + .document_url + .as_deref() + .or(context.document_title.as_deref()) + .or_else(|| { + context + .window_title + .as_deref() + .filter(|title| !generic_surface_title(title) && !transient_surface_title(title)) + }) + .map(|value| value.trim().to_ascii_lowercase()) + .filter(|value| !value.is_empty()) +} + +fn document_context_changed( + previous: Option<&ChapterContext>, + next: Option<&ChapterContext>, +) -> bool { + match (document_context_key(previous), document_context_key(next)) { + (Some(previous), Some(next)) => previous != next, + _ => false, + } +} + +fn document_context_persists_for( + next: &SceneItem, + following: &[SceneItem], + minimum_ms: i64, +) -> bool { + let Some(target) = document_context_key(next.scene.context.as_ref()) else { + return false; + }; + let mut covered_until = next.end; + if (covered_until - next.start).num_milliseconds() >= minimum_ms { + return true; + } + for later in following { + if later.scene.hard_boundary_before + || document_context_key(later.scene.context.as_ref()).as_deref() + != Some(target.as_str()) + { + break; + } + covered_until = later.end; + if (covered_until - next.start).num_milliseconds() >= minimum_ms { + return true; + } + } + false +} + +fn returns_after_transient_surface(group: &[SceneItem], next: &SceneItem) -> bool { + let Some(trailing) = group.last() else { + return false; + }; + let trailing_start = group + .iter() + .rposition(|scene| { + !same_foreground_context( + scene.scene.context.as_ref(), + trailing.scene.context.as_ref(), + ) + }) + .map_or(0, |index| index + 1); + if trailing_start == 0 + || (trailing.end - group[trailing_start].start).num_milliseconds() > TRANSIENT_SCENE_MS + { + return false; + } + group[..trailing_start].iter().rev().any(|scene| { + same_foreground_context(scene.scene.context.as_ref(), next.scene.context.as_ref()) + }) +} + +fn context_persists_for(next: &SceneItem, following: &[SceneItem], minimum_ms: i64) -> bool { + let mut covered_until = next.end; + if (covered_until - next.start).num_milliseconds() >= minimum_ms { + return true; + } + for later in following { + if later.scene.hard_boundary_before + || !same_foreground_context(later.scene.context.as_ref(), next.scene.context.as_ref()) + { + break; + } + covered_until = later.end; + if (covered_until - next.start).num_milliseconds() >= minimum_ms { + return true; + } + } + false +} + +fn semantics_persist_for(next: &SceneItem, following: &[SceneItem], minimum_ms: i64) -> bool { + let target = semantic_keywords(next); + let mut covered_until = next.end; + if (covered_until - next.start).num_milliseconds() >= minimum_ms { + return true; + } + for later in following { + if later.scene.hard_boundary_before + || !same_foreground_context(later.scene.context.as_ref(), next.scene.context.as_ref()) + || keyword_overlap(&target, &semantic_keywords(later)) < SAME_SURFACE_CONTINUITY_OVERLAP + { + break; + } + covered_until = later.end; + if (covered_until - next.start).num_milliseconds() >= minimum_ms { + return true; + } + } + false +} + +fn semantic_keywords(scene: &SceneItem) -> BTreeMap { + if !scene.visual_keywords.is_empty() { + let mut keywords = scene.visual_keywords.clone(); + for keyword in scene.system_audio_keywords.keys() { + if keywords.contains_key(keyword) { + *keywords.entry(keyword.clone()).or_default() += 1; + } + } + return keywords; + } + if !scene.system_audio_keywords.is_empty() { + return scene.system_audio_keywords.clone(); + } + scene.microphone_keywords.clone() +} + +fn keyword_overlap(left: &BTreeMap, right: &BTreeMap) -> f32 { + if left.is_empty() || right.is_empty() { + return 0.0; + } + let left = semantic_word_set(left); + let right = semantic_word_set(right); + if left.is_empty() || right.is_empty() { + return 0.0; + } + let intersection = left.intersection(&right).count(); + intersection as f32 / left.len().min(right.len()) as f32 +} + +fn strong_cross_surface_continuity( + left: &BTreeMap, + right: &BTreeMap, +) -> bool { + let left = semantic_word_set(left); + let right = semantic_word_set(right); + if left.is_empty() || right.is_empty() { + return false; + } + let shared = left.intersection(&right).count(); + let union = left.union(&right).count(); + shared >= 2 && shared as f32 / union.max(1) as f32 >= 0.60 +} + +fn semantic_word_set(keywords: &BTreeMap) -> BTreeSet { + keywords + .keys() + .flat_map(|keyword| keyword.split_whitespace()) + .filter(|word| !generic_label_word(word) && plausible_label_word(word)) + .map(str::to_owned) + .collect() +} + +fn chapter(group: Vec) -> RewindChapter { + let first = &group[0]; + let last = group.last().unwrap(); + let mut contexts = group + .iter() + .filter_map(|item| item.scene.context.clone()) + .collect::>(); + contexts.sort_by(|a, b| format!("{a:?}").cmp(&format!("{b:?}"))); + contexts.dedup(); + let partial = group.iter().any(|item| item.partial); + let mut evidence_refs = Vec::new(); + let mut visual_keywords = BTreeMap::new(); + let mut system_audio_keywords = BTreeMap::new(); + let mut microphone_keywords = BTreeMap::new(); + for item in &group { + merge_counts(&mut visual_keywords, &item.visual_keywords); + merge_counts(&mut system_audio_keywords, &item.system_audio_keywords); + merge_counts(&mut microphone_keywords, &item.microphone_keywords); + evidence_refs.extend(item.scene.evidence_refs.clone()); + } + let mut label_keywords = visual_keywords.clone(); + merge_counts(&mut label_keywords, &system_audio_keywords); + if label_keywords.is_empty() { + label_keywords = microphone_keywords.clone(); + } + let keywords = top_keywords(&label_keywords, MAX_KEYWORDS); + let label_terms = top_label_keywords(&label_keywords, 3); + let label = choose_label(&group, &contexts, &label_terms); + let scene_refs = group + .iter() + .map(|item| item.scene.id.clone()) + .collect::>(); + let mut segment_refs = group + .iter() + .flat_map(|item| item.scene.segment_refs.clone()) + .collect::>(); + segment_refs.sort_by(|a, b| { + a.started_at + .cmp(&b.started_at) + .then_with(|| a.id.cmp(&b.id)) + }); + segment_refs.dedup_by(|a, b| a.id == b.id); + let representative_moments = representative_moments(&group); + let representative_coverage = RepresentativeCoverage { + covered_scenes: representative_moments.len(), + total_scenes: group.len(), + truncated: representative_moments.len() < group.len(), + }; + let confidence = if !visual_keywords.is_empty() { + 0.85 + } else if !system_audio_keywords.is_empty() { + 0.65 + } else if !contexts.is_empty() { + 0.45 + } else { + 0.3 + }; + RewindChapter { + id: format!("chapter-{}", first.scene.id), + revision: 1, + aliases: Vec::new(), + started_at: first.scene.started_at.clone(), + ended_at: last.scene.ended_at.clone(), + duration_ms: (last.end - first.start).num_milliseconds(), + summary: if keywords.is_empty() { + format!("{label}; local semantic evidence is sparse.") + } else { + format!( + "{label}; corroborated local evidence mentions {}.", + keywords + .iter() + .take(5) + .cloned() + .collect::>() + .join(", ") + ) + }, + keywords, + label, + confidence, + scene_refs, + segment_refs, + evidence_refs: bounded_evidence_refs(evidence_refs), + contexts, + representative_moments, + representative_coverage, + ambiguity_reasons: if partial { + vec!["local indexing is still catching up".into()] + } else if visual_keywords.is_empty() && system_audio_keywords.is_empty() { + vec!["no corroborated visual, accessibility, or system-audio topic was retained".into()] + } else { + vec![] + }, + index_state: if partial { + ChapterIndexState::Partial + } else { + ChapterIndexState::Ready + }, + } +} + +fn choose_label(group: &[SceneItem], contexts: &[ChapterContext], keywords: &[String]) -> String { + let mut title_duration = BTreeMap::::new(); + for scene in group { + let Some(title) = scene + .scene + .context + .as_ref() + .and_then(|context| context.window_title.as_deref()) + else { + continue; + }; + let repeats_app_name = scene + .scene + .context + .as_ref() + .and_then(|context| context.app_name.as_deref()) + .is_some_and(|app| app.trim().eq_ignore_ascii_case(title.trim())); + if !generic_surface_title(title) && !transient_surface_title(title) && !repeats_app_name { + *title_duration.entry(title.trim().to_owned()).or_default() += scene.scene.duration_ms; + } + } + if let Some((title, _)) = title_duration + .into_iter() + .filter(|(title, duration)| *duration >= SCENE_SLICE_MS && title.len() <= 96) + .max_by_key(|(_, duration)| *duration) + { + return title; + } + if !keywords.is_empty() { + let displayed = if keywords[0].contains(' ') { + &keywords[..1] + } else { + &keywords[..keywords.len().min(3)] + }; + return format!("Working on {}", natural_keyword_list(displayed)); + } + contexts + .iter() + .find_map(|context| context.app_name.clone()) + .map(|app| format!("Work in {app} (low confidence)")) + .unwrap_or_else(|| "Recent work (low confidence)".into()) +} + +fn top_label_keywords(counts: &BTreeMap, limit: usize) -> Vec { + let weighted = counts + .iter() + .filter(|(keyword, count)| { + let words = keyword.split_whitespace().collect::>(); + !words.iter().any(|word| generic_label_word(word)) + && words.iter().all(|word| plausible_label_word(word)) + && (words.len() == 1 || **count >= 2) + }) + .map(|(keyword, count)| { + // Repeated clean phrases are more human than isolated words, but a + // fixed bonus prevents long OCR fragments from winning by length. + let words = keyword.split_whitespace().count(); + let phrase_weight = if words > 1 { words + 2 } else { 1 }; + (keyword.clone(), count.saturating_mul(phrase_weight)) + }) + .collect::>(); + let ranked = top_keywords(&weighted, weighted.len()); + let mut selected = Vec::::new(); + for candidate in ranked { + let candidate_words = candidate.split_whitespace().collect::>(); + let duplicates_selected_meaning = selected.iter().any(|selected| { + let selected_words = selected.split_whitespace().collect::>(); + let shared = candidate_words.intersection(&selected_words).count(); + shared > 0 && shared * 4 >= candidate_words.len().min(selected_words.len()) * 3 + }); + if !duplicates_selected_meaning { + selected.push(candidate); + if selected.len() == limit { + break; + } + } + } + selected +} + +fn generic_label_word(word: &str) -> bool { + if word.starts_with("ax") { + return true; + } + LABEL_GENERIC_WORDS.iter().any(|generic| { + word == *generic + || (word.len() == generic.len() + && word + .chars() + .zip(generic.chars()) + .filter(|(left, right)| left != right) + .count() + <= 1) + }) +} + +fn plausible_label_word(word: &str) -> bool { + if word.len() <= 5 { + return true; + } + let mut vowels = 0usize; + let mut consonant_run = 0usize; + let mut longest_consonant_run = 0usize; + for character in word.chars() { + if matches!(character, 'a' | 'e' | 'i' | 'o' | 'u') { + vowels += 1; + consonant_run = 0; + } else { + consonant_run += 1; + longest_consonant_run = longest_consonant_run.max(consonant_run); + } + } + vowels > 0 && longest_consonant_run <= 4 +} + +const LABEL_GENERIC_WORDS: &[&str] = &[ + "agent", + "native", + "content", + "work", + "working", + "created", + "private", + "document", + "start", + "open", + "review", + "changes", + "should", + "current", + "system", + "local", + "chatgpt", + "codex", + "claude", + "cursor", + "browser", + "terminal", + "safari", + "finder", + "axwindow", + "axwebarea", + "axapplication", +]; + +fn generic_surface_title(title: &str) -> bool { + let normalized = title.trim().to_ascii_lowercase(); + normalized.is_empty() + || [ + "chatgpt", + "codex", + "claude", + "claude code", + "cursor", + "browser", + "terminal", + "google chrome", + "safari", + "mail", + "finder", + ] + .contains(&normalized.as_str()) +} + +fn transient_surface_title(title: &str) -> bool { + let normalized = title.trim().to_ascii_lowercase(); + normalized.contains("save password") + || normalized.contains("password manager") + || normalized == "new tab" + || normalized == "downloads" + || normalized.ends_with(" notification") +} + +fn natural_keyword_list(keywords: &[String]) -> String { + match keywords { + [] => "recent work".into(), + [one] => one.clone(), + [one, two] => format!("{one} and {two}"), + [one, two, rest @ ..] => format!("{one}, {two}, and {}", rest[0]), + } +} + +fn representative_moments(group: &[SceneItem]) -> Vec { + let limit = group.len().min(MAX_REPRESENTATIVE_MOMENTS); + if limit == 0 { + return Vec::new(); + } + let selected = if group.len() <= limit { + (0..group.len()).collect::>() + } else { + (0..limit) + .map(|index| index * (group.len() - 1) / (limit - 1)) + .collect::>() + }; + selected + .into_iter() + .filter_map(|index| { + let item = &group[index]; + let segment = item.scene.segment_refs.first()?; + let segment_start = parse_utc(&segment.started_at)?; + Some(RepresentativeMoment { + moment_id: format!("moment-{}", item.scene.id), + captured_at: item.scene.started_at.clone(), + segment_id: segment.id.clone(), + offset_ms: (item.start - segment_start).num_milliseconds().max(0), + reason: if index == 0 { + "chapter start" + } else if index + 1 == group.len() { + "chapter end" + } else { + "semantic scene change" + } + .into(), + }) + }) + .collect() +} + +fn apply_previous_identity( + chapters: &mut [RewindChapter], + previous: Option<&RewindChaptersManifest>, +) { + let Some(previous) = previous else { + return; + }; + for chapter in chapters { + if let Some(prior) = previous + .chapters + .iter() + .find(|prior| prior.id == chapter.id) + { + chapter.aliases = prior.aliases.clone(); + chapter.revision = if chapter_semantics_equal(prior, chapter) { + prior.revision + } else { + prior.revision.saturating_add(1) + }; + } + for prior in &previous.chapters { + let Some(first_scene) = prior.scene_refs.first() else { + continue; + }; + if prior.id != chapter.id + && chapter.scene_refs.contains(first_scene) + && !chapter.aliases.contains(&prior.id) + { + chapter.aliases.push(prior.id.clone()); + } + } + chapter.aliases.sort(); + chapter.aliases.dedup(); + } +} + +fn chapter_semantics_equal(previous: &RewindChapter, current: &RewindChapter) -> bool { + previous.started_at == current.started_at + && previous.ended_at == current.ended_at + && previous.scene_refs == current.scene_refs + && previous.label == current.label + && previous.summary == current.summary + && previous.keywords == current.keywords + && previous.ambiguity_reasons == current.ambiguity_reasons + && previous.representative_moments.len() == current.representative_moments.len() +} +fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), String> { + let temporary = path.with_extension("json.tmp"); + fs::write(&temporary, bytes).map_err(|e| format!("chapter manifest write failed: {e}"))?; + fs::rename(temporary, path).map_err(|e| format!("chapter manifest replace failed: {e}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + fn segment(id: &str, start: &str, end: &str) -> ScreenMemorySegmentMetadata { + ScreenMemorySegmentMetadata { + id: id.into(), + path: PathBuf::from("/private/media.mp4"), + file_name: "media.mp4".into(), + mime_type: "video/mp4".into(), + started_at: start.into(), + ended_at: end.into(), + duration_ms: 60_000, + width: None, + height: None, + bytes: 1, + system_audio_path: None, + microphone_path: None, + corrupt: false, + error: None, + capture_mode: RewindCaptureMode::Visuals, + exclusion_tainted: false, + graph_epoch_id: Some("a".into()), + graph_started_elapsed_ms: 0, + graph_ended_elapsed_ms: 60_000, + } + } + fn event(at: &str, app: &str) -> ScreenMemoryEvent { + ScreenMemoryEvent { + captured_at: at.into(), + app_name: Some(app.into()), + window_title: Some(app.into()), + bundle_id: None, + source: "foreground".into(), + coverage_gap_reason: None, + accessibility: None, + } + } + fn evidence( + segment_id: &str, + source_type: &str, + source_kind: &str, + captured_at: &str, + words: &[&str], + ) -> EvidenceRef { + EvidenceRef { + source_type: source_type.into(), + source_kind: Some(source_kind.into()), + keywords: words.iter().map(|word| (*word).into()).collect(), + confidence: Some(0.9), + segment_id: Some(segment_id.into()), + offset_ms: None, + captured_at: Some(captured_at.into()), + } + } + fn signal_with(references: Vec) -> ContentSignal { + let mut signal = ContentSignal::default(); + for reference in references { + let target = match reference.source_kind.as_deref() { + Some("visual" | "accessibility") => &mut signal.visual_keywords, + Some("microphone") => &mut signal.microphone_keywords, + _ => &mut signal.system_audio_keywords, + }; + add_keyword_list(&reference.keywords, target); + signal.evidence_refs.push(reference); + } + signal + } + #[test] + fn same_context_merges_and_serialization_has_no_paths() { + let value = build( + vec![ + segment("b", "2026-01-01T00:01:00Z", "2026-01-01T00:02:00Z"), + segment("a", "2026-01-01T00:00:00Z", "2026-01-01T00:01:00Z"), + ], + vec![event("2026-01-01T00:00:30Z", "Editor")], + Utc::now(), + ); + assert_eq!(value.chapters.len(), 1, "{:#?}", value.chapters); + assert!(!String::from_utf8(serde_json::to_vec(&value).unwrap()) + .unwrap() + .contains("/private/")); + } + #[test] + fn persistent_context_splits_but_short_switch_does_not() { + let a = segment("a", "2026-01-01T00:00:00Z", "2026-01-01T00:01:00Z"); + let b = segment("b", "2026-01-01T00:01:00Z", "2026-01-01T00:02:00Z"); + let c = segment("c", "2026-01-01T00:02:00Z", "2026-01-01T00:03:00Z"); + assert_eq!( + build( + vec![a.clone(), b.clone(), c.clone()], + vec![ + event("2026-01-01T00:00:30Z", "A"), + event("2026-01-01T00:01:30Z", "B"), + event("2026-01-01T00:02:30Z", "B") + ], + Utc::now() + ) + .chapters + .len(), + 2 + ); + assert_eq!( + build( + vec![a, b, c], + vec![ + event("2026-01-01T00:00:30Z", "A"), + event("2026-01-01T00:01:30Z", "B"), + event("2026-01-01T00:02:30Z", "A") + ], + Utc::now() + ) + .chapters + .len(), + 1 + ); + } + + #[test] + fn multiple_short_foreground_interruptions_do_not_split_the_underlying_task() { + let segments = vec![ + segment("before", "2026-01-01T00:00:00Z", "2026-01-01T00:01:00Z"), + segment("maps", "2026-01-01T00:01:00Z", "2026-01-01T00:01:30Z"), + segment("message", "2026-01-01T00:01:30Z", "2026-01-01T00:02:00Z"), + segment("after", "2026-01-01T00:02:00Z", "2026-01-01T00:03:00Z"), + ]; + let content = BTreeMap::from([ + ( + "before".into(), + signal_with(vec![evidence( + "before", + "ocr", + "visual", + "2026-01-01T00:00:10Z", + &["capability", "catalog"], + )]), + ), + ( + "maps".into(), + signal_with(vec![evidence( + "maps", + "ocr", + "visual", + "2026-01-01T00:01:05Z", + &["franklin", "route"], + )]), + ), + ( + "message".into(), + signal_with(vec![evidence( + "message", + "ocr", + "visual", + "2026-01-01T00:01:35Z", + &["compose", "message"], + )]), + ), + ( + "after".into(), + signal_with(vec![evidence( + "after", + "ocr", + "visual", + "2026-01-01T00:02:05Z", + &["capability", "catalog"], + )]), + ), + ]); + let manifest = build_with_pending( + segments, + vec![ + event("2026-01-01T00:00:01Z", "Browser"), + event("2026-01-01T00:01:01Z", "Maps"), + event("2026-01-01T00:01:31Z", "Messages"), + event("2026-01-01T00:02:01Z", "Browser"), + ], + BTreeSet::new(), + content, + Utc::now(), + ); + + assert_eq!(manifest.chapters.len(), 1, "{:#?}", manifest.chapters); + } + + #[test] + fn window_title_churn_inside_one_foreground_app_does_not_make_micro_chapters() { + let mut long = segment("maps", "2026-01-01T00:00:00Z", "2026-01-01T00:03:00Z"); + long.duration_ms = 180_000; + let mut first = event("2026-01-01T00:00:05Z", "Safari"); + first.bundle_id = Some("com.apple.Safari".into()); + first.window_title = Some("Franklin - Google Maps".into()); + let mut second = event("2026-01-01T00:00:15Z", "Safari"); + second.bundle_id = Some("com.apple.Safari".into()); + second.window_title = Some("Directions to Franklin - Google Maps".into()); + let mut third = event("2026-01-01T00:00:25Z", "Safari"); + third.bundle_id = Some("com.apple.Safari".into()); + third.window_title = Some("Fishers to Franklin - Google Maps".into()); + + let manifest = build(vec![long], vec![first, second, third], Utc::now()); + + assert_eq!(manifest.chapters.len(), 1, "{:#?}", manifest.chapters); + } + + #[test] + fn first_observed_context_does_not_leave_a_tiny_open_edge_chapter() { + let mut long = segment("edge", "2026-01-01T00:00:00Z", "2026-01-01T00:03:00Z"); + long.duration_ms = 180_000; + + let manifest = build( + vec![long], + vec![event("2026-01-01T00:00:05Z", "Editor")], + Utc::now(), + ); + + assert_eq!(manifest.chapters.len(), 1, "{:#?}", manifest.chapters); + } + + #[test] + fn bounded_audit_does_not_leave_a_tiny_first_context_chapter() { + let mut long = segment("edge", "2026-01-01T00:00:00Z", "2026-01-01T00:03:00Z"); + long.duration_ms = 180_000; + + let manifest = build( + vec![long], + vec![ + event("2026-01-01T00:00:01Z", "Browser"), + event("2026-01-01T00:00:08Z", "Messages"), + event("2026-01-01T00:00:24Z", "Browser"), + ], + Utc::now(), + ); + + assert_eq!(manifest.chapters.len(), 1, "{:#?}", manifest.chapters); + } + #[test] + fn gaps_epochs_modes_and_tainted_segments_do_not_bridge() { + let a = segment("a", "2026-01-01T00:00:00Z", "2026-01-01T00:01:00Z"); + let mut b = segment("b", "2026-01-01T00:03:00Z", "2026-01-01T00:04:00Z"); + b.graph_epoch_id = Some("b".into()); + let mut bad = segment("bad", "2026-01-01T00:04:00Z", "2026-01-01T00:05:00Z"); + bad.exclusion_tainted = true; + let v = build(vec![a, b, bad], vec![], Utc::now()); + assert_eq!(v.chapters.len(), 2); + assert_eq!(v.coverage.omitted_segments, 1); + } + + #[test] + fn pending_indexes_are_partial_and_retention_rebuild_removes_references() { + let a = segment("a", "2026-01-01T00:00:00Z", "2026-01-01T00:01:00Z"); + let b = segment("b", "2026-01-01T00:01:00Z", "2026-01-01T00:02:00Z"); + let mut pending = BTreeSet::new(); + pending.insert("a".into()); + let partial = build_with_pending( + vec![a.clone(), b.clone()], + vec![], + pending, + BTreeMap::new(), + Utc::now(), + ); + assert_eq!(partial.state, ChapterIndexState::Partial); + let retained = build(vec![a], vec![], Utc::now()); + assert!(retained.chapters.iter().all(|chapter| chapter + .segment_refs + .iter() + .all(|reference| reference.id != "b"))); + } + + #[test] + fn bounded_content_keywords_are_searchable_without_raw_sidecar_bodies() { + let segment = segment("a", "2026-01-01T00:00:00Z", "2026-01-01T00:01:00Z"); + let mut signal = ContentSignal::default(); + add_keywords( + "Roadmap launch planning for the customer workshop.", + &mut signal.system_audio_keywords, + ); + add_keywords( + "password=ultra-secret-squirrel should never be retained", + &mut signal.system_audio_keywords, + ); + signal.evidence_refs.push(EvidenceRef { + source_type: "transcript".into(), + source_kind: Some("system-audio".into()), + keywords: vec!["roadmap".into(), "workshop".into()], + confidence: None, + segment_id: Some("a".into()), + offset_ms: Some(500), + captured_at: Some("2026-01-01T00:00:00.500Z".into()), + }); + let mut content = BTreeMap::new(); + content.insert("a".into(), signal); + let manifest = + build_with_pending(vec![segment], vec![], BTreeSet::new(), content, Utc::now()); + let serialized = String::from_utf8(serde_json::to_vec(&manifest).unwrap()).unwrap(); + assert!(serialized.contains("roadmap")); + assert!(serialized.contains("workshop")); + assert!(serialized.contains("\"sourceType\":\"transcript\"")); + assert!(!serialized.contains("ultra-secret-squirrel")); + assert!(!serialized.contains("password=")); + assert!(!serialized.contains("Roadmap launch planning for the customer workshop.")); + + let mut mixed = BTreeMap::new(); + add_keywords( + "Semantic chapter evaluation\nalice@example.test\nPassword: example-value", + &mut mixed, + ); + assert!(mixed.contains_key("semantic") && mixed.contains_key("chapter")); + assert!(!mixed.contains_key("alice") && !mixed.contains_key("password")); + } + + #[test] + fn chapter_labels_do_not_repeat_the_same_word_as_three_phrases() { + let counts = BTreeMap::from([ + ("account".into(), 10), + ("account passed".into(), 9), + ("account passed provisioning".into(), 8), + ("deploy".into(), 7), + ("preview".into(), 6), + ]); + + assert_eq!( + top_label_keywords(&counts, 3), + vec!["account passed provisioning", "deploy", "preview"] + ); + } + + #[test] + fn accessibility_roles_and_ocr_garble_do_not_become_labels() { + let mut context = event("2026-01-01T00:00:00Z", "WhatsApp"); + context.accessibility = Some(crate::accessibility::AccessibilityFingerprint { + document: Some(crate::accessibility::AccessibilitySemanticNode { + role: "AXWindow".into(), + title: Some("WhatsApp".into()), + description: None, + document: None, + url: None, + selected_text: None, + }), + focused: Some(crate::accessibility::AccessibilitySemanticNode { + role: "AXTextArea".into(), + title: None, + description: Some("Compose message".into()), + document: None, + url: None, + selected_text: None, + }), + visible_labels: vec![], + }); + let keywords = accessibility_keywords(&context); + assert!(!keywords.contains_key("axwindow")); + assert!(!keywords.contains_key("axtextarea")); + assert!(keywords.contains_key("compose")); + assert_eq!( + top_label_keywords( + &BTreeMap::from([ + ("axtextarea".into(), 20), + ("netltfy".into(), 18), + ("capability catalog".into(), 3), + ("catalog".into(), 8), + ]), + 2, + ), + vec!["capability catalog"] + ); + } + + #[test] + fn sustained_same_app_semantic_change_splits_fixed_scenes() { + let origin = parse_utc("2026-01-01T00:00:00Z").unwrap(); + let mut long = segment("same-app", "2026-01-01T00:00:00Z", "2026-01-01T00:05:00Z"); + long.duration_ms = 300_000; + let mut references = + repeating_evidence("same-app", "visual", origin, 5, &["custom", "blocks"]); + references.extend(repeating_evidence( + "same-app", + "visual", + origin + Duration::seconds(150), + 5, + &["daily", "brief"], + )); + let signal = signal_with(references); + let manifest = build_with_pending( + vec![long], + vec![event("2026-01-01T00:00:01Z", "ChatGPT")], + BTreeSet::new(), + BTreeMap::from([("same-app".into(), signal)]), + Utc::now(), + ); + assert_eq!(manifest.chapters.len(), 2, "{:#?}", manifest.chapters); + assert!(manifest.chapters[0].keywords.contains(&"blocks".into())); + assert!(manifest.chapters[1].keywords.contains(&"brief".into())); + } + + #[test] + fn cross_app_scenes_merge_when_visual_meaning_continues() { + let browser = segment("browser", "2026-01-01T00:00:00Z", "2026-01-01T00:01:00Z"); + let mail = segment("mail", "2026-01-01T00:01:00Z", "2026-01-01T00:02:00Z"); + let content = BTreeMap::from([ + ( + "browser".into(), + signal_with(vec![evidence( + "browser", + "ocr", + "visual", + "2026-01-01T00:00:35Z", + &["preview", "verification"], + )]), + ), + ( + "mail".into(), + signal_with(vec![evidence( + "mail", + "ocr", + "visual", + "2026-01-01T00:01:05Z", + &["preview", "verification"], + )]), + ), + ]); + let manifest = build_with_pending( + vec![browser, mail], + vec![ + event("2026-01-01T00:00:01Z", "Browser"), + event("2026-01-01T00:01:01Z", "Mail"), + ], + BTreeSet::new(), + content, + Utc::now(), + ); + assert_eq!(manifest.chapters.len(), 1, "{:#?}", manifest.chapters); + } + + #[test] + fn unrelated_microphone_words_do_not_name_visible_work() { + let item = segment("visual", "2026-01-01T00:00:00Z", "2026-01-01T00:01:00Z"); + let signal = signal_with(vec![ + evidence( + "visual", + "ocr", + "visual", + "2026-01-01T00:00:05Z", + &["content", "blocks"], + ), + evidence( + "visual", + "transcript", + "microphone", + "2026-01-01T00:00:05Z", + &["dragons", "kingdom"], + ), + ]); + let manifest = build_with_pending( + vec![item], + vec![event("2026-01-01T00:00:01Z", "ChatGPT")], + BTreeSet::new(), + BTreeMap::from([("visual".into(), signal)]), + Utc::now(), + ); + assert!( + manifest.chapters[0].label.contains("blocks"), + "{}", + manifest.chapters[0].label + ); + assert!(!manifest.chapters[0].label.contains("dragons")); + assert!(!manifest.chapters[0].keywords.contains(&"kingdom".into())); + } + + #[test] + fn exact_gaps_and_stable_ids_survive_semantic_refinement() { + let first = segment("first", "2026-01-01T00:00:00Z", "2026-01-01T00:01:00Z"); + let second = segment("second", "2026-01-01T00:03:00Z", "2026-01-01T00:04:00Z"); + let explicit_pause = ScreenMemoryEvent { + captured_at: "2026-01-01T00:01:00Z".into(), + app_name: None, + window_title: None, + bundle_id: None, + source: "coverage-gap".into(), + coverage_gap_reason: Some("user-paused".into()), + accessibility: None, + }; + let initial = build_with_pending( + vec![first.clone(), second.clone()], + vec![explicit_pause.clone()], + BTreeSet::new(), + BTreeMap::new(), + Utc::now(), + ); + assert_eq!(initial.coverage.gap_count, 1); + assert_eq!(initial.coverage.gaps[0].duration_ms, 120_000); + assert_eq!(initial.coverage.gaps[0].reason, "user-paused"); + let refined = build_with_previous( + vec![first, second], + vec![explicit_pause], + BTreeSet::new(), + BTreeMap::from([( + "first".into(), + signal_with(vec![evidence( + "first", + "ocr", + "visual", + "2026-01-01T00:00:05Z", + &["roadmap"], + )]), + )]), + Some(&initial), + Utc::now(), + ); + assert_eq!(refined.chapters[0].id, initial.chapters[0].id); + assert!(refined.chapters[0].revision > initial.chapters[0].revision); + assert_eq!(refined.coverage.gaps[0].duration_ms, 120_000); + } + + #[test] + fn representative_moments_cover_diverse_scenes_with_budget_warning() { + let mut long = segment("long", "2026-01-01T00:00:00Z", "2026-01-01T00:04:00Z"); + long.duration_ms = 240_000; + let manifest = build(vec![long], vec![], Utc::now()); + let chapter = &manifest.chapters[0]; + assert_eq!( + chapter.representative_moments.len(), + MAX_REPRESENTATIVE_MOMENTS + ); + assert_eq!(chapter.representative_coverage.total_scenes, 8); + assert!(chapter.representative_coverage.truncated); + assert_eq!(chapter.representative_moments[0].reason, "chapter start"); + assert_eq!( + chapter.representative_moments.last().unwrap().reason, + "chapter end" + ); + } + + #[test] + fn ocr_near_miss_of_generic_product_word_does_not_name_a_chapter() { + let counts = BTreeMap::from([ + ("agent native conlent".into(), 8), + ("conlent".into(), 8), + ("semantic".into(), 5), + ("chapters".into(), 5), + ]); + let labels = top_label_keywords(&counts, 3); + assert_eq!(labels, vec!["chapters", "semantic"]); + assert!(top_label_keywords( + &BTreeMap::from([("chatgpt".into(), 10), ("axwindow".into(), 10)]), + 3 + ) + .is_empty()); + } + + fn long_segment(id: &str, start: &str, end: &str) -> ScreenMemorySegmentMetadata { + let mut value = segment(id, start, end); + value.duration_ms = (parse_utc(end).unwrap() - parse_utc(start).unwrap()) + .num_milliseconds() + .try_into() + .unwrap(); + value + } + + fn repeating_evidence( + segment_id: &str, + source_kind: &str, + start: DateTime, + count: usize, + words: &[&str], + ) -> Vec { + (0..count) + .map(|index| { + evidence( + segment_id, + if source_kind == "visual" { + "ocr" + } else { + "transcript" + }, + source_kind, + &(start + Duration::seconds(index as i64 * 30 + 5)).to_rfc3339(), + words, + ) + }) + .collect() + } + + fn boundary_offsets(manifest: &RewindChaptersManifest, origin: DateTime) -> Vec { + manifest + .chapters + .iter() + .skip(1) + .filter_map(|chapter| parse_utc(&chapter.started_at)) + .map(|started_at| (started_at - origin).num_milliseconds()) + .collect() + } + + fn boundary_score( + predicted: &[i64], + expected: &[i64], + tolerance_ms: i64, + ) -> (usize, usize, usize) { + let mut used = vec![false; expected.len()]; + let matched = predicted + .iter() + .filter(|prediction| { + expected + .iter() + .enumerate() + .filter(|(index, _)| !used[*index]) + .min_by_key(|(_, boundary)| (*prediction - **boundary).abs()) + .filter(|(_, boundary)| (*prediction - **boundary).abs() <= tolerance_ms) + .map(|(index, _)| { + used[index] = true; + }) + .is_some() + }) + .count(); + (matched, predicted.len(), expected.len()) + } + + #[test] + fn three_repository_safe_gold_samples_meet_boundary_precision_and_recall_gate() { + // These hand-authored annotations preserve only the timing and topic + // shape of the private dogfood cases: cross-app continuation, a + // same-app topic change, and a short interruption plus a hard gap. + let origin = parse_utc("2026-01-01T00:00:00Z").unwrap(); + + let cross_app_segments = vec![ + long_segment("browser-a", "2026-01-01T00:00:00Z", "2026-01-01T00:05:00Z"), + long_segment("mail", "2026-01-01T00:05:00Z", "2026-01-01T00:10:00Z"), + long_segment("browser-b", "2026-01-01T00:10:00Z", "2026-01-01T00:15:00Z"), + ]; + let cross_app_content = BTreeMap::from([ + ( + "browser-a".into(), + signal_with(repeating_evidence( + "browser-a", + "visual", + origin, + 10, + &["account", "verification"], + )), + ), + ( + "mail".into(), + signal_with(repeating_evidence( + "mail", + "visual", + origin + Duration::minutes(5), + 10, + &["account", "verification"], + )), + ), + ( + "browser-b".into(), + signal_with(repeating_evidence( + "browser-b", + "visual", + origin + Duration::minutes(10), + 10, + &["account", "verification"], + )), + ), + ]); + let cross_app = build_with_pending( + cross_app_segments, + vec![ + event("2026-01-01T00:00:00Z", "Browser"), + event("2026-01-01T00:05:00Z", "Mail"), + event("2026-01-01T00:10:00Z", "Browser"), + ], + BTreeSet::new(), + cross_app_content, + Utc::now(), + ); + + let same_app_segment = + long_segment("agent", "2026-01-01T00:00:00Z", "2026-01-01T00:15:00Z"); + let mut same_app_refs = + repeating_evidence("agent", "visual", origin, 15, &["release", "roadmap"]); + same_app_refs.extend(repeating_evidence( + "agent", + "visual", + origin + Duration::seconds(450), + 15, + &["customer", "interview"], + )); + let same_app = build_with_pending( + vec![same_app_segment], + vec![event("2026-01-01T00:00:00Z", "Agent")], + BTreeSet::new(), + BTreeMap::from([("agent".into(), signal_with(same_app_refs))]), + Utc::now(), + ); + + let interrupted_segments = vec![ + long_segment("before-gap", "2026-01-01T00:00:00Z", "2026-01-01T00:10:00Z"), + long_segment("after-gap", "2026-01-01T00:11:00Z", "2026-01-01T00:20:00Z"), + ]; + let mut before_gap = + repeating_evidence("before-gap", "visual", origin, 20, &["privacy", "review"]); + before_gap.extend(repeating_evidence( + "before-gap", + "microphone", + origin, + 20, + &["dragon", "kingdom"], + )); + let interrupted = build_with_pending( + interrupted_segments, + vec![ + event("2026-01-01T00:00:00Z", "Editor"), + event("2026-01-01T00:05:00Z", "Messages"), + event("2026-01-01T00:05:30Z", "Editor"), + event("2026-01-01T00:11:00Z", "Terminal"), + ], + BTreeSet::new(), + BTreeMap::from([ + ("before-gap".into(), signal_with(before_gap)), + ( + "after-gap".into(), + signal_with(repeating_evidence( + "after-gap", + "visual", + origin + Duration::minutes(11), + 18, + &["chapter", "tests"], + )), + ), + ]), + Utc::now(), + ); + + let scored = [ + (boundary_offsets(&cross_app, origin), vec![]), + (boundary_offsets(&same_app, origin), vec![450_000]), + (boundary_offsets(&interrupted, origin), vec![660_000]), + ]; + let (mut matched, mut predicted, mut expected) = (0, 0, 0); + for (actual, annotated) in scored { + let score = boundary_score(&actual, &annotated, 30_000); + matched += score.0; + predicted += score.1; + expected += score.2; + } + let precision = matched as f32 / predicted.max(1) as f32; + let recall = matched as f32 / expected.max(1) as f32; + assert!( + precision >= 0.8, + "precision={precision}, predicted={predicted}" + ); + assert!(recall >= 0.8, "recall={recall}, expected={expected}"); + assert_eq!(cross_app.chapters.len(), 1, "{:#?}", cross_app.chapters); + let recognizable_titles = [ + cross_app.chapters[0].label.contains("verification"), + same_app.chapters[0].label.contains("roadmap"), + same_app.chapters[1].label.contains("interview"), + interrupted.chapters[0].label.contains("privacy"), + interrupted.chapters[1].label.contains("chapter"), + ]; + assert!( + recognizable_titles.iter().filter(|clear| **clear).count() as f32 + / recognizable_titles.len() as f32 + >= 0.8, + "gold titles: cross_app={:?}, same_app={:?}, interrupted={:?}", + cross_app + .chapters + .iter() + .map(|chapter| &chapter.label) + .collect::>(), + same_app + .chapters + .iter() + .map(|chapter| &chapter.label) + .collect::>(), + interrupted + .chapters + .iter() + .map(|chapter| &chapter.label) + .collect::>(), + ); + assert!(!interrupted.chapters[0].label.contains("dragon")); + } + + /// Read-only dogfood harness. It is ignored unless a developer explicitly + /// supplies a local Screen Memory directory; no archive path is committed. + #[test] + #[ignore = "requires CLIPS_REWIND_AUDIT_DIR and prints local derived evidence"] + fn audits_an_explicit_local_store_without_rewriting_it() { + let directory = std::env::var_os("CLIPS_REWIND_AUDIT_DIR") + .map(PathBuf::from) + .expect("set CLIPS_REWIND_AUDIT_DIR"); + let minutes = std::env::var("CLIPS_REWIND_AUDIT_MINUTES") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(20) + .clamp(1, 120); + let mut all_segments = fs::read_dir(&directory) + .expect("read audit directory") + .flatten() + .filter_map(|entry| { + let path = entry.path(); + (path.extension().and_then(|value| value.to_str()) == Some("json")) + .then(|| fs::read(path).ok()) + .flatten() + .and_then(|bytes| { + serde_json::from_slice::(&bytes).ok() + }) + }) + .collect::>(); + let newest_end = all_segments + .iter() + .filter_map(|segment| parse_utc(&segment.ended_at)) + .max() + .expect("at least one segment"); + let audit_end = std::env::var("CLIPS_REWIND_AUDIT_END_AT") + .ok() + .and_then(|value| parse_utc(&value)) + .unwrap_or(newest_end); + let cutoff = audit_end - Duration::minutes(minutes); + all_segments.retain(|segment| { + parse_utc(&segment.started_at).is_some_and(|started_at| started_at <= audit_end) + && parse_utc(&segment.ended_at).is_some_and(|ended_at| ended_at >= cutoff) + }); + for segment in &mut all_segments { + let started_at = parse_utc(&segment.started_at).unwrap_or(cutoff).max(cutoff); + let ended_at = parse_utc(&segment.ended_at) + .unwrap_or(audit_end) + .min(audit_end); + segment.started_at = started_at.to_rfc3339(); + segment.ended_at = ended_at.to_rfc3339(); + segment.duration_ms = (ended_at - started_at).num_milliseconds().max(0) as u128; + } + let events = File::open(directory.join("events.jsonl")) + .ok() + .into_iter() + .flat_map(|file| BufReader::new(file).lines().map_while(Result::ok)) + .filter_map(|line| serde_json::from_str::(&line).ok()) + .filter(|event| { + parse_utc(&event.captured_at).is_some_and(|at| at >= cutoff && at <= audit_end) + }) + .collect::>(); + let content = content_signals(&directory, &all_segments); + let manifest = + build_with_pending(all_segments, events, BTreeSet::new(), content, Utc::now()); + let report = serde_json::json!({ + "coverage": manifest.coverage, + "chapters": manifest.chapters.iter().map(|chapter| serde_json::json!({ + "id": chapter.id, + "startedAt": chapter.started_at, + "endedAt": chapter.ended_at, + "label": chapter.label, + "keywords": chapter.keywords, + "sceneCount": chapter.scene_refs.len(), + "representativeCoverage": chapter.representative_coverage, + "ambiguityReasons": chapter.ambiguity_reasons, + })).collect::>(), + }); + println!("{}", serde_json::to_string_pretty(&report).unwrap()); + } +} diff --git a/templates/clips/desktop/src-tauri/src/rewind_clip.rs b/templates/clips/desktop/src-tauri/src/rewind_clip.rs new file mode 100644 index 0000000000..dd43703d0c --- /dev/null +++ b/templates/clips/desktop/src-tauri/src/rewind_clip.rs @@ -0,0 +1,1650 @@ +use crate::capture_graph::{CaptureConsumer, CaptureGraphState, CaptureInterval, CaptureSource}; +use crate::native_screen::{ + self, FinalizedNativeArtifact, NativeAudioSelection, NativeFullscreenSaveResult, + NativeFullscreenUploadResult, NativeMediaSlice, NativeUploadMode, +}; +use crate::screen_memory::{self, ScreenMemorySegmentMetadata}; +use chrono::{DateTime, Utc}; +use serde::Serialize; +use std::collections::BTreeSet; +use std::path::PathBuf; +#[cfg(target_os = "macos")] +use std::process::{Command, Stdio}; +use std::sync::Mutex; +use tauri::{AppHandle, Manager, State}; + +const RETROSPECTIVE_30_SECONDS: u64 = 30; +const RETROSPECTIVE_5_MINUTES: u64 = 5 * 60; +const REWIND_CLIP_AUDIO_OWNER: &str = "rewind-clip"; + +#[derive(Default)] +pub(crate) struct RewindClipState(Mutex>); + +pub(crate) fn is_active(app: &AppHandle) -> bool { + app.try_state::() + .and_then(|state| { + state + .0 + .lock() + .ok() + .map(|active| active.as_ref().is_some_and(|clip| clip.activated)) + }) + .unwrap_or(false) +} + +struct ActiveRewindClip { + activated: bool, + lease_id: Option, + pin_id: String, + started_elapsed_ms: u64, + pinned_segments: BTreeSet, + sources: Vec, + retrospective_seconds: u64, + intervals: Vec, + paused: bool, + temporary_audio: Option, + #[cfg(target_os = "macos")] + shared_sink: Option, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum RewindClipCompatibility { + Compatible, + NotCompatible, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RewindClipStatus { + compatibility: RewindClipCompatibility, + active: bool, + retrospective_seconds: u64, + paused: bool, + sources: Vec, +} + +fn not_compatible(detail: impl AsRef) -> String { + format!("rewind-not-compatible: {}", detail.as_ref()) +} + +#[tauri::command] +pub(crate) fn rewind_clip_status( + app: AppHandle, + state: State<'_, RewindClipState>, +) -> Result { + let active_guard = state.0.lock().map_err(|error| error.to_string())?; + let active = active_guard.as_ref().is_some_and(|clip| clip.activated); + let compatible = screen_memory::rewind_clip_compatible(&app)?; + Ok(RewindClipStatus { + compatibility: if compatible { + RewindClipCompatibility::Compatible + } else { + RewindClipCompatibility::NotCompatible + }, + active, + retrospective_seconds: active_guard + .as_ref() + .map(|active| active.retrospective_seconds) + .unwrap_or(0), + paused: active_guard.as_ref().is_some_and(|active| active.paused), + sources: active_guard + .as_ref() + .map(|active| active.sources.clone()) + .unwrap_or_else(|| screen_memory::rewind_clip_sources(&app)), + }) +} + +/// Perform every potentially slow operation before the numeric countdown. +/// The resulting secondary writer shares Rewind's physical producer but is +/// still closed to samples until `rewind_clip_start` activates it at zero. +#[tauri::command] +pub(crate) fn rewind_clip_prepare( + app: AppHandle, + state: State<'_, RewindClipState>, + artifact_label: String, + server_url: Option, + recording_id: Option, + auth_token: Option, + cookie: Option, + include_mic: bool, + include_system_audio: bool, + has_camera: bool, +) -> Result { + if !screen_memory::rewind_clip_compatible(&app)? { + return Err(not_compatible( + "Screen Memory is unavailable or not recording", + )); + } + if state.0.lock().map_err(|error| error.to_string())?.is_some() { + return Err("a Rewind-derived clip is already prepared or active".into()); + } + let temporary_audio = if include_mic || include_system_audio { + screen_memory::acquire_temporary_audio_consumer( + &app, + REWIND_CLIP_AUDIO_OWNER, + CaptureConsumer::Clip, + include_mic, + include_system_audio, + )? + } else { + None + }; + if (include_mic || include_system_audio) && temporary_audio.is_none() { + return Err(not_compatible("Screen Memory has no active audio producer")); + } + let sources = screen_memory::rewind_clip_sources(&app); + let output = artifact_path(&app, &artifact_label)?; + #[cfg(target_os = "macos")] + let shared_sink = match screen_memory::prepare_shared_clip_sink( + &app, + output, + include_mic, + include_system_audio, + has_camera, + native_screen::ClipLiveUploadConfig { + server_url, + recording_id, + auth_token, + cookie, + }, + ) { + Ok(sink) => sink, + Err(error) => { + release_temporary_audio(&app, temporary_audio); + return Err(error); + } + }; + #[cfg(not(target_os = "macos"))] + { + let _ = ( + output, + server_url, + recording_id, + auth_token, + cookie, + has_camera, + ); + release_temporary_audio(&app, temporary_audio); + return Err(not_compatible("shared Rewind Clip sinks require macOS")); + } + let response_sources = sources.clone(); + let mut active = state.0.lock().map_err(|error| error.to_string())?; + if active.is_some() { + drop(active); + #[cfg(target_os = "macos")] + shared_sink.cancel(); + release_temporary_audio(&app, temporary_audio); + return Err("a Rewind-derived clip is already prepared or active".into()); + } + *active = Some(ActiveRewindClip { + activated: false, + pin_id: format!("rewind-clip-{}", Utc::now().timestamp_micros()), + lease_id: None, + started_elapsed_ms: 0, + pinned_segments: BTreeSet::new(), + sources, + retrospective_seconds: 0, + intervals: Vec::new(), + paused: false, + temporary_audio, + #[cfg(target_os = "macos")] + shared_sink: Some(shared_sink), + }); + Ok(RewindClipStatus { + compatibility: RewindClipCompatibility::Compatible, + active: false, + retrospective_seconds: 0, + paused: false, + sources: response_sources, + }) +} + +/// Countdown zero/Enter boundary. Preparation has already installed the Clip +/// writer, so this path performs only in-memory graph and callback admission. +#[tauri::command] +pub(crate) fn rewind_clip_start( + app: AppHandle, + state: State<'_, RewindClipState>, +) -> Result { + let started = std::time::Instant::now(); + let mut active_guard = state.0.lock().map_err(|error| error.to_string())?; + let active = active_guard + .as_mut() + .ok_or_else(|| "no prepared Rewind-derived clip is available".to_string())?; + if active.activated { + return Err("a Rewind-derived clip is already active".into()); + } + let boundary = std::time::Instant::now(); + let lease = app + .state::() + .0 + .lock() + .map_err(|error| error.to_string())? + .start_consumer_at( + CaptureConsumer::Clip, + active.sources.iter().copied(), + boundary, + ) + .map_err(|error| error.to_string())?; + #[cfg(target_os = "macos")] + if let Err(error) = active + .shared_sink + .as_mut() + .ok_or_else(|| "prepared Rewind Clip sink is missing".to_string())? + .activate() + { + end_clip_graph_lease(&app, &lease.id); + return Err(error); + } + active.lease_id = Some(lease.id); + active.started_elapsed_ms = lease.interval.started_elapsed_ms; + active.activated = true; + crate::logfile::diagnostic(&format!( + "[rewind-latency] start boundary accepted in {}ms", + started.elapsed().as_millis() + )); + Ok(RewindClipStatus { + compatibility: RewindClipCompatibility::Compatible, + active: true, + retrospective_seconds: 0, + paused: false, + sources: active.sources.clone(), + }) +} + +fn release_temporary_audio(app: &AppHandle, lease: Option) { + if let Some(lease) = lease { + if let Err(error) = screen_memory::release_temporary_audio_consumer(app, lease) { + eprintln!("[clips-tray] Rewind Clip audio lease release failed: {error}"); + } + } +} + +fn end_clip_graph_lease(app: &AppHandle, lease_id: &str) { + if let Ok(mut graph) = app.state::().0.lock() { + let _ = graph.end_consumer(lease_id); + } +} + +#[tauri::command] +pub(crate) fn rewind_clip_extend( + app: AppHandle, + state: State<'_, RewindClipState>, + seconds: u64, +) -> Result { + if !matches!(seconds, RETROSPECTIVE_30_SECONDS | RETROSPECTIVE_5_MINUTES) { + return Err("retrospective extension must be exactly 30 or 300 seconds".into()); + } + let lease_id = state + .0 + .lock() + .map_err(|error| error.to_string())? + .as_ref() + .and_then(|active| active.lease_id.clone()) + .ok_or_else(|| "no Rewind-derived clip is active".to_string())?; + let lease = app + .state::() + .0 + .lock() + .map_err(|error| error.to_string())? + .extend_retrospectively(&lease_id, std::time::Duration::from_secs(seconds)) + .map_err(|error| error.to_string())?; + { + let mut active = state.0.lock().map_err(|error| error.to_string())?; + let active = active + .as_mut() + .ok_or_else(|| "no Rewind-derived clip is active".to_string())?; + active.started_elapsed_ms = lease.interval.started_elapsed_ms; + active.retrospective_seconds = seconds; + for segment in screen_memory::finalized_segments_in_graph_interval( + &app, + lease.interval.started_elapsed_ms, + lease.interval.ended_elapsed_ms, + )? { + screen_memory::pin_segment(&app, &segment.id, &active.pin_id)?; + active.pinned_segments.insert(segment.id); + } + } + Ok(RewindClipStatus { + compatibility: RewindClipCompatibility::Compatible, + active: true, + retrospective_seconds: lease.retrospective_extension_ms / 1000, + paused: false, + sources: state + .0 + .lock() + .map_err(|error| error.to_string())? + .as_ref() + .map(|active| active.sources.clone()) + .unwrap_or_default(), + }) +} + +#[tauri::command] +pub(crate) fn rewind_clip_pause( + app: AppHandle, + state: State<'_, RewindClipState>, +) -> Result { + let mut active_guard = state.0.lock().map_err(|error| error.to_string())?; + let active = active_guard + .as_mut() + .ok_or_else(|| "no Rewind-derived clip is active".to_string())?; + if !active.activated || active.paused { + return Err("Rewind-derived clip is already paused".into()); + } + let lease_id = active + .lease_id + .clone() + .ok_or_else(|| "active Clip lease is missing".to_string())?; + let sources = active.sources.clone(); + let retrospective_seconds = active.retrospective_seconds; + #[cfg(target_os = "macos")] + let sink = active + .shared_sink + .as_ref() + .ok_or_else(|| "active Rewind Clip sink is missing".to_string())?; + #[cfg(target_os = "macos")] + sink.pause()?; + + let close_result = app + .state::() + .0 + .lock() + .map_err(|error| error.to_string()) + .and_then(|mut graph| { + graph + .end_consumer(&lease_id) + .map_err(|error| error.to_string()) + }); + let closed = match close_result { + Ok(closed) => closed, + Err(error) => { + #[cfg(target_os = "macos")] + if let Err(rollback_error) = sink.resume() { + return Err(format!( + "{error}. Clip pause rollback also failed: {rollback_error}" + )); + } + return Err(error); + } + }; + let interval = closed.lease.interval; + active.intervals.push(interval); + active.lease_id = None; + active.paused = true; + Ok(RewindClipStatus { + compatibility: RewindClipCompatibility::Compatible, + active: true, + retrospective_seconds, + paused: true, + sources, + }) +} + +#[tauri::command] +pub(crate) fn rewind_clip_resume( + app: AppHandle, + state: State<'_, RewindClipState>, +) -> Result { + let sources = { + let active = state.0.lock().map_err(|e| e.to_string())?; + let active = active + .as_ref() + .ok_or_else(|| "no Rewind-derived clip is active".to_string())?; + if !active.paused { + return Err("Rewind-derived clip is not paused".into()); + } + active.sources.clone() + }; + let lease = match app + .state::() + .0 + .lock() + .map_err(|e| e.to_string())? + .start_consumer(CaptureConsumer::Clip, sources.iter().copied()) + { + Ok(lease) => lease, + Err(error) => return Err(error.to_string()), + }; + let mut active = state.0.lock().map_err(|e| e.to_string())?; + let active = active + .as_mut() + .ok_or_else(|| "no Rewind-derived clip is active".to_string())?; + #[cfg(target_os = "macos")] + if let Err(error) = active + .shared_sink + .as_ref() + .ok_or_else(|| "active Rewind Clip sink is missing".to_string())? + .resume() + { + end_clip_graph_lease(&app, &lease.id); + return Err(error); + } + active.lease_id = Some(lease.id); + active.started_elapsed_ms = lease.interval.started_elapsed_ms; + active.sources = sources; + active.paused = false; + Ok(RewindClipStatus { + compatibility: RewindClipCompatibility::Compatible, + active: true, + retrospective_seconds: active.retrospective_seconds, + paused: false, + sources: active.sources.clone(), + }) +} + +fn take_active(state: &State<'_, RewindClipState>) -> Result { + state + .0 + .lock() + .map_err(|error| error.to_string())? + .take() + .ok_or_else(|| "no Rewind-derived clip is active".to_string()) +} + +fn close_direct_clip_interval( + app: &AppHandle, + active: &mut ActiveRewindClip, +) -> Result<(), String> { + if !active.activated { + return Err("prepared Rewind-derived clip never reached countdown zero".into()); + } + #[cfg(target_os = "macos")] + active + .shared_sink + .as_ref() + .ok_or_else(|| "active Rewind Clip sink is missing".to_string())? + .deactivate(); + if let Some(lease_id) = active.lease_id.take() { + let closed = app + .state::() + .0 + .lock() + .map_err(|error| error.to_string())? + .end_consumer(&lease_id) + .map_err(|error| error.to_string())?; + active.intervals.push(closed.lease.interval); + } + active.paused = false; + Ok(()) +} + +fn release_active_resources(app: &AppHandle, active: &mut ActiveRewindClip) { + for segment_id in std::mem::take(&mut active.pinned_segments) { + let _ = screen_memory::unpin_segment(app, &segment_id, &active.pin_id); + } + release_temporary_audio(app, active.temporary_audio.take()); +} + +fn artifact_path(app: &AppHandle, label: &str) -> Result { + let dir = app + .path() + .app_local_data_dir() + .map_err(|error| format!("local data directory unavailable: {error}"))? + .join("pending-recordings"); + std::fs::create_dir_all(&dir) + .map_err(|error| format!("pending recording directory unavailable: {error}"))?; + Ok(dir.join(format!( + "rewind-{label}-{}.mp4", + chrono::Utc::now().timestamp_millis() + ))) +} + +fn validate_and_plan( + segments: &[ScreenMemorySegmentMetadata], + start_ms: u64, + end_ms: u64, +) -> Result, String> { + const MAX_SEGMENT_BOUNDARY_QUANTIZATION_MS: u64 = 2; + let mut segments = segments.to_vec(); + segments.sort_by_key(|segment| segment.graph_started_elapsed_ms); + let mut cursor = start_ms; + let mut slices = Vec::new(); + for segment in segments { + if segment.corrupt || segment.error.is_some() || segment.exclusion_tainted { + return Err("selected Rewind media is corrupt, tainted, or incomplete".into()); + } + let overlap_start = start_ms.max(segment.graph_started_elapsed_ms); + let overlap_end = end_ms.min(segment.graph_ended_elapsed_ms); + if overlap_start >= overlap_end { + continue; + } + if overlap_start.saturating_sub(cursor) > MAX_SEGMENT_BOUNDARY_QUANTIZATION_MS { + return Err("selected Rewind interval contains a coverage gap".into()); + } + slices.push(NativeMediaSlice { + path: segment.path, + system_audio_path: segment.system_audio_path, + microphone_path: segment.microphone_path, + start_ms: overlap_start - segment.graph_started_elapsed_ms, + end_ms: overlap_end - segment.graph_started_elapsed_ms, + }); + cursor = cursor.max(overlap_end); + } + if end_ms.saturating_sub(cursor) > MAX_SEGMENT_BOUNDARY_QUANTIZATION_MS || slices.is_empty() { + return Err("selected Rewind interval is not fully covered".into()); + } + Ok(slices) +} + +fn validate_and_plan_wall_clock( + segments: &[ScreenMemorySegmentMetadata], + started: DateTime, + ended: DateTime, +) -> Result<(Vec, Vec), String> { + const MAX_SEGMENT_BOUNDARY_QUANTIZATION_MS: i64 = 2; + let mut timed = segments + .iter() + .filter_map(|segment| { + let segment_started = DateTime::parse_from_rfc3339(&segment.started_at) + .ok()? + .with_timezone(&Utc); + let segment_ended = DateTime::parse_from_rfc3339(&segment.ended_at) + .ok()? + .with_timezone(&Utc); + Some((segment_started, segment_ended, segment.clone())) + }) + .collect::>(); + timed.sort_by_key(|(segment_started, _, _)| *segment_started); + + let mut cursor = started; + let mut slices = Vec::new(); + let mut selected = Vec::new(); + let mut previous: Option<(Option, u64, DateTime)> = None; + for (wall_started, wall_ended, segment) in timed { + let graph_duration_ms = segment + .graph_ended_elapsed_ms + .checked_sub(segment.graph_started_elapsed_ms) + .filter(|duration| *duration > 0); + let duration_ms = graph_duration_ms.unwrap_or_else(|| { + wall_ended + .signed_duration_since(wall_started) + .num_milliseconds() + .max(0) as u64 + }); + let normalized_started = previous + .as_ref() + .and_then( + |(previous_epoch, previous_graph_end, previous_normalized_end)| { + let same_epoch = segment.graph_epoch_id.is_some() + && segment.graph_epoch_id == *previous_epoch; + let graph_skew = segment + .graph_started_elapsed_ms + .abs_diff(*previous_graph_end); + (same_epoch && graph_skew <= 2).then_some(*previous_normalized_end) + }, + ) + .unwrap_or(wall_started); + let normalized_ended = normalized_started + + chrono::Duration::milliseconds(i64::try_from(duration_ms).unwrap_or(i64::MAX)); + previous = Some(( + segment.graph_epoch_id.clone(), + segment.graph_ended_elapsed_ms, + normalized_ended, + )); + + let overlap_start = started.max(normalized_started).max(cursor); + let overlap_end = ended.min(normalized_ended); + if overlap_start >= overlap_end { + continue; + } + if segment.corrupt || segment.error.is_some() || segment.exclusion_tainted { + return Err("selected Rewind media is corrupt, tainted, or incomplete".into()); + } + if overlap_start + .signed_duration_since(cursor) + .num_milliseconds() + > MAX_SEGMENT_BOUNDARY_QUANTIZATION_MS + { + return Err("selected Rewind interval contains a coverage gap".into()); + } + let start_ms = overlap_start + .signed_duration_since(normalized_started) + .num_milliseconds() + .max(0) as u64; + let end_ms = overlap_end + .signed_duration_since(normalized_started) + .num_milliseconds() + .max(0) as u64; + slices.push(NativeMediaSlice { + path: segment.path.clone(), + system_audio_path: segment.system_audio_path.clone(), + microphone_path: segment.microphone_path.clone(), + start_ms, + end_ms, + }); + selected.push(segment); + cursor = cursor.max(overlap_end); + } + if ended.signed_duration_since(cursor).num_milliseconds() > MAX_SEGMENT_BOUNDARY_QUANTIZATION_MS + || slices.is_empty() + { + return Err("selected Rewind interval is not fully covered".into()); + } + Ok((slices, selected)) +} + +fn select_audio( + sources: &[CaptureSource], + include_mic: bool, + include_system_audio: bool, +) -> Result { + if (include_mic && !sources.contains(&CaptureSource::Microphone)) + || (include_system_audio && !sources.contains(&CaptureSource::SystemAudio)) + { + return Err(not_compatible("requested audio was not captured by Rewind")); + } + Ok(match (include_mic, include_system_audio) { + (false, false) => NativeAudioSelection::None, + (false, true) => NativeAudioSelection::System, + (true, false) => NativeAudioSelection::Microphone, + (true, true) => NativeAudioSelection::MixBoth, + }) +} + +/// Materialize an explicit recent range as one exact local MP4. This powers +/// “Save what just happened” without copying whole five-minute container +/// segments (which could otherwise retain media from before the chosen range). +pub(crate) fn materialize_recent_exact( + app: &AppHandle, + duration: std::time::Duration, + output: PathBuf, +) -> Result { + let duration_ms = u64::try_from(duration.as_millis()) + .map_err(|_| "requested Rewind export is too long".to_string())?; + if duration_ms == 0 || duration > crate::capture_graph::MAX_RETROSPECTIVE_EXTENSION { + return Err("Rewind export must be between one millisecond and five minutes".into()); + } + let fenced = screen_memory::fence_active_for_clip(app)?; + let end_ms = fenced.graph_ended_elapsed_ms; + let start_ms = end_ms + .checked_sub(duration_ms) + .ok_or_else(|| "Rewind does not yet contain the full requested range".to_string())?; + materialize_graph_interval_exact( + app, + start_ms, + end_ms, + fenced.started_at, + fenced.ended_at, + output, + "explicit recent Rewind export", + None, + ) +} + +fn materialize_wall_clock_exact( + app: &AppHandle, + started_at: &str, + ended_at: &str, + output: PathBuf, + include_mic: bool, + include_system_audio: bool, +) -> Result { + let started = DateTime::parse_from_rfc3339(started_at) + .map_err(|_| "Rewind handoff start must be an RFC3339 timestamp.".to_string())? + .with_timezone(&Utc); + let ended = DateTime::parse_from_rfc3339(ended_at) + .map_err(|_| "Rewind handoff end must be an RFC3339 timestamp.".to_string())? + .with_timezone(&Utc); + let duration = ended.signed_duration_since(started); + if duration.num_milliseconds() < 1 + || duration.num_milliseconds() + > i64::try_from(crate::capture_graph::MAX_RETROSPECTIVE_EXTENSION.as_millis()) + .unwrap_or(i64::MAX) + { + return Err("Rewind handoff must be between one millisecond and five minutes.".into()); + } + let materialize = || { + materialize_retained_wall_clock_exact( + app, + started, + ended, + output.clone(), + include_mic, + include_system_audio, + ) + }; + match materialize() { + Ok(artifact) => Ok(artifact), + Err(initial_error) => { + // A request ending near “now” may overlap the still-open segment. + // Fence once and retry. Older retained ranges never depend on the + // current capture graph, so they survive app restarts and pauses. + if Utc::now().signed_duration_since(ended).num_minutes() <= 6 + && screen_memory::fence_active_for_clip(app).is_ok() + { + materialize() + } else { + Err(initial_error) + } + } + } +} + +fn materialize_retained_wall_clock_exact( + app: &AppHandle, + started: DateTime, + ended: DateTime, + output: PathBuf, + include_mic: bool, + include_system_audio: bool, +) -> Result { + let segments = screen_memory::finalized_segments(app)?; + let (slices, selected) = validate_and_plan_wall_clock(&segments, started, ended)?; + let pin_id = format!("rewind-wall-clock-{}", Utc::now().timestamp_micros()); + let mut pinned_segments = Vec::new(); + let outcome = (|| { + for segment in &selected { + screen_memory::pin_segment(app, &segment.id, &pin_id)?; + pinned_segments.push(segment.id.clone()); + } + let audio_available = selected + .iter() + .all(|segment| segment.capture_mode == crate::config::RewindCaptureMode::VisualsAudio); + if (include_mic || include_system_audio) && !audio_available { + return Err("The selected Rewind range does not contain the requested audio.".into()); + } + let audio_selection = match (include_mic, include_system_audio) { + (false, false) => NativeAudioSelection::None, + (true, false) => NativeAudioSelection::Microphone, + (false, true) => NativeAudioSelection::System, + (true, true) => NativeAudioSelection::MixBoth, + }; + native_screen::materialize_mp4_slices_exact(&slices, &output, audio_selection)?; + let first = selected + .first() + .ok_or_else(|| "no Rewind media selected".to_string())?; + Ok(FinalizedNativeArtifact::rewind_mp4( + output, + ended + .signed_duration_since(started) + .num_milliseconds() + .max(0) as u128, + first.width, + first.height, + include_mic, + include_system_audio, + )) + })(); + for segment_id in pinned_segments { + let _ = screen_memory::unpin_segment(app, &segment_id, &pin_id); + } + outcome +} + +fn materialize_graph_interval_exact( + app: &AppHandle, + start_ms: u64, + end_ms: u64, + started_at: String, + ended_at: String, + output: PathBuf, + label: &str, + requested_audio: Option<(bool, bool)>, +) -> Result { + if end_ms <= start_ms { + return Err("The selected Rewind handoff range is empty.".into()); + } + let interval = CaptureInterval { + started_at, + ended_at, + started_elapsed_ms: start_ms, + ended_elapsed_ms: end_ms, + }; + let graph_state = app.state::(); + let (pin_id, has_screen_gap) = { + let mut graph = graph_state.0.lock().map_err(|error| error.to_string())?; + let status = graph + .status_at(std::time::Instant::now()) + .map_err(|error| error.to_string())?; + let has_gap = status.coverage_gaps.iter().any(|gap| { + gap.source == CaptureSource::Screen + && gap.interval.started_elapsed_ms < end_ms + && gap.interval.ended_elapsed_ms > start_ms + }); + let pin = graph + .pin_interval(&interval, Some(label.into())) + .map_err(|error| error.to_string())?; + (pin.id, has_gap) + }; + let mut pinned_segments = Vec::new(); + let outcome = (|| { + if has_screen_gap { + return Err("selected Rewind interval contains a recorded coverage gap".into()); + } + let segments = screen_memory::finalized_segments_in_graph_interval(app, start_ms, end_ms)?; + for segment in &segments { + screen_memory::pin_segment(app, &segment.id, &pin_id)?; + pinned_segments.push(segment.id.clone()); + } + let slices = validate_and_plan(&segments, start_ms, end_ms)?; + let audio_available = segments + .iter() + .all(|segment| segment.capture_mode == crate::config::RewindCaptureMode::VisualsAudio); + let (include_mic, include_system_audio) = + requested_audio.unwrap_or((audio_available, audio_available)); + if (include_mic || include_system_audio) && !audio_available { + return Err("The selected Rewind range does not contain the requested audio.".into()); + } + let audio_selection = match (include_mic, include_system_audio) { + (false, false) => NativeAudioSelection::None, + (true, false) => NativeAudioSelection::Microphone, + (false, true) => NativeAudioSelection::System, + (true, true) => NativeAudioSelection::MixBoth, + }; + native_screen::materialize_mp4_slices_exact(&slices, &output, audio_selection)?; + let first = segments + .first() + .ok_or_else(|| "no Rewind media selected".to_string())?; + Ok(FinalizedNativeArtifact::rewind_mp4( + output, + u128::from(end_ms.saturating_sub(start_ms)), + first.width, + first.height, + include_mic, + include_system_audio, + )) + })(); + for segment_id in pinned_segments { + let _ = screen_memory::unpin_segment(app, &segment_id, &pin_id); + } + if let Ok(mut graph) = graph_state.0.lock() { + graph.release_pin(&pin_id); + } + outcome +} + +#[tauri::command] +pub(crate) async fn rewind_agent_handoff_upload( + app: AppHandle, + request_id: String, + started_at: String, + ended_at: String, + server_url: String, + recording_id: String, + auth_token: Option, + cookie: Option, + upload_mode: Option, + include_mic: bool, + include_system_audio: bool, +) -> Result { + let safe_request_id: String = request_id + .chars() + .filter(|character| { + character.is_ascii_alphanumeric() || *character == '-' || *character == '_' + }) + .collect(); + if safe_request_id != request_id || !safe_request_id.starts_with("handoff-") { + return Err("Invalid Rewind handoff request ID.".into()); + } + let output_dir = app + .path() + .app_data_dir() + .map_err(|error| format!("app data directory unavailable: {error}"))? + .join("screen-memory") + .join("agent-handoffs"); + std::fs::create_dir_all(&output_dir) + .map_err(|error| format!("agent handoff directory unavailable: {error}"))?; + let output = output_dir.join(format!("{safe_request_id}.mp4")); + let artifact = materialize_wall_clock_exact( + &app, + &started_at, + &ended_at, + output.clone(), + include_mic, + include_system_audio, + )?; + let result = native_screen::upload_finalized_native_artifact( + &app, + &artifact, + server_url, + recording_id, + auth_token.unwrap_or_default(), + cookie.unwrap_or_default(), + NativeUploadMode::from_option(upload_mode), + artifact.mic_captured || artifact.system_audio_captured, + false, + ) + .await; + let _ = std::fs::remove_file(output); + result +} + +#[tauri::command] +pub(crate) async fn rewind_agent_handoff_preview( + app: AppHandle, + request_id: String, + started_at: String, + ended_at: String, + include_mic: bool, + include_system_audio: bool, +) -> Result { + let safe_request_id: String = request_id + .chars() + .filter(|character| { + character.is_ascii_alphanumeric() || *character == '-' || *character == '_' + }) + .collect(); + if safe_request_id != request_id || !safe_request_id.starts_with("handoff-") { + return Err("Invalid Rewind handoff request ID.".into()); + } + let output_dir = app + .path() + .app_local_data_dir() + .map_err(|error| format!("local preview directory unavailable: {error}"))? + .join("rewind-previews"); + cleanup_expired_preview_artifacts(&output_dir, std::time::Duration::from_secs(15 * 60))?; + std::fs::create_dir_all(&output_dir) + .map_err(|error| format!("local preview directory unavailable: {error}"))?; + let output = output_dir.join(format!("{safe_request_id}-preview.mp4")); + let preview = output.clone(); + tauri::async_runtime::spawn_blocking(move || { + materialize_wall_clock_exact( + &app, + &started_at, + &ended_at, + preview, + include_mic, + include_system_audio, + ) + }) + .await + .map_err(|error| format!("local handoff preview worker failed: {error}"))??; + + #[cfg(target_os = "macos")] + Command::new("/usr/bin/open") + .arg(&output) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| format!("could not open Rewind handoff preview: {error}"))?; + + let expiring_output = output.clone(); + std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_secs(15 * 60)); + let _ = std::fs::remove_file(expiring_output); + }); + + Ok(output.to_string_lossy().to_string()) +} + +fn cleanup_expired_preview_artifacts( + directory: &std::path::Path, + max_age: std::time::Duration, +) -> Result<(), String> { + let now = std::time::SystemTime::now(); + let Ok(entries) = std::fs::read_dir(directory) else { + return Ok(()); + }; + for entry in entries.flatten() { + let path = entry.path(); + let expired = entry + .metadata() + .ok() + .and_then(|metadata| metadata.modified().ok()) + .and_then(|modified| now.duration_since(modified).ok()) + .is_some_and(|age| age >= max_age); + if path.is_file() && expired { + std::fs::remove_file(&path) + .map_err(|error| format!("could not remove expired preview: {error}"))?; + } + } + Ok(()) +} + +pub(crate) fn clear_preview_artifacts(app: &AppHandle) -> Result<(), String> { + let directory = app + .path() + .app_local_data_dir() + .map_err(|error| format!("local preview directory unavailable: {error}"))? + .join("rewind-previews"); + match std::fs::remove_dir_all(&directory) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(format!("could not clear local Rewind previews: {error}")), + } +} + +fn materialize( + app: &AppHandle, + state: &State<'_, RewindClipState>, + label: &str, + include_mic: bool, + include_system_audio: bool, +) -> Result { + let active = take_active(state)?; + let mut intervals = active.intervals.clone(); + let mut lease_ended = active.lease_id.is_none(); + let mut graph_pin_ids = Vec::new(); + let mut pinned = BTreeSet::new(); + let outcome = (|| -> Result { + if let Some(lease_id) = active.lease_id.as_deref() { + let fenced = screen_memory::fence_active_for_clip(app)?; + let closed = app + .state::() + .0 + .lock() + .map_err(|e| e.to_string())? + .end_consumer(lease_id) + .map_err(|e| e.to_string())?; + lease_ended = true; + let mut interval = closed.lease.interval; + interval.ended_elapsed_ms = + interval.ended_elapsed_ms.min(fenced.graph_ended_elapsed_ms); + intervals.push(interval); + } + if intervals.is_empty() { + return Err("Rewind-derived clip has no recorded intervals".into()); + } + let graph_status = app + .state::() + .0 + .lock() + .map_err(|error| error.to_string())? + .status_at(std::time::Instant::now()) + .map_err(|error| error.to_string())?; + if graph_status.coverage_gaps.iter().any(|gap| { + intervals.iter().any(|interval| { + gap.interval.started_elapsed_ms < interval.ended_elapsed_ms + && gap.interval.ended_elapsed_ms > interval.started_elapsed_ms + && active.sources.contains(&gap.source) + }) + }) { + return Err("selected Rewind interval contains a recorded coverage gap".into()); + } + let mut slices = Vec::new(); + let mut first_segment = None; + let mut duration_ms = 0u128; + for interval in &intervals { + let pin = app + .state::() + .0 + .lock() + .map_err(|e| e.to_string())? + .pin_interval(interval, Some("rewind-derived clip".into())) + .map_err(|e| e.to_string())?; + graph_pin_ids.push(pin.id.clone()); + let segments = screen_memory::finalized_segments_in_graph_interval( + app, + interval.started_elapsed_ms, + interval.ended_elapsed_ms, + )?; + for segment in &segments { + screen_memory::pin_segment(app, &segment.id, &pin.id)?; + pinned.insert((segment.id.clone(), pin.id.clone())); + } + if first_segment.is_none() { + first_segment = segments.first().cloned(); + } + slices.extend(validate_and_plan( + &segments, + interval.started_elapsed_ms, + interval.ended_elapsed_ms, + )?); + duration_ms += (interval.ended_elapsed_ms - interval.started_elapsed_ms) as u128; + } + let output = artifact_path(app, label)?; + let audio = select_audio(&active.sources, include_mic, include_system_audio)?; + native_screen::materialize_mp4_slices_exact(&slices, &output, audio)?; + let first = first_segment + .as_ref() + .ok_or_else(|| "no Rewind media selected".to_string())?; + Ok(FinalizedNativeArtifact::rewind_mp4( + output, + duration_ms, + first.width, + first.height, + include_mic, + include_system_audio, + )) + })(); + if !lease_ended { + if let (Some(lease_id), Ok(mut graph)) = ( + active.lease_id.as_deref(), + app.state::().0.lock(), + ) { + let _ = graph.end_consumer(lease_id); + } + } + for (segment_id, pin_id) in pinned { + let _ = screen_memory::unpin_segment(app, &segment_id, &pin_id); + } + for pin_id in graph_pin_ids { + if let Ok(mut graph) = app.state::().0.lock() { + graph.release_pin(&pin_id); + } + } + for segment_id in &active.pinned_segments { + let _ = screen_memory::unpin_segment(app, segment_id, &active.pin_id); + } + release_temporary_audio(app, active.temporary_audio); + outcome +} + +#[tauri::command] +pub(crate) async fn rewind_clip_stop_and_upload( + app: AppHandle, + state: State<'_, RewindClipState>, + server_url: String, + recording_id: String, + auth_token: Option, + cookie: Option, + upload_mode: Option, + include_mic: bool, + include_system_audio: bool, + has_camera: bool, +) -> Result { + #[cfg(target_os = "macos")] + { + let stop_started = std::time::Instant::now(); + let mut active = take_active(&state)?; + if let Err(error) = close_direct_clip_interval(&app, &mut active) { + if let Some(sink) = active.shared_sink.take() { + sink.cancel(); + } + release_active_resources(&app, &mut active); + return Err(error); + } + let uploaded_before_stop = active + .shared_sink + .as_ref() + .and_then(|sink| sink.uploaded_bytes_now()); + crate::logfile::diagnostic(&format!( + "[rewind-latency] stop boundary accepted in {}ms", + stop_started.elapsed().as_millis() + )); + let mut sink = active + .shared_sink + .take() + .ok_or_else(|| "active Rewind Clip sink is missing".to_string())?; + if active.retrospective_seconds == 0 { + let sink_path = sink.path().to_path_buf(); + let (sink_width, sink_height) = sink.dimensions(); + let logical_duration_ms = sink.duration_ms(); + let result = sink.finalize_writer(); + release_active_resources(&app, &mut active); + let mut result = match result { + Ok(result) => result, + Err(error) => { + let recovery_error = native_screen::persist_shared_clip_recording( + &app, + &sink_path, + &server_url, + &recording_id, + logical_duration_ms as u128, + sink_width, + sink_height, + include_mic, + include_system_audio, + has_camera, + Some(&error), + ) + .err(); + let error = match recovery_error { + Some(recovery_error) => { + format!("{error}. Recovery metadata also failed: {recovery_error}") + } + None => error, + }; + native_screen::emit_native_upload_finished( + &app, + &server_url, + &recording_id, + false, + Some(error.clone()), + Some(&sink_path), + ); + return Err(error); + } + }; + if let Err(error) = native_screen::persist_shared_clip_recording( + &app, + &result.path, + &server_url, + &recording_id, + result.duration_ms as u128, + result.width, + result.height, + include_mic, + include_system_audio, + has_camera, + None, + ) { + sink.cancel_upload(); + native_screen::emit_native_upload_finished( + &app, + &server_url, + &recording_id, + false, + Some(error.clone()), + Some(&result.path), + ); + return Err(error); + } + result.verification_pending = match sink.finalize_upload(result.duration_ms).await { + Ok(pending) => pending, + Err(error) => { + let _ = native_screen::persist_shared_clip_recording( + &app, + &result.path, + &server_url, + &recording_id, + result.duration_ms as u128, + result.width, + result.height, + include_mic, + include_system_audio, + has_camera, + Some(&error), + ); + native_screen::emit_native_upload_finished( + &app, + &server_url, + &recording_id, + false, + Some(error.clone()), + Some(&result.path), + ); + return Err(format!( + "{error}. The clip was saved locally and can be retried from the Clips menu." + )); + } + }; + if let Some(uploaded) = uploaded_before_stop { + let percent = if result.bytes == 0 { + 0 + } else { + uploaded.saturating_mul(100) / result.bytes + }; + crate::logfile::diagnostic(&format!( + "[rewind-latency] uploaded before Stop: {uploaded}/{} bytes ({percent}%)", + result.bytes + )); + } + native_screen::emit_native_upload_finished( + &app, + &server_url, + &recording_id, + true, + None, + Some(&result.path), + ); + if !result.verification_pending { + native_screen::clear_shared_clip_recording(&app, &recording_id, &result.path); + } + return Ok(result.into_native_upload_result(recording_id)); + } + + if let Err(error) = sink.abandon_for_rewind_materialization().await { + release_active_resources(&app, &mut active); + return Err(error); + } + *state.0.lock().map_err(|error| error.to_string())? = Some(active); + } + let artifact = materialize( + &app, + &state, + &recording_id, + include_mic, + include_system_audio, + )?; + native_screen::upload_finalized_native_artifact( + &app, + &artifact, + server_url, + recording_id, + auth_token.unwrap_or_default(), + cookie.unwrap_or_default(), + NativeUploadMode::from_option(upload_mode), + include_mic || include_system_audio, + has_camera, + ) + .await +} + +#[tauri::command] +pub(crate) async fn rewind_clip_stop_and_save( + app: AppHandle, + state: State<'_, RewindClipState>, + folder_name: String, + file_role: String, + include_mic: bool, + include_system_audio: bool, +) -> Result { + #[cfg(target_os = "macos")] + { + let stop_started = std::time::Instant::now(); + let mut active = take_active(&state)?; + if let Err(error) = close_direct_clip_interval(&app, &mut active) { + if let Some(sink) = active.shared_sink.take() { + sink.cancel(); + } + release_active_resources(&app, &mut active); + return Err(error); + } + eprintln!( + "[rewind-latency] local stop boundary accepted in {}ms", + stop_started.elapsed().as_millis() + ); + let mut sink = active + .shared_sink + .take() + .ok_or_else(|| "active Rewind Clip sink is missing".to_string())?; + if active.retrospective_seconds == 0 { + let result = sink.finalize_writer(); + release_active_resources(&app, &mut active); + let result = result?; + let artifact = FinalizedNativeArtifact::rewind_mp4( + result.path, + result.duration_ms as u128, + Some(result.width), + Some(result.height), + include_mic, + include_system_audio, + ); + let saved = native_screen::save_finalized_native_artifact_to_local_export( + &app, + &artifact, + &folder_name, + &file_role, + )?; + native_screen::clear_shared_clip_recording(&app, &folder_name, &artifact.path); + return Ok(saved); + } + if let Err(error) = sink.abandon_for_rewind_materialization().await { + release_active_resources(&app, &mut active); + return Err(error); + } + *state.0.lock().map_err(|error| error.to_string())? = Some(active); + } + let artifact = materialize( + &app, + &state, + &folder_name, + include_mic, + include_system_audio, + )?; + native_screen::save_finalized_native_artifact_to_local_export( + &app, + &artifact, + &folder_name, + &file_role, + ) +} + +#[tauri::command] +pub(crate) fn rewind_clip_cancel( + app: AppHandle, + state: State<'_, RewindClipState>, +) -> Result<(), String> { + let active = state.0.lock().map_err(|error| error.to_string())?.take(); + if let Some(mut active) = active { + if let Some(lease_id) = active.lease_id.as_deref() { + let _ = app + .state::() + .0 + .lock() + .map_err(|error| error.to_string())? + .end_consumer(lease_id); + } + #[cfg(target_os = "macos")] + if let Some(sink) = active.shared_sink.take() { + sink.cancel(); + } + release_active_resources(&app, &mut active); + } + Ok(()) +} + +/// Called by Screen Memory whenever a segment becomes finalized. Keeping this +/// hook in the consumer module prevents rotation/pruning from racing a clip +/// whose materialization has not happened yet. +pub(crate) fn pin_finalized_segment_if_active( + app: &AppHandle, + segment: &ScreenMemorySegmentMetadata, +) -> Result<(), String> { + let Some(state) = app.try_state::() else { + return Ok(()); + }; + let Ok(mut active) = state.0.try_lock() else { + return Ok(()); + }; + let Some(active) = active.as_mut() else { + return Ok(()); + }; + if !active.activated || active.paused { + return Ok(()); + } + if segment.graph_ended_elapsed_ms >= active.started_elapsed_ms { + screen_memory::pin_segment(app, &segment.id, &active.pin_id)?; + active.pinned_segments.insert(segment.id.clone()); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::RewindCaptureMode; + + fn segment(id: &str, start: u64, end: u64) -> ScreenMemorySegmentMetadata { + ScreenMemorySegmentMetadata { + id: id.into(), + path: PathBuf::from(format!("{id}.mp4")), + file_name: format!("{id}.mp4"), + mime_type: "video/mp4".into(), + started_at: String::new(), + ended_at: String::new(), + duration_ms: (end - start) as u128, + width: Some(1280), + height: Some(720), + bytes: 1, + system_audio_path: None, + microphone_path: None, + corrupt: false, + error: None, + capture_mode: RewindCaptureMode::VisualsAudio, + exclusion_tainted: false, + graph_epoch_id: Some("test-epoch".into()), + graph_started_elapsed_ms: start, + graph_ended_elapsed_ms: end, + } + } + + fn wall_segment( + id: &str, + started_at: &str, + ended_at: &str, + epoch: &str, + ) -> ScreenMemorySegmentMetadata { + let mut value = segment(id, 0, 100); + value.started_at = started_at.into(); + value.ended_at = ended_at.into(); + value.graph_epoch_id = Some(epoch.into()); + value + } + + #[test] + fn slice_math_has_no_pre_roll_and_exact_overlap() { + let slices = + validate_and_plan(&[segment("a", 0, 100), segment("b", 100, 200)], 50, 150).unwrap(); + assert_eq!( + slices, + vec![ + NativeMediaSlice { + path: PathBuf::from("a.mp4"), + system_audio_path: None, + microphone_path: None, + start_ms: 50, + end_ms: 100 + }, + NativeMediaSlice { + path: PathBuf::from("b.mp4"), + system_audio_path: None, + microphone_path: None, + start_ms: 0, + end_ms: 50 + } + ] + ); + } + + #[test] + fn gaps_and_taint_fail_closed() { + assert!(validate_and_plan(&[segment("a", 0, 100), segment("b", 101, 200)], 0, 200).is_ok()); + assert!( + validate_and_plan(&[segment("a", 0, 100), segment("b", 103, 200)], 0, 200).is_err() + ); + assert!(validate_and_plan(&[segment("a", 0, 90), segment("b", 100, 200)], 0, 200).is_err()); + let mut tainted = segment("a", 0, 100); + tainted.exclusion_tainted = true; + assert!(validate_and_plan(&[tainted], 0, 100).is_err()); + } + + #[test] + fn retrospective_choices_are_strictly_bounded() { + assert!(matches!(30, RETROSPECTIVE_30_SECONDS)); + assert!(matches!(300, RETROSPECTIVE_5_MINUTES)); + } + + #[test] + fn visuals_reject_audio_and_audio_mode_preserves_each_choice() { + assert!(select_audio(&[CaptureSource::Screen], true, false).is_err()); + let sources = [ + CaptureSource::Screen, + CaptureSource::SystemAudio, + CaptureSource::Microphone, + ]; + assert_eq!( + select_audio(&sources, false, false).unwrap(), + NativeAudioSelection::None + ); + assert_eq!( + select_audio(&sources, false, true).unwrap(), + NativeAudioSelection::System + ); + assert_eq!( + select_audio(&sources, true, false).unwrap(), + NativeAudioSelection::Microphone + ); + assert_eq!( + select_audio(&sources, true, true).unwrap(), + NativeAudioSelection::MixBoth + ); + } + + #[test] + fn ordered_intervals_exclude_paused_media() { + let segments = [ + segment("before", 0, 100), + segment("paused", 100, 200), + segment("after", 200, 300), + ]; + let mut slices = validate_and_plan(&segments, 0, 100).unwrap(); + slices.extend(validate_and_plan(&segments, 200, 300).unwrap()); + assert_eq!(slices.len(), 2); + assert_eq!(slices[0].path, PathBuf::from("before.mp4")); + assert_eq!(slices[1].path, PathBuf::from("after.mp4")); + } + + #[test] + fn wall_clock_planning_crosses_capture_graph_epochs() { + let started = DateTime::parse_from_rfc3339("2026-07-19T10:00:00Z") + .unwrap() + .with_timezone(&Utc); + let ended = DateTime::parse_from_rfc3339("2026-07-19T10:00:00.200Z") + .unwrap() + .with_timezone(&Utc); + let segments = [ + wall_segment( + "before-restart", + "2026-07-19T10:00:00Z", + "2026-07-19T10:00:00.100Z", + "epoch-before-restart", + ), + wall_segment( + "after-restart", + "2026-07-19T10:00:00.100Z", + "2026-07-19T10:00:00.200Z", + "epoch-after-restart", + ), + ]; + let (slices, selected) = validate_and_plan_wall_clock(&segments, started, ended).unwrap(); + assert_eq!(slices.len(), 2); + assert_eq!(selected[0].id, "before-restart"); + assert_eq!(selected[1].id, "after-restart"); + } + + #[test] + fn wall_clock_planning_uses_graph_continuity_across_rotation_skew() { + let started = DateTime::parse_from_rfc3339("2026-07-19T10:00:00Z") + .unwrap() + .with_timezone(&Utc); + let ended = DateTime::parse_from_rfc3339("2026-07-19T10:00:00.200Z") + .unwrap() + .with_timezone(&Utc); + let first = wall_segment( + "before-rotation", + "2026-07-19T10:00:00Z", + "2026-07-19T10:00:00.100Z", + "same-epoch", + ); + let mut second = wall_segment( + "after-rotation", + "2026-07-19T10:00:00.150Z", + "2026-07-19T10:00:00.250Z", + "same-epoch", + ); + second.graph_started_elapsed_ms = 100; + second.graph_ended_elapsed_ms = 200; + let (slices, selected) = + validate_and_plan_wall_clock(&[first, second], started, ended).unwrap(); + assert_eq!(slices.len(), 2); + assert_eq!(selected[1].id, "after-rotation"); + assert_eq!(slices[1].start_ms, 0); + assert_eq!(slices[1].end_ms, 100); + } + + #[test] + fn wall_clock_planning_rejects_real_gaps() { + let started = DateTime::parse_from_rfc3339("2026-07-19T10:00:00Z") + .unwrap() + .with_timezone(&Utc); + let ended = DateTime::parse_from_rfc3339("2026-07-19T10:00:00.200Z") + .unwrap() + .with_timezone(&Utc); + let mut first = wall_segment( + "a", + "2026-07-19T10:00:00Z", + "2026-07-19T10:00:00.090Z", + "one", + ); + first.graph_ended_elapsed_ms = 90; + let segments = [ + first, + wall_segment( + "b", + "2026-07-19T10:00:00.100Z", + "2026-07-19T10:00:00.200Z", + "two", + ), + ]; + assert!(validate_and_plan_wall_clock(&segments, started, ended).is_err()); + } + + #[test] + fn expired_preview_cleanup_removes_files() { + let directory = std::env::temp_dir().join(format!( + "clips-rewind-preview-test-{}", + Utc::now().timestamp_micros() + )); + std::fs::create_dir_all(&directory).unwrap(); + let preview = directory.join("preview.mp4"); + std::fs::write(&preview, b"preview").unwrap(); + cleanup_expired_preview_artifacts(&directory, std::time::Duration::ZERO).unwrap(); + assert!(!preview.exists()); + let _ = std::fs::remove_dir_all(directory); + } +} diff --git a/templates/clips/desktop/src-tauri/src/rewind_egress.rs b/templates/clips/desktop/src-tauri/src/rewind_egress.rs new file mode 100644 index 0000000000..81e76b9d05 --- /dev/null +++ b/templates/clips/desktop/src-tauri/src/rewind_egress.rs @@ -0,0 +1,643 @@ +//! Local validation and activity trail for bounded Rewind evidence. +//! +//! This module never sends a network request. It only validates a bounded text +//! packet and records intent before a future caller is allowed to send it. + +use chrono::Utc; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::fs::{File, OpenOptions}; +use std::io::{BufRead, BufReader, Write}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; +use tauri::{AppHandle, Manager}; + +const EGRESS_LOG_NAME: &str = "egress.jsonl"; +const MAX_QUESTION_CHARS: usize = 4_000; +const MAX_EVIDENCE_ITEMS: usize = 20; +const MAX_EXCERPT_CHARS: usize = 1_200; +const MAX_ID_CHARS: usize = 200; +const MAX_TIMESTAMP_CHARS: usize = 80; +const MAX_ERROR_CHARS: usize = 1_000; +const MAX_LIST_LIMIT: usize = 500; + +static REQUEST_COUNTER: AtomicU64 = AtomicU64::new(1); +static EGRESS_IO_LOCK: Mutex<()> = Mutex::new(()); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RewindEvidenceSource { + AppContext, + Transcript, + Ocr, + Chapter, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RewindEvidenceForEgress { + pub id: String, + pub moment_id: String, + pub source_type: RewindEvidenceSource, + pub captured_at: Option, + pub excerpt: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RewindEvidencePacket { + pub question: String, + pub evidence: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RewindEgressState { + Prepared, + Completed, + Failed, + LocalEvidenceRead, + HandoffRequested, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RewindEgressEvent { + pub request_id: String, + pub occurred_at: String, + pub state: RewindEgressState, + #[serde(default)] + pub packet: Option, + #[serde(default)] + pub operation: Option, + #[serde(default)] + pub receipt: Option, + pub evidence_count: usize, + #[serde(default)] + pub packet_bytes: usize, + pub error: Option, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RewindEgressPrepared { + pub request_id: String, + pub packet: RewindEvidencePacket, + pub evidence_count: usize, + pub packet_bytes: usize, +} + +struct EgressStore { + path: PathBuf, +} + +impl EgressStore { + fn new(path: PathBuf) -> Self { + Self { path } + } + + fn append(&self, event: &RewindEgressEvent) -> Result<(), String> { + let _guard = EGRESS_IO_LOCK + .lock() + .map_err(|err| format!("Rewind egress log lock failed: {err}"))?; + self.append_unlocked(event) + } + + fn append_unlocked(&self, event: &RewindEgressEvent) -> Result<(), String> { + if let Some(parent) = self.path.parent() { + std::fs::create_dir_all(parent) + .map_err(|err| format!("Rewind egress log directory unavailable: {err}"))?; + } + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(&self.path) + .map_err(|err| format!("Rewind egress log unavailable: {err}"))?; + serde_json::to_writer(&mut file, event) + .map_err(|err| format!("Rewind egress event encoding failed: {err}"))?; + file.write_all(b"\n") + .and_then(|_| file.flush()) + .map_err(|err| format!("Rewind egress event write failed: {err}")) + } + + fn scrub_legacy_content(&self) -> Result<(), String> { + let _guard = EGRESS_IO_LOCK + .lock() + .map_err(|err| format!("Rewind egress log lock failed: {err}"))?; + if !self.path.exists() { + return Ok(()); + } + let file = File::open(&self.path) + .map_err(|err| format!("Rewind egress log unavailable: {err}"))?; + let mut changed = false; + let events = BufReader::new(file) + .lines() + .map_while(Result::ok) + .filter_map(|line| { + let mut event: Value = serde_json::from_str(&line).ok()?; + let object = event.as_object_mut()?; + if let Some(packet) = object.remove("packet") { + changed = true; + if !object.contains_key("receipt") { + let evidence = packet + .get("evidence") + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .map(|item| { + json!({ + "id": item.get("id"), + "momentId": item.get("momentId"), + "sourceType": item.get("sourceType"), + "capturedAt": item.get("capturedAt").unwrap_or(&Value::Null), + }) + }) + .collect::>() + }) + .unwrap_or_default(); + object.insert("receipt".to_string(), json!({ "evidence": evidence })); + } + } + if object.remove("reason").is_some() { + changed = true; + } + object + .entry("packetBytes".to_string()) + .or_insert(json!(0)); + Some(event) + }) + .collect::>(); + if !changed { + return Ok(()); + } + let temporary_path = self + .path + .with_extension(format!("jsonl.sanitize-{}", std::process::id())); + let mut temporary = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&temporary_path) + .map_err(|err| format!("Rewind egress migration unavailable: {err}"))?; + for event in events { + serde_json::to_writer(&mut temporary, &event) + .map_err(|err| format!("Rewind egress migration encoding failed: {err}"))?; + temporary + .write_all(b"\n") + .map_err(|err| format!("Rewind egress migration write failed: {err}"))?; + } + temporary + .flush() + .map_err(|err| format!("Rewind egress migration write failed: {err}"))?; + std::fs::rename(&temporary_path, &self.path) + .map_err(|err| format!("Rewind egress migration failed: {err}")) + } + + fn read_all(&self) -> Result, String> { + let _guard = EGRESS_IO_LOCK + .lock() + .map_err(|err| format!("Rewind egress log lock failed: {err}"))?; + self.read_all_unlocked() + } + + fn read_all_unlocked(&self) -> Result, String> { + if !self.path.exists() { + return Ok(Vec::new()); + } + let file = File::open(&self.path) + .map_err(|err| format!("Rewind egress log unavailable: {err}"))?; + Ok(BufReader::new(file) + .lines() + .map_while(Result::ok) + .filter_map(|line| serde_json::from_str(&line).ok()) + .collect()) + } + + fn terminal_event( + &self, + request_id: &str, + state: RewindEgressState, + error: Option, + ) -> Result { + validate_id("request id", request_id)?; + let _guard = EGRESS_IO_LOCK + .lock() + .map_err(|err| format!("Rewind egress log lock failed: {err}"))?; + let events = self.read_all_unlocked()?; + let prepared = events.iter().find(|event| { + event.request_id == request_id && event.state == RewindEgressState::Prepared + }); + let Some(prepared) = prepared else { + return Err("Unknown Rewind egress request id.".to_string()); + }; + if events.iter().any(|event| { + event.request_id == request_id && event.state != RewindEgressState::Prepared + }) { + return Err("Rewind egress request is already complete.".to_string()); + } + let event = RewindEgressEvent { + request_id: request_id.to_string(), + occurred_at: now_iso(), + state, + packet: None, + operation: prepared.operation.clone(), + receipt: None, + evidence_count: prepared.evidence_count, + packet_bytes: prepared.packet_bytes, + error, + }; + self.append_unlocked(&event)?; + Ok(event) + } +} + +fn egress_store(app: &AppHandle) -> Result { + let dir = app + .path() + .app_data_dir() + .map_err(|err| format!("app data directory unavailable: {err}"))? + .join("screen-memory"); + let store = EgressStore::new(dir.join(EGRESS_LOG_NAME)); + store.scrub_legacy_content()?; + Ok(store) +} + +fn now_iso() -> String { + Utc::now().to_rfc3339() +} + +fn next_request_id() -> String { + let counter = REQUEST_COUNTER.fetch_add(1, Ordering::Relaxed); + format!("egress-{}-{counter}", Utc::now().timestamp_millis()) +} + +fn char_count(value: &str) -> usize { + value.chars().count() +} + +fn validate_id(label: &str, value: &str) -> Result<(), String> { + if value.trim().is_empty() || char_count(value) > MAX_ID_CHARS { + return Err(format!("{label} must be 1-{MAX_ID_CHARS} characters.")); + } + Ok(()) +} + +fn contains_encoded_media(value: &str) -> bool { + let value = value.to_ascii_lowercase(); + ["data:image/", "data:video/", "data:audio/", ";base64,"] + .iter() + .any(|marker| value.contains(marker)) +} + +fn looks_like_credential_token(value: &str) -> bool { + let trimmed = value.trim_matches(|ch: char| { + ch.is_ascii_whitespace() || matches!(ch, ',' | ';' | '"' | '\'' | '(' | ')' | '[' | ']') + }); + (trimmed.starts_with("sk-") && trimmed.len() >= 15) + || (["ghp_", "gho_", "ghu_", "ghs_", "ghr_"] + .iter() + .any(|prefix| trimmed.starts_with(prefix)) + && trimmed.len() >= 16) + || (trimmed.starts_with("AKIA") + && trimmed.len() == 20 + && trimmed + .chars() + .all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit())) +} + +fn redact_obvious_credentials(value: &str) -> String { + let mut redact_next_bearer = false; + value + .split_whitespace() + .map(|token| { + if redact_next_bearer { + redact_next_bearer = false; + return "[REDACTED]".to_string(); + } + if token.eq_ignore_ascii_case("bearer") { + redact_next_bearer = true; + return token.to_string(); + } + let lower = token.to_ascii_lowercase(); + let assignment = [ + "api_key=", + "api-key=", + "access_token=", + "access-token=", + "password=", + "secret=", + ] + .iter() + .find(|prefix| lower.starts_with(**prefix)); + if let Some(prefix) = assignment { + return format!("{}[REDACTED]", &token[..prefix.len()]); + } + if looks_like_credential_token(token) { + "[REDACTED CREDENTIAL]".to_string() + } else { + token.to_string() + } + }) + .collect::>() + .join(" ") +} + +fn validate_packet( + mut packet: RewindEvidencePacket, + includes_frames: bool, +) -> Result { + if includes_frames { + return Err( + "Frame egress requires per-question approval and is not supported by this text gate." + .to_string(), + ); + } + packet.question = packet.question.trim().to_string(); + if packet.question.is_empty() || char_count(&packet.question) > MAX_QUESTION_CHARS { + return Err(format!( + "Question must be 1-{MAX_QUESTION_CHARS} characters." + )); + } + if contains_encoded_media(&packet.question) { + return Err("Rewind text packets cannot contain encoded media.".to_string()); + } + packet.question = redact_obvious_credentials(&packet.question); + if packet.evidence.is_empty() || packet.evidence.len() > MAX_EVIDENCE_ITEMS { + return Err(format!( + "Evidence packet must contain 1-{MAX_EVIDENCE_ITEMS} text items." + )); + } + for evidence in &mut packet.evidence { + evidence.id = evidence.id.trim().to_string(); + evidence.moment_id = evidence.moment_id.trim().to_string(); + evidence.excerpt = evidence.excerpt.trim().to_string(); + validate_id("Evidence id", &evidence.id)?; + validate_id("Moment id", &evidence.moment_id)?; + if evidence.excerpt.is_empty() || char_count(&evidence.excerpt) > MAX_EXCERPT_CHARS { + return Err(format!( + "Evidence excerpt must be 1-{MAX_EXCERPT_CHARS} characters." + )); + } + if contains_encoded_media(&evidence.excerpt) { + return Err("Rewind text packets cannot contain encoded media.".to_string()); + } + evidence.excerpt = redact_obvious_credentials(&evidence.excerpt); + if let Some(timestamp) = &mut evidence.captured_at { + *timestamp = timestamp.trim().to_string(); + if timestamp.is_empty() || char_count(timestamp) > MAX_TIMESTAMP_CHARS { + return Err("Evidence timestamp is invalid.".to_string()); + } + chrono::DateTime::parse_from_rfc3339(timestamp) + .map_err(|_| "Evidence timestamp must be RFC3339.".to_string())?; + } + } + Ok(packet) +} + +fn prepare_evidence( + store: &EgressStore, + packet: RewindEvidencePacket, + includes_frames: bool, +) -> Result { + let packet = validate_packet(packet, includes_frames)?; + let packet_bytes = serde_json::to_vec(&packet) + .map_err(|err| format!("Rewind evidence packet encoding failed: {err}"))? + .len(); + let request_id = next_request_id(); + let event = RewindEgressEvent { + request_id: request_id.clone(), + occurred_at: now_iso(), + state: RewindEgressState::Prepared, + packet: None, + operation: Some("agent-query".to_string()), + receipt: Some(json!({ + "evidence": packet.evidence.iter().map(|evidence| json!({ + "id": evidence.id, + "momentId": evidence.moment_id, + "sourceType": evidence.source_type, + "capturedAt": evidence.captured_at, + })).collect::>(), + })), + evidence_count: packet.evidence.len(), + packet_bytes, + error: None, + }; + // This durable append is deliberately before the packet is returned to a + // caller that might send it. + store.append(&event)?; + Ok(RewindEgressPrepared { + request_id, + evidence_count: packet.evidence.len(), + packet_bytes, + packet, + }) +} + +#[tauri::command] +pub async fn rewind_prepare_evidence_egress( + app: AppHandle, + packet: RewindEvidencePacket, + includes_frames: Option, +) -> Result { + prepare_evidence( + &egress_store(&app)?, + packet, + includes_frames.unwrap_or(false), + ) +} + +#[tauri::command] +pub async fn rewind_complete_evidence_egress( + app: AppHandle, + request_id: String, +) -> Result { + egress_store(&app)?.terminal_event(&request_id, RewindEgressState::Completed, None) +} + +#[tauri::command] +pub async fn rewind_fail_evidence_egress( + app: AppHandle, + request_id: String, + error: String, +) -> Result { + let error = error.trim(); + if error.is_empty() || char_count(error) > MAX_ERROR_CHARS { + return Err(format!("Error must be 1-{MAX_ERROR_CHARS} characters.")); + } + egress_store(&app)?.terminal_event( + &request_id, + RewindEgressState::Failed, + Some(error.to_string()), + ) +} + +#[tauri::command] +pub async fn rewind_list_evidence_egress( + app: AppHandle, + limit: Option, +) -> Result, String> { + let mut events = egress_store(&app)?.read_all()?; + events.reverse(); + events.truncate(limit.unwrap_or(100).clamp(1, MAX_LIST_LIMIT)); + Ok(events) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_store(name: &str) -> EgressStore { + let dir = std::env::temp_dir().join(format!( + "clips-rewind-egress-{name}-{}-{}", + std::process::id(), + REQUEST_COUNTER.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + EgressStore::new(dir.join(EGRESS_LOG_NAME)) + } + + fn packet() -> RewindEvidencePacket { + RewindEvidencePacket { + question: "What happened?".to_string(), + evidence: vec![RewindEvidenceForEgress { + id: "evidence-1".to_string(), + moment_id: "moment-1".to_string(), + source_type: RewindEvidenceSource::Transcript, + captured_at: Some("2026-07-14T12:00:00Z".to_string()), + excerpt: "A bounded local excerpt.".to_string(), + }], + } + } + + #[test] + fn redacts_obvious_credentials_before_logging_or_returning_packets() { + let store = temp_store("redaction"); + let mut packet = packet(); + packet.evidence[0].excerpt = + "api_key=super-secret Bearer abcdefghijklmnop sk-abcdefghijklmnop".into(); + let prepared = prepare_evidence(&store, packet, false).unwrap(); + assert_eq!( + prepared.packet.evidence[0].excerpt, + "api_key=[REDACTED] Bearer [REDACTED] [REDACTED CREDENTIAL]" + ); + let logged = store.read_all().unwrap(); + assert!(logged[0].packet.is_none()); + assert_eq!(logged[0].operation.as_deref(), Some("agent-query")); + assert_eq!( + logged[0].receipt.as_ref().unwrap()["evidence"][0]["id"], + "evidence-1" + ); + } + + #[test] + fn reads_agent_neutral_chapter_and_local_evidence_activity() { + let chapter: RewindEgressEvent = serde_json::from_str( + r#"{"requestId":"chapter-1","occurredAt":"2026-01-01T00:00:00Z","state":"prepared","packet":{"question":"recent work","evidence":[{"id":"c1","momentId":"c1","sourceType":"chapter","capturedAt":"2026-01-01T00:00:00Z","excerpt":"Editor work"}]},"evidenceCount":1,"packetBytes":10,"error":null}"#, + ) + .unwrap(); + assert_eq!( + chapter.packet.unwrap().evidence[0].source_type, + RewindEvidenceSource::Chapter + ); + let local: RewindEgressEvent = serde_json::from_str( + r#"{"requestId":"local-1","occurredAt":"2026-01-01T00:00:00Z","state":"local-evidence-read","packet":{"question":"inspect frame","evidence":[]},"evidenceCount":1,"packetBytes":0,"error":null}"#, + ) + .unwrap(); + assert_eq!(local.state, RewindEgressState::LocalEvidenceRead); + } + + #[test] + fn frames_fail_before_writing() { + let store = temp_store("frames"); + assert!(prepare_evidence(&store, packet(), true) + .unwrap_err() + .contains("approval")); + assert!(store.read_all().unwrap().is_empty()); + } + + #[test] + fn prepared_event_precedes_one_terminal_event() { + let store = temp_store("complete"); + let prepared = prepare_evidence(&store, packet(), false).unwrap(); + let completed = store + .terminal_event(&prepared.request_id, RewindEgressState::Completed, None) + .unwrap(); + let events = store.read_all().unwrap(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].state, RewindEgressState::Prepared); + assert_eq!(events[1].state, RewindEgressState::Completed); + assert!(events[0].packet.is_none()); + assert!(events[0].receipt.is_some()); + assert!(completed.packet.is_none()); + assert!(store + .terminal_event( + &prepared.request_id, + RewindEgressState::Failed, + Some("late".into()) + ) + .is_err()); + } + + #[test] + fn scrubs_legacy_packet_text_while_preserving_provenance() { + let store = temp_store("legacy-scrub"); + if let Some(parent) = store.path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write( + &store.path, + r#"{"requestId":"legacy-1","occurredAt":"2026-01-01T00:00:00Z","state":"prepared","packet":{"question":"private question","evidence":[{"id":"e1","momentId":"m1","sourceType":"transcript","capturedAt":"2026-01-01T00:00:00Z","excerpt":"private excerpt"}]},"evidenceCount":1,"packetBytes":42,"error":null} +"#, + ) + .unwrap(); + + store.scrub_legacy_content().unwrap(); + + let raw = std::fs::read_to_string(&store.path).unwrap(); + assert!(!raw.contains("private question")); + assert!(!raw.contains("private excerpt")); + let events = store.read_all().unwrap(); + assert_eq!( + events[0].receipt.as_ref().unwrap()["evidence"][0]["id"], + "e1" + ); + } + + #[test] + fn failure_and_unknown_ids_are_auditable_and_bounded() { + let store = temp_store("failure"); + assert!(store + .terminal_event("missing", RewindEgressState::Failed, Some("no".into())) + .is_err()); + let prepared = prepare_evidence(&store, packet(), false).unwrap(); + store + .terminal_event( + &prepared.request_id, + RewindEgressState::Failed, + Some("network unavailable".into()), + ) + .unwrap(); + assert_eq!( + store.read_all().unwrap()[1].state, + RewindEgressState::Failed + ); + + let mut too_many = packet(); + too_many.evidence = (0..=MAX_EVIDENCE_ITEMS) + .map(|index| RewindEvidenceForEgress { + id: format!("e-{index}"), + moment_id: format!("m-{index}"), + source_type: RewindEvidenceSource::Ocr, + captured_at: None, + excerpt: "text".into(), + }) + .collect(); + assert!(prepare_evidence(&store, too_many, false).is_err()); + + let mut encoded = packet(); + encoded.evidence[0].excerpt = "data:image/png;base64,example".into(); + assert!(prepare_evidence(&store, encoded, false).is_err()); + } +} diff --git a/templates/clips/desktop/src-tauri/src/rewind_local_ask.rs b/templates/clips/desktop/src-tauri/src/rewind_local_ask.rs new file mode 100644 index 0000000000..602c90052e --- /dev/null +++ b/templates/clips/desktop/src-tauri/src/rewind_local_ask.rs @@ -0,0 +1,803 @@ +//! First-party, fully local Rewind search and replay. +//! +//! This module reads local indexes inside Clips, returns bounded text-only evidence to the +//! renderer, and never performs a network request. Raw archive paths stay on +//! the Rust side of the command boundary. + +use crate::capture_graph::{CaptureGraphState, CaptureSource}; +use crate::config::RewindCaptureMode; +use crate::native_screen::{self, NativeAudioSelection, NativeMediaSlice}; +use crate::screen_memory::{self, ScreenMemoryEvent, ScreenMemorySegmentMetadata}; +use crate::screen_memory_ocr::ScreenMemoryOcrRow; +use crate::screen_memory_transcript::ScreenMemoryTranscriptRow; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, SystemTime}; +use tauri::{AppHandle, Manager}; + +const MAX_QUERY_BYTES: usize = 500; +const DEFAULT_EVIDENCE_LIMIT: usize = 12; +const MAX_EVIDENCE_LIMIT: usize = 20; +const MAX_EXCERPT_BYTES: usize = 600; +const REPLAY_BEFORE_MS: u64 = 5_000; +const REPLAY_AFTER_MS: u64 = 10_000; +const MAX_REPLAY_MS: u64 = 30_000; +const PREVIEW_MAX_FILES: usize = 8; +const PREVIEW_MAX_AGE: Duration = Duration::from_secs(60 * 60); + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "kebab-case")] +enum LocalEvidenceSource { + AppContext, + Transcript, + Ocr, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RewindLocalEvidence { + id: String, + source_type: LocalEvidenceSource, + captured_at: String, + excerpt: String, + confidence: Option, + segment_id: String, + offset_ms: u64, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RewindLocalCoverageGap { + kind: String, + source: String, + started_at: Option, + ended_at: Option, + detail: String, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RewindLocalCoverage { + segments_considered: usize, + transcript_indexes_ready: usize, + ocr_indexes_ready: usize, + gaps: Vec, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RewindLocalAskResult { + query: String, + answer_summary: String, + evidence: Vec, + coverage: RewindLocalCoverage, + confidence: String, + truncated: bool, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RewindReplayResult { + opened: bool, + segment_id: String, + offset_ms: u64, + preview_started_ms: u64, + preview_ended_ms: u64, +} + +#[derive(Clone, Debug)] +struct ScoredEvidence { + score: usize, + evidence: RewindLocalEvidence, +} + +#[derive(Debug, Deserialize)] +struct IndexStatus { + state: String, +} + +fn bounded_text(value: &str, max_bytes: usize) -> String { + if value.len() <= max_bytes { + return value.to_owned(); + } + let mut end = max_bytes; + while !value.is_char_boundary(end) { + end -= 1; + } + format!("{}…", value[..end].trim_end()) +} + +fn normalized_terms(query: &str) -> Vec { + let stop_words = [ + "a", "about", "an", "and", "did", "do", "for", "from", "i", "in", "is", "it", "me", "my", + "of", "on", "the", "to", "was", "what", "when", "where", "with", + ]; + let mut terms = query + .split(|character: char| !character.is_alphanumeric()) + .map(str::trim) + .filter(|term| term.len() >= 2) + .map(str::to_lowercase) + .filter(|term| !stop_words.contains(&term.as_str())) + .collect::>(); + terms.sort(); + terms.dedup(); + if terms.is_empty() { + let fallback = query.trim().to_lowercase(); + if !fallback.is_empty() { + terms.push(fallback); + } + } + terms +} + +fn match_score(text: &str, terms: &[String]) -> usize { + let haystack = text.to_lowercase(); + let counts = terms + .iter() + .map(|term| haystack.matches(term).count()) + .collect::>(); + let matched_terms = counts.iter().filter(|count| **count > 0).count(); + let minimum_terms = if terms.len() <= 1 { + 1 + } else { + terms.len().div_ceil(2) + }; + if matched_terms < minimum_terms { + return 0; + } + matched_terms * 1_000 + counts.into_iter().sum::() +} + +fn matching_excerpt(text: &str, terms: &[String], max_bytes: usize) -> String { + let lines = text.lines().collect::>(); + let Some((best_index, _)) = lines + .iter() + .enumerate() + .map(|(index, line)| (index, match_score(line, terms))) + .max_by_key(|(_, score)| *score) + .filter(|(_, score)| *score > 0) + else { + return bounded_text(text.trim(), max_bytes); + }; + let start = best_index.saturating_sub(1); + let end = (best_index + 2).min(lines.len()); + bounded_text(lines[start..end].join("\n").trim(), max_bytes) +} + +fn parse_time(value: &str) -> Option> { + DateTime::parse_from_rfc3339(value) + .ok() + .map(|value| value.with_timezone(&Utc)) +} + +fn event_segment<'a>( + event: &ScreenMemoryEvent, + segments: &'a [ScreenMemorySegmentMetadata], +) -> Option<(&'a ScreenMemorySegmentMetadata, u64)> { + let captured = parse_time(&event.captured_at)?; + segments.iter().find_map(|segment| { + let started = parse_time(&segment.started_at)?; + let ended = parse_time(&segment.ended_at)?; + if captured < started || captured > ended { + return None; + } + let offset = captured + .signed_duration_since(started) + .num_milliseconds() + .max(0) as u64; + Some((segment, offset.min(segment.duration_ms as u64))) + }) +} + +fn read_jsonl Deserialize<'de>>(path: &Path) -> Vec { + let Ok(file) = File::open(path) else { + return Vec::new(); + }; + BufReader::new(file) + .lines() + .map_while(Result::ok) + .filter_map(|line| serde_json::from_str(&line).ok()) + .collect() +} + +fn index_ready(path: &Path) -> bool { + std::fs::read(path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .is_some_and(|status| status.state == "ready") +} + +fn adjacent_path(segment: &ScreenMemorySegmentMetadata, suffix: &str) -> Option { + segment + .path + .parent() + .map(|directory| directory.join(format!("{}.{suffix}", segment.id))) +} + +fn evidence_id(source: LocalEvidenceSource, segment_id: &str, offset_ms: u64) -> String { + let source = match source { + LocalEvidenceSource::AppContext => "app", + LocalEvidenceSource::Transcript => "transcript", + LocalEvidenceSource::Ocr => "ocr", + }; + format!("{source}:{segment_id}:{offset_ms}") +} + +fn push_match( + matches: &mut Vec, + terms: &[String], + source_type: LocalEvidenceSource, + captured_at: String, + text: String, + confidence: Option, + segment_id: String, + offset_ms: u64, +) { + let score = match_score(&text, terms); + if score == 0 { + return; + } + matches.push(ScoredEvidence { + score, + evidence: RewindLocalEvidence { + id: evidence_id(source_type, &segment_id, offset_ms), + source_type, + captured_at, + excerpt: matching_excerpt(&text, terms, MAX_EXCERPT_BYTES), + confidence, + segment_id, + offset_ms, + }, + }); +} + +fn local_query( + directory: &Path, + segments: Vec, + query: &str, + limit: usize, + mut gaps: Vec, +) -> RewindLocalAskResult { + let terms = normalized_terms(query); + let valid_segments = segments + .into_iter() + .filter(|segment| { + segment.path.exists() + && !segment.corrupt + && segment.error.is_none() + && !segment.exclusion_tainted + }) + .collect::>(); + let mut matches = Vec::new(); + + for event in read_jsonl::(&directory.join("events.jsonl")) { + if event.source == "coverage-gap" { + continue; + } + let Some((segment, offset_ms)) = event_segment(&event, &valid_segments) else { + continue; + }; + let text = [ + event.app_name.as_deref(), + event.window_title.as_deref(), + event.bundle_id.as_deref(), + ] + .into_iter() + .flatten() + .collect::>() + .join(" — "); + push_match( + &mut matches, + &terms, + LocalEvidenceSource::AppContext, + event.captured_at, + text, + None, + segment.id.clone(), + offset_ms, + ); + } + + let mut transcript_ready = 0; + let mut ocr_ready = 0; + for segment in &valid_segments { + let transcript_status = adjacent_path(segment, "transcript-status.json"); + let transcript_rows = adjacent_path(segment, "transcript.jsonl"); + if transcript_status.as_deref().is_some_and(index_ready) { + transcript_ready += 1; + if let Some(path) = transcript_rows { + for row in read_jsonl::(&path) { + push_match( + &mut matches, + &terms, + LocalEvidenceSource::Transcript, + row.captured_at, + row.text, + None, + segment.id.clone(), + row.start_ms.min(segment.duration_ms as u64), + ); + } + } + } else { + gaps.push(RewindLocalCoverageGap { + kind: "index".into(), + source: "transcript".into(), + started_at: Some(segment.started_at.clone()), + ended_at: Some(segment.ended_at.clone()), + detail: "Local transcript is not ready for this segment.".into(), + }); + } + + let ocr_status = adjacent_path(segment, "ocr-status.json"); + let ocr_rows = adjacent_path(segment, "ocr.jsonl"); + if ocr_status.as_deref().is_some_and(index_ready) { + ocr_ready += 1; + if let Some(path) = ocr_rows { + for row in read_jsonl::(&path) { + push_match( + &mut matches, + &terms, + LocalEvidenceSource::Ocr, + row.captured_at, + row.ocr_text, + Some(row.confidence), + segment.id.clone(), + row.offset_ms.max(0) as u64, + ); + } + } + } else { + gaps.push(RewindLocalCoverageGap { + kind: "index".into(), + source: "ocr".into(), + started_at: Some(segment.started_at.clone()), + ended_at: Some(segment.ended_at.clone()), + detail: "Local visual index is not ready for this segment.".into(), + }); + } + } + + matches.sort_by(|left, right| { + right + .score + .cmp(&left.score) + .then_with(|| right.evidence.captured_at.cmp(&left.evidence.captured_at)) + }); + let total_matches = matches.len(); + let limit = limit.clamp(1, MAX_EVIDENCE_LIMIT); + let evidence = matches + .into_iter() + .take(limit) + .map(|item| item.evidence) + .collect::>(); + let truncated = total_matches > evidence.len(); + let answer_summary = match evidence.first() { + Some(_) => format!( + "Found {total_matches} local match{} in Rewind. The strongest match is shown first; this is search evidence, not a generated conclusion.", + if total_matches == 1 { "" } else { "es" } + ), + None => "No matching local evidence was found in the indexed Rewind coverage. This does not prove the event did not happen; coverage or indexes may be incomplete.".into(), + }; + let confidence = if evidence.is_empty() { + "No matching evidence".into() + } else if gaps.is_empty() && !truncated { + "Direct local matches; complete for the indexed segments shown".into() + } else { + "Direct local matches; incomplete coverage or indexing".into() + }; + RewindLocalAskResult { + query: query.to_owned(), + answer_summary, + evidence, + coverage: RewindLocalCoverage { + segments_considered: valid_segments.len(), + transcript_indexes_ready: transcript_ready, + ocr_indexes_ready: ocr_ready, + gaps, + }, + confidence, + truncated, + } +} + +fn graph_coverage_gaps(app: &AppHandle) -> Vec { + let graph_state = app.state::(); + let Ok(graph) = graph_state.0.lock() else { + return vec![RewindLocalCoverageGap { + kind: "capture".into(), + source: "screen".into(), + started_at: None, + ended_at: None, + detail: "Capture coverage state is unavailable.".into(), + }]; + }; + let Ok(status) = graph.status_at(std::time::Instant::now()) else { + return Vec::new(); + }; + status + .coverage_gaps + .into_iter() + .map(|gap| RewindLocalCoverageGap { + kind: "capture".into(), + source: match gap.source { + CaptureSource::Screen => "screen", + CaptureSource::SystemAudio => "system-audio", + CaptureSource::Microphone => "microphone", + CaptureSource::Camera => "camera", + } + .into(), + started_at: Some(gap.interval.started_at), + ended_at: Some(gap.interval.ended_at), + detail: format!("Capture gap: {:?}.", gap.reason), + }) + .collect() +} + +#[tauri::command] +pub(crate) fn rewind_local_ask( + app: AppHandle, + query: String, + limit: Option, +) -> Result { + let query = query.trim(); + if query.is_empty() { + return Err("Ask Rewind needs a question or search phrase.".into()); + } + if query.len() > MAX_QUERY_BYTES { + return Err(format!( + "Ask Rewind queries must be at most {MAX_QUERY_BYTES} bytes." + )); + } + let segments = screen_memory::finalized_segments(&app)?; + let directory = segments + .first() + .and_then(|segment| segment.path.parent().map(Path::to_path_buf)) + .or_else(|| { + app.path() + .app_data_dir() + .ok() + .map(|path| path.join("screen-memory")) + }) + .ok_or_else(|| "Rewind storage directory is unavailable.".to_string())?; + Ok(local_query( + &directory, + segments, + query, + limit.unwrap_or(DEFAULT_EVIDENCE_LIMIT), + graph_coverage_gaps(&app), + )) +} + +fn replay_slice(duration_ms: u64, offset_ms: u64) -> Result<(u64, u64), String> { + if duration_ms == 0 || offset_ms >= duration_ms { + return Err("Replay moment is outside the retained segment.".into()); + } + let started_ms = offset_ms.saturating_sub(REPLAY_BEFORE_MS); + let ended_ms = offset_ms.saturating_add(REPLAY_AFTER_MS).min(duration_ms); + if ended_ms <= started_ms || ended_ms - started_ms > MAX_REPLAY_MS { + return Err("Replay moment could not be bounded safely.".into()); + } + Ok((started_ms, ended_ms)) +} + +fn cleanup_previews(directory: &Path) { + let now = SystemTime::now(); + let mut files = std::fs::read_dir(directory) + .into_iter() + .flatten() + .flatten() + .filter_map(|entry| { + let metadata = entry.metadata().ok()?; + if !metadata.is_file() { + return None; + } + Some(( + entry.path(), + metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH), + )) + }) + .collect::>(); + for (path, modified) in &files { + if now.duration_since(*modified).unwrap_or_default() > PREVIEW_MAX_AGE { + let _ = std::fs::remove_file(path); + } + } + files.retain(|(path, _)| path.exists()); + files.sort_by_key(|(_, modified)| std::cmp::Reverse(*modified)); + for (path, _) in files.into_iter().skip(PREVIEW_MAX_FILES.saturating_sub(1)) { + let _ = std::fs::remove_file(path); + } +} + +fn slice_overlaps_graph_gap( + app: &AppHandle, + segment: &ScreenMemorySegmentMetadata, + started_ms: u64, + ended_ms: u64, +) -> bool { + let selected_start = segment.graph_started_elapsed_ms.saturating_add(started_ms); + let selected_end = segment.graph_started_elapsed_ms.saturating_add(ended_ms); + app.state::() + .0 + .lock() + .ok() + .and_then(|graph| graph.status_at(std::time::Instant::now()).ok()) + .is_some_and(|status| { + status.coverage_gaps.iter().any(|gap| { + gap.source == CaptureSource::Screen + && gap.interval.started_elapsed_ms < selected_end + && gap.interval.ended_elapsed_ms > selected_start + }) + }) +} + +fn validate_replay_candidate( + segment: &ScreenMemorySegmentMetadata, + overlaps_gap: bool, +) -> Result<(), String> { + if segment.corrupt || segment.error.is_some() || segment.exclusion_tainted { + return Err("Replay evidence is corrupt, incomplete, or privacy-tainted.".into()); + } + if segment.graph_ended_elapsed_ms <= segment.graph_started_elapsed_ms { + return Err("Replay evidence has no trustworthy capture coverage mapping.".into()); + } + if overlaps_gap { + return Err("Replay evidence overlaps a recorded capture gap.".into()); + } + Ok(()) +} + +#[tauri::command] +pub(crate) async fn rewind_replay_moment( + app: AppHandle, + segment_id: String, + offset_ms: u64, +) -> Result { + if segment_id.trim().is_empty() + || segment_id.len() > 255 + || !segment_id + .chars() + .all(|character| character.is_ascii_alphanumeric() || matches!(character, '-' | '_')) + { + return Err("Replay segment ID is invalid.".into()); + } + let segment = screen_memory::finalized_segments(&app)? + .into_iter() + .find(|segment| segment.id == segment_id) + .ok_or_else(|| "Replay segment is missing from retained Rewind coverage.".to_string())?; + let duration_ms = u64::try_from(segment.duration_ms) + .map_err(|_| "Replay segment duration is invalid.".to_string())?; + let (started_ms, ended_ms) = replay_slice(duration_ms, offset_ms)?; + validate_replay_candidate( + &segment, + slice_overlaps_graph_gap(&app, &segment, started_ms, ended_ms), + )?; + + let preview_dir = app + .path() + .app_local_data_dir() + .map_err(|error| format!("local preview directory unavailable: {error}"))? + .join("rewind-previews"); + std::fs::create_dir_all(&preview_dir) + .map_err(|error| format!("local preview directory unavailable: {error}"))?; + cleanup_previews(&preview_dir); + let output = preview_dir.join(format!( + "moment-{}-{}-{}.mp4", + segment.id, + offset_ms, + Utc::now().timestamp_millis() + )); + let slice = NativeMediaSlice { + path: segment.path.clone(), + system_audio_path: segment.system_audio_path.clone(), + microphone_path: segment.microphone_path.clone(), + start_ms: started_ms, + end_ms: ended_ms, + }; + let audio = if segment.capture_mode == RewindCaptureMode::VisualsAudio { + NativeAudioSelection::MixBoth + } else { + NativeAudioSelection::None + }; + let materialized = output.clone(); + tauri::async_runtime::spawn_blocking(move || { + native_screen::materialize_mp4_slices_exact(&[slice], &materialized, audio) + }) + .await + .map_err(|error| format!("local replay worker failed: {error}"))??; + + #[cfg(target_os = "macos")] + Command::new("/usr/bin/open") + .arg(&output) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| format!("could not open local replay: {error}"))?; + #[cfg(not(target_os = "macos"))] + return Err("Local Rewind replay is currently available on macOS.".into()); + + Ok(RewindReplayResult { + opened: true, + segment_id, + offset_ms, + preview_started_ms: started_ms, + preview_ended_ms: ended_ms, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use std::sync::atomic::{AtomicU64, Ordering}; + + static TEST_COUNTER: AtomicU64 = AtomicU64::new(0); + + fn test_dir(label: &str) -> PathBuf { + let path = std::env::temp_dir().join(format!( + "clips-rewind-local-{label}-{}-{}", + std::process::id(), + TEST_COUNTER.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&path).unwrap(); + path + } + + fn segment(directory: &Path, id: &str, tainted: bool) -> ScreenMemorySegmentMetadata { + let path = directory.join(format!("{id}.mp4")); + fs::write(&path, b"media").unwrap(); + ScreenMemorySegmentMetadata { + id: id.into(), + path, + file_name: format!("{id}.mp4"), + mime_type: "video/mp4".into(), + started_at: "2026-07-14T12:00:00Z".into(), + ended_at: "2026-07-14T12:05:00Z".into(), + duration_ms: 300_000, + width: Some(1920), + height: Some(1080), + bytes: 5, + system_audio_path: None, + microphone_path: None, + corrupt: false, + error: None, + capture_mode: RewindCaptureMode::Visuals, + exclusion_tainted: tainted, + graph_epoch_id: Some("test-epoch".into()), + graph_started_elapsed_ms: 1_000, + graph_ended_elapsed_ms: 301_000, + } + } + + #[test] + fn query_matches_all_local_sources_and_stays_bounded() { + let directory = test_dir("query"); + let segment = segment(&directory, "segment-one", false); + fs::write( + directory.join("events.jsonl"), + r#"{"capturedAt":"2026-07-14T12:00:10Z","appName":"Zoom","windowTitle":"Lilian audience review","bundleId":"us.zoom.xos","source":"accessibility"} +"#, + ) + .unwrap(); + fs::write( + directory.join("segment-one.transcript-status.json"), + r#"{"state":"ready"}"#, + ) + .unwrap(); + fs::write( + directory.join("segment-one.transcript.jsonl"), + r#"{"schemaVersion":1,"segmentId":"segment-one","source":"microphone","capturedAt":"2026-07-14T12:00:20Z","startMs":20000,"endMs":22000,"text":"Lilian explained the audience"} +"#, + ) + .unwrap(); + fs::write( + directory.join("segment-one.ocr-status.json"), + r#"{"state":"ready"}"#, + ) + .unwrap(); + fs::write( + directory.join("segment-one.ocr.jsonl"), + r#"{"schemaVersion":1,"segmentId":"segment-one","capturedAt":"2026-07-14T12:00:30Z","offsetMs":30000,"source":"ocr","ocrText":"Audience segments by Lilian","confidence":0.91,"frameWidth":1920,"frameHeight":1080} +"#, + ) + .unwrap(); + + let result = local_query(&directory, vec![segment], "Lilian audience", 2, vec![]); + assert_eq!(result.evidence.len(), 2); + assert!(result.truncated); + assert_eq!(result.coverage.transcript_indexes_ready, 1); + assert_eq!(result.coverage.ocr_indexes_ready, 1); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn renderer_result_has_no_archive_path_field() { + let directory = test_dir("path-denial"); + let segment = segment(&directory, "secret-segment", false); + fs::write( + directory.join("events.jsonl"), + r#"{"capturedAt":"2026-07-14T12:00:10Z","appName":"Zoom","windowTitle":"needle","bundleId":null,"source":"accessibility"} +"#, + ) + .unwrap(); + let result = local_query(&directory, vec![segment], "needle", 5, vec![]); + let json = serde_json::to_value(result).unwrap(); + assert!(json.get("path").is_none()); + assert!(json["evidence"][0].get("path").is_none()); + assert_eq!(json["evidence"][0]["segmentId"], "secret-segment"); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn tainted_segments_are_excluded_and_report_no_match() { + let directory = test_dir("tainted"); + let segment = segment(&directory, "tainted", true); + fs::write( + directory.join("events.jsonl"), + r#"{"capturedAt":"2026-07-14T12:00:10Z","appName":"Secrets","windowTitle":"needle","bundleId":null,"source":"accessibility"} +"#, + ) + .unwrap(); + let result = local_query(&directory, vec![segment], "needle", 5, vec![]); + assert!(result.evidence.is_empty()); + assert_eq!(result.coverage.segments_considered, 0); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn replay_slice_clamps_without_pre_range_leakage() { + assert_eq!(replay_slice(60_000, 20_000).unwrap(), (15_000, 30_000)); + assert_eq!(replay_slice(60_000, 2_000).unwrap(), (0, 12_000)); + assert_eq!(replay_slice(60_000, 58_000).unwrap(), (53_000, 60_000)); + assert!(replay_slice(60_000, 60_000).is_err()); + assert!(replay_slice(0, 0).is_err()); + } + + #[test] + fn replay_rejects_invalid_segment_ids_before_path_lookup() { + for id in ["../segment", "/tmp/archive.mp4", "a b", ""] { + assert!( + id.is_empty() + || id.len() > 255 + || !id.chars().all(|character| character.is_ascii_alphanumeric() + || matches!(character, '-' | '_')) + ); + } + } + + #[test] + fn replay_rejects_tainted_and_gapped_evidence() { + let directory = test_dir("replay-rejection"); + let tainted = segment(&directory, "tainted", true); + assert!(validate_replay_candidate(&tainted, false) + .unwrap_err() + .contains("privacy-tainted")); + let clean = segment(&directory, "clean", false); + assert!(validate_replay_candidate(&clean, true) + .unwrap_err() + .contains("capture gap")); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn query_terms_and_excerpts_are_bounded() { + let oversized = "x".repeat(MAX_EXCERPT_BYTES + 100); + let bounded = bounded_text(&oversized, MAX_EXCERPT_BYTES); + assert!(bounded.len() <= MAX_EXCERPT_BYTES + '…'.len_utf8()); + assert_eq!( + normalized_terms("What did I do in the meeting?"), + vec!["meeting"] + ); + let terms = normalized_terms("orchid submarine 731"); + let ocr = "unrelated Plaud content\norchid subnlfv 731\nmore unrelated content"; + assert!(match_score(ocr, &terms) > 0); + assert_eq!(matching_excerpt(ocr, &terms, MAX_EXCERPT_BYTES), ocr); + assert_eq!(match_score("unrelated item 731", &terms), 0); + } +} diff --git a/templates/clips/desktop/src-tauri/src/rewind_meeting_history.rs b/templates/clips/desktop/src-tauri/src/rewind_meeting_history.rs new file mode 100644 index 0000000000..eb6c56556d --- /dev/null +++ b/templates/clips/desktop/src-tauri/src/rewind_meeting_history.rs @@ -0,0 +1,595 @@ +//! Explicit, local-only meeting transcript recovery from the rolling Rewind. +//! +//! The normal meeting workflow is still authoritative. This command only +//! prepends locally indexed transcript rows when a person deliberately asks to +//! include the scheduled portion they missed before pressing Start notes. + +use crate::capture_graph::{CaptureConsumer, CaptureSource}; +use crate::config::RewindCaptureMode; +use crate::screen_memory::{self, ScreenMemoryRuntimeState, ScreenMemorySegmentMetadata}; +use crate::screen_memory_transcript::{ + ScreenMemoryTranscriptRow, ScreenMemoryTranscriptState, ScreenMemoryTranscriptStatus, +}; +use chrono::{DateTime, Duration as ChronoDuration, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; +use tauri::{AppHandle, Manager}; + +const COVERAGE_TOLERANCE_MS: i64 = 2_500; +const MAX_MEETING_HISTORY_HOURS: i64 = 8; +const INDEX_WAIT_TIMEOUT: Duration = Duration::from_secs(45); +const INDEX_POLL_INTERVAL: Duration = Duration::from_millis(200); +static HISTORY_REQUEST_COUNTER: AtomicU64 = AtomicU64::new(1); + +#[derive(Default)] +pub struct RewindMeetingHistoryState { + requests: Mutex>, +} + +#[derive(Clone)] +struct PreparedMeetingHistory { + scheduled_start: DateTime, + captured_until: DateTime, + segments: Vec, + pin_id: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RewindMeetingHistoryAvailability { + pub available: bool, + pub reason: Option, + pub covered_from: Option, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RewindMeetingTranscriptSegment { + pub start_ms: u64, + pub end_ms: u64, + pub text: String, + pub source: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RewindMeetingHistoryResult { + pub scheduled_start: String, + pub captured_until: String, + pub segments: Vec, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RewindMeetingHistoryPrepared { + pub token: String, + pub scheduled_start: String, + pub captured_until: String, +} + +#[tauri::command] +pub async fn rewind_meeting_history_status( + app: AppHandle, + scheduled_start: String, +) -> Result { + let requested_start = parse_scheduled_start(&scheduled_start)?; + availability(&app, requested_start).await +} + +#[tauri::command] +pub async fn rewind_meeting_history_prepare( + app: AppHandle, + scheduled_start: String, +) -> Result { + let requested_start = parse_scheduled_start(&scheduled_start)?; + let availability = availability(&app, requested_start).await?; + if !availability.available { + return Err(availability.reason.unwrap_or_else(|| { + "Rewind does not have continuous local audio coverage from this meeting's start." + .to_string() + })); + } + + authorize_explicit_meeting_extension(&app, requested_start)?; + + // Close the current logical fragment without stopping the one physical + // capture producer. Its transcript is queued locally by the normal segment + // finalization path; no raw media leaves the Mac. + screen_memory::fence_active_for_clip(&app) + .map_err(|error| format!("Could not include the earlier meeting: {error}"))?; + let captured_until = Utc::now(); + + let segments = relevant_segments(&app, requested_start, captured_until)?; + validate_continuous_audio_coverage(&segments, requested_start, captured_until)?; + reject_graph_gaps(&app, requested_start, captured_until)?; + let token = format!( + "meeting-history-{}-{}", + Utc::now().timestamp_millis(), + HISTORY_REQUEST_COUNTER.fetch_add(1, Ordering::Relaxed) + ); + let pin_id = format!("{token}-segments"); + for segment in &segments { + if let Err(error) = screen_memory::pin_segment(&app, &segment.id, &pin_id) { + for pinned in &segments { + let _ = screen_memory::unpin_segment(&app, &pinned.id, &pin_id); + } + return Err(error); + } + } + app.state::() + .requests + .lock() + .map_err(|error| error.to_string())? + .insert( + token.clone(), + PreparedMeetingHistory { + scheduled_start: requested_start, + captured_until, + segments, + pin_id, + }, + ); + + Ok(RewindMeetingHistoryPrepared { + token, + scheduled_start: requested_start.to_rfc3339(), + captured_until: captured_until.to_rfc3339(), + }) +} + +#[tauri::command] +pub async fn rewind_meeting_history_collect( + app: AppHandle, + token: String, +) -> Result { + let prepared = app + .state::() + .requests + .lock() + .map_err(|error| error.to_string())? + .get(&token) + .cloned() + .ok_or_else(|| { + "This prepared meeting-history request is no longer available.".to_string() + })?; + + let result = async { + wait_for_local_indexes(&app, &prepared.segments).await?; + + let mut rows = Vec::new(); + for segment in &prepared.segments { + rows.extend(read_transcript_rows(&app, &segment.id)?); + } + let mut mapped = + map_rows_to_meeting_timeline(rows, prepared.scheduled_start, prepared.captured_until); + mapped.sort_by_key(|row| (row.start_ms, row.end_ms, row.source.clone())); + + Ok(RewindMeetingHistoryResult { + scheduled_start: prepared.scheduled_start.to_rfc3339(), + captured_until: prepared.captured_until.to_rfc3339(), + segments: mapped, + }) + } + .await; + + if let Ok(mut requests) = app.state::().requests.lock() { + requests.remove(&token); + } + for segment in &prepared.segments { + let _ = screen_memory::unpin_segment(&app, &segment.id, &prepared.pin_id); + } + result +} + +#[tauri::command] +pub fn rewind_meeting_history_cancel(app: AppHandle, token: String) -> Result<(), String> { + let prepared = app + .state::() + .requests + .lock() + .map_err(|error| error.to_string())? + .remove(&token); + if let Some(prepared) = prepared { + for segment in &prepared.segments { + let _ = screen_memory::unpin_segment(&app, &segment.id, &prepared.pin_id); + } + } + Ok(()) +} + +fn parse_scheduled_start(value: &str) -> Result, String> { + let parsed = DateTime::parse_from_rfc3339(value) + .map_err(|_| "This meeting does not have a valid scheduled start time.".to_string())? + .with_timezone(&Utc); + let now = Utc::now(); + if parsed > now + ChronoDuration::minutes(1) { + return Err("The meeting has not started yet.".to_string()); + } + if now - parsed > ChronoDuration::hours(MAX_MEETING_HISTORY_HOURS) { + return Err(format!( + "Include from meeting start is limited to the previous {MAX_MEETING_HISTORY_HOURS} hours." + )); + } + Ok(parsed) +} + +async fn availability( + app: &AppHandle, + requested_start: DateTime, +) -> Result { + let status = screen_memory::screen_memory_status(app.clone()).await?; + let unavailable = |reason: &str| RewindMeetingHistoryAvailability { + available: false, + reason: Some(reason.to_string()), + covered_from: None, + }; + if !status.available || !status.config.enabled || status.config.paused { + return Ok(unavailable( + "Turn on Rewind before using Include from meeting start.", + )); + } + if status.config.capture_mode != RewindCaptureMode::VisualsAudio { + return Ok(unavailable( + "Include from meeting start needs Rewind set to Visuals + audio.", + )); + } + if status.exclusion_active { + return Ok(unavailable( + "Rewind is currently stopped for an excluded app.", + )); + } + if !matches!(status.state, ScreenMemoryRuntimeState::Recording) { + return Ok(unavailable("Rewind is not currently recording.")); + } + + let now = Utc::now(); + let mut segments = screen_memory::finalized_segments(app)?; + if let Some(active) = status.active_segment { + let active_start = parse_time(&active.started_at)?; + if active_start <= requested_start + ChronoDuration::milliseconds(COVERAGE_TOLERANCE_MS) + && validate_existing_prefix(&mut segments, requested_start, active_start).is_ok() + { + let covered_from = earliest_coverage_start(&segments) + .unwrap_or(active_start) + .to_rfc3339(); + if reject_graph_gaps(app, requested_start, now).is_err() { + return Ok(unavailable( + "Rewind recorded a source coverage gap after this meeting started.", + )); + } + return Ok(RewindMeetingHistoryAvailability { + available: true, + reason: None, + covered_from: Some(covered_from), + }); + } + if active_start <= now + && validate_existing_prefix(&mut segments, requested_start, active_start).is_ok() + { + let covered_from = earliest_coverage_start(&segments) + .unwrap_or(active_start) + .to_rfc3339(); + if reject_graph_gaps(app, requested_start, now).is_err() { + return Ok(unavailable( + "Rewind recorded a source coverage gap after this meeting started.", + )); + } + return Ok(RewindMeetingHistoryAvailability { + available: true, + reason: None, + covered_from: Some(covered_from), + }); + } + } + Ok(unavailable( + "Rewind does not have continuous local audio coverage from this meeting's start.", + )) +} + +/// Records that this longer-than-a-normal-Clip retrospective range was an +/// explicit scheduled-meeting choice. The graph's dedicated meeting-start API +/// intentionally permits this while ordinary Rewind extensions remain capped +/// at five minutes. +fn authorize_explicit_meeting_extension( + app: &AppHandle, + requested_start: DateTime, +) -> Result<(), String> { + let age = (Utc::now() - requested_start) + .to_std() + .map_err(|_| "The scheduled meeting start is in the future.".to_string())?; + let now = Instant::now(); + let requested_instant = now.checked_sub(age).unwrap_or(now); + let graph = app.state::(); + let mut graph = graph.0.lock().map_err(|error| error.to_string())?; + let lease = graph + .start_consumer( + CaptureConsumer::Meeting, + [CaptureSource::SystemAudio, CaptureSource::Microphone], + ) + .map_err(|error| error.to_string())?; + if let Err(error) = graph.extend_retrospectively_to_start(&lease.id, requested_instant) { + let _ = graph.end_consumer(&lease.id); + return Err(error.to_string()); + } + graph + .end_consumer(&lease.id) + .map_err(|error| error.to_string())?; + Ok(()) +} + +fn validate_existing_prefix( + segments: &mut Vec, + requested_start: DateTime, + active_start: DateTime, +) -> Result<(), String> { + let mut relevant = segments + .iter() + .filter_map(|segment| { + let start = parse_time(&segment.started_at).ok()?; + let end = parse_time(&segment.ended_at).ok()?; + (end >= requested_start && start <= active_start).then_some(segment.clone()) + }) + .collect::>(); + relevant.sort_by_key(|segment| segment.started_at.clone()); + if relevant.is_empty() { + if active_start <= requested_start + ChronoDuration::milliseconds(COVERAGE_TOLERANCE_MS) { + return Ok(()); + } + return Err("missing local coverage".to_string()); + } + validate_continuous_audio_coverage(&relevant, requested_start, active_start) +} + +fn earliest_coverage_start(segments: &[ScreenMemorySegmentMetadata]) -> Option> { + segments + .iter() + .filter_map(|segment| parse_time(&segment.started_at).ok()) + .min() +} + +fn relevant_segments( + app: &AppHandle, + requested_start: DateTime, + captured_until: DateTime, +) -> Result, String> { + let mut segments = screen_memory::finalized_segments(app)? + .into_iter() + .filter_map(|segment| { + let start = parse_time(&segment.started_at).ok()?; + let end = parse_time(&segment.ended_at).ok()?; + (end >= requested_start && start <= captured_until).then_some(segment) + }) + .collect::>(); + segments.sort_by_key(|segment| segment.started_at.clone()); + Ok(segments) +} + +fn validate_continuous_audio_coverage( + segments: &[ScreenMemorySegmentMetadata], + requested_start: DateTime, + captured_until: DateTime, +) -> Result<(), String> { + let first = segments.first().ok_or_else(|| { + "Rewind has no finalized local audio for the requested meeting range.".to_string() + })?; + let first_start = parse_time(&first.started_at)?; + if first_start > requested_start + ChronoDuration::milliseconds(COVERAGE_TOLERANCE_MS) { + return Err("Rewind coverage begins after the scheduled meeting start.".to_string()); + } + let mut prior_end = requested_start; + for segment in segments { + if segment.corrupt || segment.exclusion_tainted || !segment.path.exists() { + return Err( + "Rewind found an incomplete or privacy-excluded segment in the requested range." + .to_string(), + ); + } + if segment.capture_mode != RewindCaptureMode::VisualsAudio { + return Err("Part of this meeting was captured without audio.".to_string()); + } + let start = parse_time(&segment.started_at)?; + let end = parse_time(&segment.ended_at)?; + if start > prior_end + ChronoDuration::milliseconds(COVERAGE_TOLERANCE_MS) { + return Err("Rewind has a coverage gap after the meeting started.".to_string()); + } + if end > prior_end { + prior_end = end; + } + } + if prior_end + ChronoDuration::milliseconds(COVERAGE_TOLERANCE_MS) < captured_until { + return Err( + "Rewind could not finalize audio through the moment notes started.".to_string(), + ); + } + Ok(()) +} + +fn reject_graph_gaps( + app: &AppHandle, + requested_start: DateTime, + captured_until: DateTime, +) -> Result<(), String> { + let graph = app.state::(); + let status = graph + .0 + .lock() + .map_err(|error| error.to_string())? + .status_at(Instant::now()) + .map_err(|error| error.to_string())?; + for gap in status.coverage_gaps { + let start = parse_time(&gap.interval.started_at)?; + let end = parse_time(&gap.interval.ended_at)?; + if start < captured_until && end > requested_start { + return Err("Rewind recorded a source coverage gap during this meeting.".to_string()); + } + } + Ok(()) +} + +async fn wait_for_local_indexes( + app: &AppHandle, + segments: &[ScreenMemorySegmentMetadata], +) -> Result<(), String> { + let deadline = Instant::now() + INDEX_WAIT_TIMEOUT; + loop { + let mut pending = false; + for segment in segments { + let status = read_transcript_status(app, &segment.id)?; + match status.state { + ScreenMemoryTranscriptState::Ready => {} + ScreenMemoryTranscriptState::Pending + | ScreenMemoryTranscriptState::Transcribing => pending = true, + ScreenMemoryTranscriptState::Failed => { + return Err(status.error.unwrap_or_else(|| { + "Local transcription failed for part of this meeting.".to_string() + })) + } + ScreenMemoryTranscriptState::Skipped => { + return Err( + "Local transcription was unavailable for part of this meeting.".to_string(), + ) + } + } + } + if !pending { + return Ok(()); + } + if Instant::now() >= deadline { + return Err("The earlier meeting audio is still being indexed locally. Start notes without including history, or try again in a moment.".to_string()); + } + tokio::time::sleep(INDEX_POLL_INTERVAL).await; + } +} + +fn screen_memory_dir(app: &AppHandle) -> Result { + app.path() + .app_data_dir() + .map(|path| path.join("screen-memory")) + .map_err(|error| format!("app data directory unavailable: {error}")) +} + +fn read_transcript_status( + app: &AppHandle, + segment_id: &str, +) -> Result { + let path = screen_memory_dir(app)?.join(format!("{segment_id}.transcript-status.json")); + let bytes = std::fs::read(&path).map_err(|_| { + "The earlier meeting audio has not finished indexing locally yet.".to_string() + })?; + serde_json::from_slice(&bytes) + .map_err(|_| "A local transcript status file is malformed.".to_string()) +} + +fn read_transcript_rows( + app: &AppHandle, + segment_id: &str, +) -> Result, String> { + let path = screen_memory_dir(app)?.join(format!("{segment_id}.transcript.jsonl")); + read_jsonl(&path) +} + +fn read_jsonl Deserialize<'de>>(path: &Path) -> Result, String> { + let file = File::open(path) + .map_err(|_| "A ready local meeting transcript is missing its rows.".to_string())?; + BufReader::new(file) + .lines() + .filter_map(|line| match line { + Ok(line) if line.trim().is_empty() => None, + other => Some(other), + }) + .map(|line| { + let line = line.map_err(|error| format!("local transcript read failed: {error}"))?; + serde_json::from_str(&line) + .map_err(|_| "A local meeting transcript row is malformed.".to_string()) + }) + .collect() +} + +fn map_rows_to_meeting_timeline( + rows: Vec, + scheduled_start: DateTime, + captured_until: DateTime, +) -> Vec { + rows.into_iter() + .filter_map(|row| { + let captured_at = parse_time(&row.captured_at).ok()?; + if captured_at > captured_until + || captured_at < scheduled_start - ChronoDuration::minutes(1) + { + return None; + } + let start_ms = (captured_at - scheduled_start).num_milliseconds().max(0) as u64; + let duration_ms = row.end_ms.saturating_sub(row.start_ms); + let end_ms = start_ms.saturating_add(duration_ms); + let source = match row.source.as_str() { + "system-audio" => "system", + "microphone" => "mic", + _ => return None, + }; + Some(RewindMeetingTranscriptSegment { + start_ms, + end_ms, + text: row.text, + source: source.to_string(), + }) + }) + .collect() +} + +fn parse_time(value: &str) -> Result, String> { + DateTime::parse_from_rfc3339(value) + .map(|time| time.with_timezone(&Utc)) + .map_err(|_| "Rewind found malformed local capture timing metadata.".to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn maps_rows_to_scheduled_meeting_timeline() { + let scheduled = DateTime::parse_from_rfc3339("2026-07-14T12:00:00Z") + .unwrap() + .with_timezone(&Utc); + let rows = vec![ScreenMemoryTranscriptRow { + schema_version: 1, + segment_id: "one".to_string(), + source: "system-audio".to_string(), + captured_at: "2026-07-14T12:00:03Z".to_string(), + start_ms: 3_000, + end_ms: 4_250, + text: "hello".to_string(), + }]; + let mapped = + map_rows_to_meeting_timeline(rows, scheduled, scheduled + ChronoDuration::minutes(1)); + assert_eq!(mapped.len(), 1); + assert_eq!(mapped[0].start_ms, 3_000); + assert_eq!(mapped[0].end_ms, 4_250); + assert_eq!(mapped[0].source, "system"); + } + + #[test] + fn prepared_history_never_absorbs_rows_after_its_fence() { + let scheduled = DateTime::parse_from_rfc3339("2026-07-14T12:00:00Z") + .unwrap() + .with_timezone(&Utc); + let rows = vec![ScreenMemoryTranscriptRow { + schema_version: 1, + segment_id: "replacement-segment".to_string(), + source: "microphone".to_string(), + captured_at: "2026-07-14T12:00:11Z".to_string(), + start_ms: 0, + end_ms: 1_000, + text: "already belongs to live capture".to_string(), + }]; + let mapped = + map_rows_to_meeting_timeline(rows, scheduled, scheduled + ChronoDuration::seconds(10)); + assert!(mapped.is_empty()); + } +} diff --git a/templates/clips/desktop/src-tauri/src/screen_memory.rs b/templates/clips/desktop/src-tauri/src/screen_memory.rs index a2e0817fce..08f49019d9 100644 --- a/templates/clips/desktop/src-tauri/src/screen_memory.rs +++ b/templates/clips/desktop/src-tauri/src/screen_memory.rs @@ -1,10 +1,14 @@ -use crate::config::{FeatureConfig, ScreenMemoryConfig}; +use crate::capture_graph::{ + CaptureConsumer, CaptureGraphError, CaptureGraphState, CaptureSource, CoverageGapReason, +}; +use crate::config::{FeatureConfig, RewindCaptureMode, ScreenMemoryConfig}; use crate::native_screen::{ self, NativeFullscreenBackend, DISK_SPACE_BLOCK_BYTES, MP4_RECORDING_MIME_TYPE, QUICKTIME_RECORDING_MIME_TYPE, }; use chrono::{DateTime, Duration as ChronoDuration, Utc}; use serde::{Deserialize, Serialize}; +use std::collections::{BTreeSet, HashMap, VecDeque}; use std::fs::{File, OpenOptions}; use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; @@ -19,23 +23,74 @@ const SCREEN_MEMORY_EVENTS_JSONL: &str = "events.jsonl"; const MIN_SEGMENT_SECONDS: u64 = 15; const MAX_SEGMENT_SECONDS: u64 = 30 * 60; const MIN_RETENTION_HOURS: u32 = 1; -const MAX_RETENTION_HOURS: u32 = 24 * 30; +const MAX_RETENTION_HOURS: u32 = 24; const MIN_MAX_BYTES: u64 = 100 * 1024 * 1024; const MAX_MAX_BYTES: u64 = 1024 * 1024 * 1024 * 1024; const ROTATOR_TICK: Duration = Duration::from_millis(500); +/// Privacy detection must not inherit the user-facing context sampling cadence. +/// This bounds how long recognized excluded pixels/audio may reach the current +/// logical segment before that entire segment is discarded. +const EXCLUSION_POLL_INTERVAL: Duration = Duration::from_millis(250); +const SCREEN_MEMORY_MCP_NAME: &str = "clips-screen-memory"; static SEGMENT_COUNTER: AtomicU64 = AtomicU64::new(0); #[derive(Default)] pub struct ScreenMemoryState { inner: Mutex, + /// OCR and Whisper indexing share this gate so completed segments never + /// fan out into concurrent CPU/memory-heavy post-processing jobs. + indexing: Mutex<()>, + /// Serializes physical producer stop/start transitions. Callers that hold + /// this lock use `rotate_segment_inner`; ordinary callers use + /// `rotate_segment`, which acquires it for them. + transition: Mutex<()>, } #[derive(Default)] struct ScreenMemoryRuntime { active: Option, worker_stop: Option>, + rewind_lease_id: Option, + exclusion_active: bool, + exclusion_gap: Option, + temporary_audio_consumers: HashMap, last_error: Option, + ocr_queue: VecDeque, + ocr_worker_running: bool, + ocr_active_segment: Option, + ocr_cancelled_segments: BTreeSet, + transcript_queue: VecDeque, + transcript_worker_running: bool, + transcript_active_segment: Option, + transcript_cancelled_segments: BTreeSet, + /// Ephemeral artifact pins. These intentionally do not survive a crash: + /// callers must have copied any artifact they need before relying on it. + segment_pins: HashMap>, +} + +struct ActiveExclusionGap { + started_at: Instant, + sources: Vec, +} + +fn active_exclusion_gap( + started_at: Instant, + capture_mode: RewindCaptureMode, +) -> ActiveExclusionGap { + ActiveExclusionGap { + started_at, + sources: requested_sources(capture_mode), + } +} + +struct TemporaryAudioConsumer { + graph_lease_id: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TemporaryAudioLease { + owner_id: String, } struct ActiveScreenMemorySegment { @@ -47,6 +102,21 @@ struct ActiveScreenMemorySegment { started_at_iso: String, width: Option, height: Option, + capture_mode: RewindCaptureMode, + exclusion_tainted: Arc, + graph_epoch_id: String, + graph_started_elapsed_ms: u64, +} + +#[derive(Clone)] +struct ScreenMemoryOcrJob { + segment: ScreenMemorySegmentMetadata, + sample_interval_seconds: u64, +} + +#[derive(Clone)] +struct ScreenMemoryTranscriptJob { + segment: ScreenMemorySegmentMetadata, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -62,8 +132,55 @@ pub struct ScreenMemorySegmentMetadata { pub width: Option, pub height: Option, pub bytes: u64, + #[serde(default)] + pub system_audio_path: Option, + #[serde(default)] + pub microphone_path: Option, pub corrupt: bool, pub error: Option, + #[serde(default)] + pub capture_mode: RewindCaptureMode, + #[serde(default)] + pub exclusion_tainted: bool, + /// Monotonic offsets from the shared capture graph; wall-clock fields stay + /// for human-facing local files and backwards compatibility. + #[serde(default)] + pub graph_epoch_id: Option, + #[serde(default)] + pub graph_started_elapsed_ms: u64, + #[serde(default)] + pub graph_ended_elapsed_ms: u64, +} + +fn audio_sidecar_path(video_path: &Path, source: &str) -> PathBuf { + let stem = video_path + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or("segment"); + video_path.with_file_name(format!("{stem}.{source}.wav")) +} + +fn existing_audio_sidecars( + video_path: &Path, + mode: RewindCaptureMode, +) -> (Option, Option) { + if mode != RewindCaptureMode::VisualsAudio { + return (None, None); + } + let system = audio_sidecar_path(video_path, "system"); + let microphone = audio_sidecar_path(video_path, "microphone"); + ( + system.exists().then_some(system), + microphone.exists().then_some(microphone), + ) +} + +fn media_unit_bytes(video_path: &Path, system: Option<&Path>, microphone: Option<&Path>) -> u64 { + [Some(video_path), system, microphone] + .into_iter() + .flatten() + .filter_map(|path| std::fs::metadata(path).ok().map(|metadata| metadata.len())) + .fold(0u64, u64::saturating_add) } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -74,6 +191,14 @@ pub struct ScreenMemoryEvent { pub window_title: Option, pub bundle_id: Option, pub source: String, + /// Present only for explicit coverage-gap markers. Kept optional so old + /// event logs remain readable and ordinary context rows stay compact. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub coverage_gap_reason: Option, + /// A small local semantic summary, never a raw accessibility tree. Older + /// events intentionally deserialize without this field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub accessibility: Option, } #[derive(Debug, Clone, Serialize)] @@ -116,6 +241,8 @@ pub struct ScreenMemoryStatus { pub active_segment: Option, pub recent_segments: Vec, pub last_error: Option, + pub exclusion_active: bool, + pub coverage: String, } #[derive(Debug, Clone, Serialize)] @@ -153,6 +280,29 @@ pub fn sync_from_config(app: &AppHandle, feature_config: &FeatureConfig) { return; } + let effective_mode = effective_capture_mode(app, config.capture_mode); + let mode_changed_while_active = app + .try_state::() + .and_then(|state| { + state + .inner + .lock() + .ok() + .and_then(|runtime| runtime.active.as_ref().map(|active| active.capture_mode)) + }) + .is_some_and(|active_mode| active_mode != effective_mode); + if mode_changed_while_active { + // Reconfigure the one physical producer at a segment boundary rather + // than letting its audio contract drift under an existing lease. + // This is deliberately unlike an ordinary rotation: the requested + // source set changed, so retaining the old lease would leave the graph + // claiming the previous mode even after the producer restarts. + if let Err(err) = rotate_segment(app, &config) { + record_error(app, err); + } + return; + } + if let Err(err) = ensure_running(app, &config) { record_error(app, err); } @@ -191,6 +341,8 @@ pub async fn screen_memory_pause(app: AppHandle) -> Result Result Result<(), String> { crate::clips::open_local_recording_folder(dir.to_string_lossy().to_string()) } +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ScreenMemoryAgentConnectionStatus { + pub client: String, + pub configured: bool, + pub config_path: String, + pub store_dir: String, +} + +fn home_dir() -> Result { + std::env::var_os("HOME") + .map(PathBuf::from) + .ok_or_else(|| "Could not resolve the current user's home directory.".to_string()) +} + +fn toml_quote(value: &str) -> String { + format!("\"{}\"", value.replace('\\', "\\\\").replace('"', "\\\"")) +} + +fn codex_screen_memory_block(store_dir: &Path) -> String { + let args = [ + "-y", + "@agent-native/core@latest", + "mcp", + "screen-memory", + "--dir", + &store_dir.to_string_lossy(), + ]; + format!( + "[mcp_servers.\"{SCREEN_MEMORY_MCP_NAME}\"]\ncommand = \"npx\"\nargs = [{}]\n", + args.iter() + .map(|arg| toml_quote(arg)) + .collect::>() + .join(", ") + ) +} + +fn table_header(line: &str) -> Option<&str> { + let trimmed = line.trim(); + if !trimmed.starts_with('[') { + return None; + } + let end = trimmed.find(']')?; + Some(trimmed[1..end].trim()) +} + +fn is_screen_memory_table(header: &str) -> bool { + header == format!("mcp_servers.\"{SCREEN_MEMORY_MCP_NAME}\"") + || header == format!("mcp_servers.{SCREEN_MEMORY_MCP_NAME}") + || header.starts_with(&format!("mcp_servers.\"{SCREEN_MEMORY_MCP_NAME}\".")) + || header.starts_with(&format!("mcp_servers.{SCREEN_MEMORY_MCP_NAME}.")) +} + +fn replace_codex_screen_memory_block(existing: &str, block: &str) -> String { + let lines = existing.lines().collect::>(); + let mut kept = Vec::new(); + let mut index = 0; + while index < lines.len() { + if table_header(lines[index]).is_some_and(is_screen_memory_table) { + index += 1; + while index < lines.len() && table_header(lines[index]).is_none() { + index += 1; + } + continue; + } + kept.push(lines[index]); + index += 1; + } + let base = kept.join("\n").trim_end().to_string(); + if base.is_empty() { + format!("{block}\n") + } else { + format!("{base}\n\n{block}\n") + } +} + +fn write_atomic(path: &Path, contents: &[u8]) -> Result<(), String> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|err| format!("Could not create {}: {err}", parent.display()))?; + } + let temporary = path.with_extension(format!("tmp-{}", std::process::id())); + std::fs::write(&temporary, contents) + .map_err(|err| format!("Could not write {}: {err}", temporary.display()))?; + std::fs::rename(&temporary, path) + .map_err(|err| format!("Could not replace {}: {err}", path.display())) +} + +fn agent_connection_config_path(client: &str) -> Result { + let home = home_dir()?; + match client { + "codex" => Ok(home.join(".codex/config.toml")), + "claude-code" => Ok(home.join(".claude.json")), + _ => Err("Supported agents are Codex and Claude Code.".to_string()), + } +} + +fn install_agent_connection(client: &str, store_dir: &Path) -> Result { + let config_path = agent_connection_config_path(client)?; + if client == "codex" { + let existing = std::fs::read_to_string(&config_path).unwrap_or_default(); + let next = + replace_codex_screen_memory_block(&existing, &codex_screen_memory_block(store_dir)); + write_atomic(&config_path, next.as_bytes())?; + } else { + let mut root = std::fs::read_to_string(&config_path) + .ok() + .and_then(|value| serde_json::from_str::(&value).ok()) + .unwrap_or_else(|| serde_json::json!({})); + let object = root + .as_object_mut() + .ok_or_else(|| format!("{} is not a JSON object.", config_path.display()))?; + let servers = object + .entry("mcpServers") + .or_insert_with(|| serde_json::json!({})) + .as_object_mut() + .ok_or_else(|| "Claude Code mcpServers is not a JSON object.".to_string())?; + servers.insert( + SCREEN_MEMORY_MCP_NAME.to_string(), + serde_json::json!({ + "command": "npx", + "args": [ + "-y", + "@agent-native/core@latest", + "mcp", + "screen-memory", + "--dir", + store_dir.to_string_lossy() + ] + }), + ); + let bytes = serde_json::to_vec_pretty(&root) + .map_err(|err| format!("Could not encode Claude Code configuration: {err}"))?; + write_atomic(&config_path, &bytes)?; + } + Ok(config_path) +} + +#[tauri::command] +pub async fn screen_memory_install_agent_connection( + app: AppHandle, + client: String, +) -> Result { + let store_dir = screen_memory_dir(&app)?; + let config_path = install_agent_connection(&client, &store_dir)?; + Ok(ScreenMemoryAgentConnectionStatus { + client, + configured: true, + config_path: config_path.to_string_lossy().to_string(), + store_dir: store_dir.to_string_lossy().to_string(), + }) +} + #[tauri::command] pub async fn screen_memory_delete( app: AppHandle, @@ -279,9 +586,21 @@ fn normalize_screen_memory_config(mut config: ScreenMemoryConfig) -> ScreenMemor config.sample_interval_seconds = config .sample_interval_seconds .clamp(1, config.segment_seconds); + config.excluded_bundle_ids = normalize_excluded_bundle_ids(config.excluded_bundle_ids); config } +fn normalize_excluded_bundle_ids(bundle_ids: Vec) -> Vec { + let mut normalized = bundle_ids + .into_iter() + .map(|bundle_id| bundle_id.trim().to_ascii_lowercase()) + .filter(|bundle_id| !bundle_id.is_empty()) + .collect::>(); + normalized.sort(); + normalized.dedup(); + normalized +} + fn screen_memory_available() -> bool { #[cfg(target_os = "macos")] { @@ -301,6 +620,12 @@ fn screen_memory_dir(app: &AppHandle) -> Result { .join(SCREEN_MEMORY_DIR); std::fs::create_dir_all(&dir) .map_err(|e| format!("screen memory directory unavailable: {e}"))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)) + .map_err(|e| format!("screen memory directory protection failed: {e}"))?; + } Ok(dir) } @@ -308,10 +633,229 @@ fn screen_memory_events_path(app: &AppHandle) -> Result { Ok(screen_memory_dir(app)?.join(SCREEN_MEMORY_EVENTS_JSONL)) } +fn agent_handoffs_dir(app: &AppHandle) -> Result { + let dir = screen_memory_dir(app)?.join("agent-handoffs"); + std::fs::create_dir_all(&dir) + .map_err(|err| format!("agent handoff directory unavailable: {err}"))?; + Ok(dir) +} + +fn agent_handoff_path(app: &AppHandle, request_id: &str) -> Result { + let safe = sanitize_segment_id(request_id); + if safe != request_id || !safe.starts_with("handoff-") { + return Err("Invalid Rewind handoff request ID.".to_string()); + } + Ok(agent_handoffs_dir(app)?.join(format!("{safe}.json"))) +} + +fn read_agent_handoff(path: &Path) -> Result { + serde_json::from_slice( + &std::fs::read(path).map_err(|err| format!("Could not read {}: {err}", path.display()))?, + ) + .map_err(|err| format!("Could not parse {}: {err}", path.display())) +} + +fn write_agent_handoff(path: &Path, value: &serde_json::Value) -> Result<(), String> { + let bytes = serde_json::to_vec_pretty(value) + .map_err(|err| format!("Could not encode agent handoff: {err}"))?; + write_atomic(path, &bytes) +} + +#[tauri::command] +pub async fn screen_memory_next_agent_handoff( + app: AppHandle, +) -> Result, String> { + let mut pending = Vec::new(); + for entry in std::fs::read_dir(agent_handoffs_dir(&app)?) + .map_err(|err| format!("Could not list agent handoffs: {err}"))? + { + let path = entry.map_err(|err| err.to_string())?.path(); + if path.extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let value = match read_agent_handoff(&path) { + Ok(value) => value, + Err(_) => continue, + }; + if value.get("status").and_then(|value| value.as_str()) == Some("pending") { + let requested_at = value + .get("requestedAt") + .and_then(|value| value.as_str()) + .unwrap_or_default() + .to_string(); + pending.push((requested_at, value)); + } + } + pending.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(pending.into_iter().next().map(|(_, value)| value)) +} + +#[tauri::command] +pub async fn screen_memory_update_agent_handoff( + app: AppHandle, + request_id: String, + status: String, + result: Option, + error: Option, +) -> Result<(), String> { + if !matches!( + status.as_str(), + "processing" | "ready" | "declined" | "failed" + ) { + return Err("Invalid Rewind handoff status.".to_string()); + } + let path = agent_handoff_path(&app, &request_id)?; + let mut value = read_agent_handoff(&path)?; + let object = value + .as_object_mut() + .ok_or_else(|| "Rewind handoff is not an object.".to_string())?; + object.insert("status".to_string(), serde_json::json!(status)); + object.insert("updatedAt".to_string(), serde_json::json!(now_iso())); + scrub_terminal_handoff_fields(object, &status); + if let Some(result) = result.and_then(|value| value.as_object().cloned()) { + for (key, value) in result { + if matches!( + key.as_str(), + "recordingId" | "agentUrl" | "contextUrl" | "expiresAt" | "autoDeleteAt" + ) { + object.insert(key, value); + } + } + } + if let Some(error) = error { + object.insert( + "error".to_string(), + serde_json::json!(error.chars().take(1_000).collect::()), + ); + } + write_agent_handoff(&path, &value) +} + +fn scrub_terminal_handoff_fields( + object: &mut serde_json::Map, + status: &str, +) { + if matches!(status, "ready" | "declined" | "failed") { + // The request reason is useful while the user is reviewing or the + // handoff is processing, but it must not become a second durable + // transcript once the request reaches a terminal state. + object.remove("reason"); + } +} + +#[tauri::command] +pub async fn screen_memory_due_agent_handoffs( + app: AppHandle, +) -> Result, String> { + let now = Utc::now(); + let mut due = Vec::new(); + for entry in std::fs::read_dir(agent_handoffs_dir(&app)?) + .map_err(|err| format!("Could not list agent handoffs: {err}"))? + { + let path = entry.map_err(|err| err.to_string())?.path(); + if path.extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let value = match read_agent_handoff(&path) { + Ok(value) => value, + Err(_) => continue, + }; + if value.get("status").and_then(|value| value.as_str()) != Some("ready") + || value.get("autoDeletedAt").is_some() + { + continue; + } + let Some(auto_delete_at) = value + .get("autoDeleteAt") + .and_then(|value| value.as_str()) + .and_then(|value| DateTime::parse_from_rfc3339(value).ok()) + else { + continue; + }; + if auto_delete_at.with_timezone(&Utc) <= now { + due.push(serde_json::json!({ + "requestId": value.get("requestId"), + "recordingId": value.get("recordingId") + })); + } + } + Ok(due) +} + +#[tauri::command] +pub async fn screen_memory_mark_agent_handoff_deleted( + app: AppHandle, + request_id: String, +) -> Result<(), String> { + let path = agent_handoff_path(&app, &request_id)?; + let mut value = read_agent_handoff(&path)?; + let object = value + .as_object_mut() + .ok_or_else(|| "Rewind handoff is not an object.".to_string())?; + object.insert("status".to_string(), serde_json::json!("deleted")); + object.insert("autoDeletedAt".to_string(), serde_json::json!(now_iso())); + write_agent_handoff(&path, &value) +} + +#[tauri::command] +pub async fn screen_memory_cancel_agent_handoff_cleanup( + app: AppHandle, + request_id: String, +) -> Result<(), String> { + let path = agent_handoff_path(&app, &request_id)?; + let mut value = read_agent_handoff(&path)?; + let object = value + .as_object_mut() + .ok_or_else(|| "Rewind handoff is not an object.".to_string())?; + object.remove("autoDeleteAt"); + object.insert( + "cleanupCanceledAt".to_string(), + serde_json::json!(now_iso()), + ); + object.insert( + "cleanupCanceledReason".to_string(), + serde_json::json!("promoted"), + ); + write_agent_handoff(&path, &value) +} + fn segment_metadata_path(app: &AppHandle, segment_id: &str) -> Result { Ok(screen_memory_dir(app)?.join(format!("{}.json", sanitize_segment_id(segment_id)))) } +fn segment_ocr_rows_path(app: &AppHandle, segment_id: &str) -> Result { + Ok(screen_memory_dir(app)?.join(format!("{}.ocr.jsonl", sanitize_segment_id(segment_id)))) +} + +fn segment_ocr_status_path(app: &AppHandle, segment_id: &str) -> Result { + Ok(screen_memory_dir(app)?.join(format!( + "{}.ocr-status.json", + sanitize_segment_id(segment_id) + ))) +} + +fn segment_transcript_rows_path(app: &AppHandle, segment_id: &str) -> Result { + Ok(transcript_sidecar_paths(&screen_memory_dir(app)?, segment_id).0) +} + +fn segment_transcript_status_path(app: &AppHandle, segment_id: &str) -> Result { + Ok(transcript_sidecar_paths(&screen_memory_dir(app)?, segment_id).1) +} + +fn transcript_sidecar_paths(dir: &Path, segment_id: &str) -> (PathBuf, PathBuf) { + let id = sanitize_segment_id(segment_id); + ( + dir.join(format!("{id}.transcript.jsonl")), + dir.join(format!("{id}.transcript-status.json")), + ) +} + +fn remove_transcript_sidecars(dir: &Path, segment_id: &str) -> Result<(), String> { + let (rows, status) = transcript_sidecar_paths(dir, segment_id); + remove_file_if_exists(&rows)?; + remove_file_if_exists(&status) +} + fn segment_media_path( app: &AppHandle, segment_id: &str, @@ -344,7 +888,7 @@ fn next_segment_id() -> String { fn build_status(app: &AppHandle) -> Result { let config = normalize_screen_memory_config(crate::config::feature_config(app).screen_memory); let storage_dir = screen_memory_dir(app)?; - let (active_segment, last_error) = { + let (active_segment, last_error, exclusion_active) = { let state = app.state::(); let guard = state.inner.lock().map_err(|e| e.to_string())?; ( @@ -361,6 +905,7 @@ fn build_status(app: &AppHandle) -> Result { height: active.height, }), guard.last_error.clone(), + guard.exclusion_active, ) }; let state = if !config.enabled { @@ -373,6 +918,8 @@ fn build_status(app: &AppHandle) -> Result { ScreenMemoryRuntimeState::Idle }; + let producer_active = active_segment.is_some(); + let coverage = coverage_language(&config, exclusion_active, producer_active); Ok(ScreenMemoryStatus { available: screen_memory_available(), state, @@ -381,10 +928,359 @@ fn build_status(app: &AppHandle) -> Result { active_segment, recent_segments: recent_segments(app, Some(5))?, last_error, + exclusion_active, + coverage, }) } +fn coverage_language( + config: &ScreenMemoryConfig, + exclusion_active: bool, + producer_active: bool, +) -> String { + if exclusion_active { + return "Capture stopped for an excluded app. The entire in-flight media segment was discarded, and this interval is recorded as a source coverage gap.".to_string(); + } + if !config.enabled { + return "Rewind capture is disabled. Existing local segments remain available.".to_string(); + } + if config.paused { + return "Rewind capture is paused. Existing local segments remain available, and no new source coverage is being retained.".to_string(); + } + if producer_active { + return "Rewind is retaining local media coverage. Excluded applications stop capture and create explicit source coverage gaps.".to_string(); + } + "Rewind is not currently retaining source coverage.".to_string() +} + +pub(crate) fn rewind_clip_compatible(app: &AppHandle) -> Result { + let status = build_status(app)?; + Ok(status.available + && matches!(status.state, ScreenMemoryRuntimeState::Recording) + && status.last_error.is_none()) +} + +pub(crate) fn rewind_clip_sources(app: &AppHandle) -> Vec { + let capture_mode = app + .try_state::() + .and_then(|state| { + state + .inner + .lock() + .ok() + .and_then(|runtime| runtime.active.as_ref().map(|active| active.capture_mode)) + }) + .unwrap_or_else(|| { + let base_mode = crate::config::feature_config(app) + .screen_memory + .capture_mode; + effective_capture_mode(app, base_mode) + }); + requested_sources(capture_mode) +} + +fn effective_mode( + base_mode: RewindCaptureMode, + temporary_audio_consumers: usize, +) -> RewindCaptureMode { + if temporary_audio_consumers > 0 { + RewindCaptureMode::VisualsAudio + } else { + base_mode + } +} + +fn effective_capture_mode(app: &AppHandle, base_mode: RewindCaptureMode) -> RewindCaptureMode { + let temporary_audio_consumers = app + .try_state::() + .and_then(|state| { + state + .inner + .lock() + .ok() + .map(|runtime| runtime.temporary_audio_consumers.len()) + }) + .unwrap_or(0); + effective_mode(base_mode, temporary_audio_consumers) +} + +fn temporary_audio_lease_available( + enabled: bool, + paused: bool, + producer_active: bool, + include_mic: bool, + include_system_audio: bool, +) -> bool { + enabled && !paused && producer_active && (include_mic || include_system_audio) +} + +/// Temporarily upgrades the active Rewind producer to its audio-capable mode. +/// The persisted Screen Memory mode is never changed. Repeated acquisition by +/// the same stable owner is idempotent; distinct owners refcount the effective +/// audio demand through separate graph leases. +pub(crate) fn acquire_temporary_audio_consumer( + app: &AppHandle, + owner_id: &str, + consumer: CaptureConsumer, + include_mic: bool, + include_system_audio: bool, +) -> Result, String> { + if owner_id.trim().is_empty() { + return Err("temporary audio consumer owner id cannot be empty".into()); + } + let state = app.state::(); + let _transition = state.transition.lock().map_err(|error| error.to_string())?; + let config = normalize_screen_memory_config(crate::config::feature_config(app).screen_memory); + let (mut producer_active, already_present, exclusion_active) = { + let runtime = state.inner.lock().map_err(|error| error.to_string())?; + ( + runtime.active.is_some(), + runtime.temporary_audio_consumers.contains_key(owner_id), + runtime.exclusion_active, + ) + }; + // Config writes become visible before their sync callback completes. If a + // meeting arrives in that narrow window, finish stopping the old producer + // under the same transition lock before returning `None`; the caller can + // then open the legacy recorder without ever overlapping Rewind. + if !config.enabled || config.paused { + if producer_active { + stop_active_segment_inner(app)?; + } + return Ok(None); + } + if exclusion_active { + return Ok(None); + } + if !producer_active { + ensure_running_inner(app, &config)?; + producer_active = true; + } + if already_present { + return Ok(Some(TemporaryAudioLease { + owner_id: owner_id.to_owned(), + })); + } + if !temporary_audio_lease_available( + config.enabled, + config.paused, + producer_active, + include_mic, + include_system_audio, + ) { + return Ok(None); + } + + let mut sources = Vec::new(); + if include_mic { + sources.push(CaptureSource::Microphone); + } + if include_system_audio { + sources.push(CaptureSource::SystemAudio); + } + let graph_lease = app + .state::() + .0 + .lock() + .map_err(|error| error.to_string())? + .start_consumer(consumer, sources) + .map_err(capture_graph_error)?; + { + let mut runtime = state.inner.lock().map_err(|error| error.to_string())?; + runtime.temporary_audio_consumers.insert( + owner_id.to_owned(), + TemporaryAudioConsumer { + graph_lease_id: graph_lease.id.clone(), + }, + ); + } + + let needs_upgrade = state + .inner + .lock() + .map_err(|error| error.to_string())? + .active + .as_ref() + .is_some_and(|active| active.capture_mode != RewindCaptureMode::VisualsAudio); + if needs_upgrade { + if let Err(error) = rotate_segment_inner(app, &config) { + rollback_temporary_audio_consumer(app, owner_id); + return Err(format!("rewind-audio-upgrade-failed: {error}")); + } + } + Ok(Some(TemporaryAudioLease { + owner_id: owner_id.to_owned(), + })) +} + +pub(crate) fn release_temporary_audio_consumer( + app: &AppHandle, + lease: TemporaryAudioLease, +) -> Result<(), String> { + let state = app.state::(); + let _transition = state.transition.lock().map_err(|error| error.to_string())?; + let Some(consumer) = state + .inner + .lock() + .map_err(|error| error.to_string())? + .temporary_audio_consumers + .remove(&lease.owner_id) + else { + return Ok(()); + }; + end_graph_lease(app, &consumer.graph_lease_id); + + let config = normalize_screen_memory_config(crate::config::feature_config(app).screen_memory); + if !config.enabled || config.paused { + return Ok(()); + } + let desired = effective_capture_mode(app, config.capture_mode); + let needs_restore = state + .inner + .lock() + .map_err(|error| error.to_string())? + .active + .as_ref() + .is_some_and(|active| active.capture_mode != desired); + if needs_restore { + rotate_segment_inner(app, &config)?; + } + Ok(()) +} + +fn rollback_temporary_audio_consumer(app: &AppHandle, owner_id: &str) { + let consumer = app + .state::() + .inner + .lock() + .ok() + .and_then(|mut runtime| runtime.temporary_audio_consumers.remove(owner_id)); + if let Some(consumer) = consumer { + end_graph_lease(app, &consumer.graph_lease_id); + } +} + +fn end_graph_lease(app: &AppHandle, lease_id: &str) { + if let Ok(mut graph) = app.state::().0.lock() { + if let Err(error) = graph.end_consumer(lease_id) { + eprintln!("[clips-tray] temporary audio graph lease close failed: {error}"); + } + } +} + +/// Fence the rolling producer at an exact media-fragment edge and persist the +/// closed logical segment while keeping the replacement capture active. +pub(crate) fn fence_active_for_clip( + app: &AppHandle, +) -> Result { + let state = app.state::(); + let _transition = state.transition.lock().map_err(|error| error.to_string())?; + let active = { + let active = state + .inner + .lock() + .map_err(|error| error.to_string())? + .active + .take(); + active + } + .ok_or_else(|| "rewind-not-compatible: Screen Memory is not recording".to_string())?; + if active.exclusion_tainted.load(Ordering::SeqCst) { + let gap = active_exclusion_gap(active.started_at, active.capture_mode); + let segment_id = active.id.clone(); + { + let mut runtime = state.inner.lock().map_err(|error| error.to_string())?; + runtime.exclusion_active = true; + runtime.exclusion_gap = Some(gap); + } + discard_active_segment(active); + discard_segment_artifacts(app, &segment_id)?; + end_rewind_lease(app); + let _ = app.emit(SCREEN_MEMORY_EVENT, ()); + return Err( + "rewind-not-compatible: recognized privacy exclusion created a coverage gap".into(), + ); + } + match fence_rotate_segment(app, active) { + Ok((segment, next)) => { + write_segment_metadata(app, &segment)?; + install_rotated_segment(app, next)?; + Ok(segment) + } + Err(active) => { + if let Ok(mut runtime) = app.state::().inner.lock() { + runtime.active = Some(active); + } + Err("rewind-not-compatible: active Screen Memory backend cannot fence".into()) + } + } +} + +/// Prepare an independent Clip artifact writer on the already-running Rewind +/// producer. The transition lock keeps the backend stable while the writer is +/// installed; no samples enter the Clip until its handle is activated at zero. +#[cfg(target_os = "macos")] +pub(crate) fn prepare_shared_clip_sink( + app: &AppHandle, + path: PathBuf, + include_mic: bool, + include_system_audio: bool, + has_camera: bool, + upload: native_screen::ClipLiveUploadConfig, +) -> Result { + let state = app.state::(); + let _transition = state.transition.lock().map_err(|error| error.to_string())?; + let runtime = state.inner.lock().map_err(|error| error.to_string())?; + let active = runtime + .active + .as_ref() + .ok_or_else(|| "rewind-not-compatible: Screen Memory is not recording".to_string())?; + if active.exclusion_tainted.load(Ordering::SeqCst) { + return Err(not_compatible_clip_sink( + "recognized privacy exclusion created a coverage gap", + )); + } + let width = active + .width + .ok_or_else(|| not_compatible_clip_sink("Screen Memory width is unavailable"))?; + let height = active + .height + .ok_or_else(|| not_compatible_clip_sink("Screen Memory height is unavailable"))?; + native_screen::prepare_shared_clip_sink( + app, + &active.backend, + native_screen::SharedClipSinkRequest { + path, + width, + height, + include_mic, + include_system_audio, + has_camera, + upload, + }, + ) +} + +#[cfg(target_os = "macos")] +fn not_compatible_clip_sink(detail: &str) -> String { + format!("rewind-not-compatible: {detail}") +} + fn ensure_running(app: &AppHandle, config: &ScreenMemoryConfig) -> Result<(), String> { + let state = app.state::(); + let _transition = state.transition.lock().map_err(|error| error.to_string())?; + ensure_running_inner(app, config) +} + +fn ensure_running_inner(app: &AppHandle, config: &ScreenMemoryConfig) -> Result<(), String> { + // An incompatible ordinary Clip owns the physical capture graph until its + // final suspension lease releases. Config sync and temporary audio demand + // must not silently restart Rewind in the middle of that handoff. + if crate::rewind_capture_suspension::is_active(app) { + return Ok(()); + } + let mut config = config.clone(); + config.capture_mode = effective_capture_mode(app, config.capture_mode); if !screen_memory_available() { return Err("Screen Memory capture is currently macOS-only.".to_string()); } @@ -392,12 +1288,12 @@ fn ensure_running(app: &AppHandle, config: &ScreenMemoryConfig) -> Result<(), St { let state = app.state::(); let guard = state.inner.lock().map_err(|e| e.to_string())?; - if guard.active.is_some() { + if guard.active.is_some() || guard.exclusion_active { return Ok(()); } } - let active = start_new_segment(app)?; + let active = start_new_segment(app, config.capture_mode)?; let stop = Arc::new(AtomicBool::new(false)); { let state = app.state::(); @@ -412,8 +1308,13 @@ fn ensure_running(app: &AppHandle, config: &ScreenMemoryConfig) -> Result<(), St guard.last_error = None; } + if let Err(err) = start_rewind_lease(app, config.capture_mode) { + let _ = stop_active_segment_inner(app); + return Err(err); + } + spawn_rotator(app.clone(), stop); - prune_segments(app, config)?; + prune_segments(app, &config)?; let _ = app.emit(SCREEN_MEMORY_EVENT, ()); Ok(()) } @@ -422,8 +1323,10 @@ fn spawn_rotator(app: AppHandle, stop: Arc) { std::thread::spawn(move || loop { let config = normalize_screen_memory_config(crate::config::feature_config(&app).screen_memory); - if wait_for_rotation(&app, &stop, &config) { - return; + match wait_for_rotation(&app, &stop, &config) { + RotationWait::Stop => return, + RotationWait::Resumed => continue, + RotationWait::Rotate => {} } let config = normalize_screen_memory_config(crate::config::feature_config(&app).screen_memory); @@ -436,49 +1339,377 @@ fn spawn_rotator(app: AppHandle, stop: Arc) { }); } -fn wait_for_rotation(app: &AppHandle, stop: &AtomicBool, config: &ScreenMemoryConfig) -> bool { +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RotationWait { + Stop, + Rotate, + Resumed, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ExclusionTransition { + KeepCapturing, + Suspend, + StaySuspended, + Resume, +} + +fn exclusion_transition(exclusion_active: bool, recognized: bool) -> ExclusionTransition { + match (exclusion_active, recognized) { + (false, false) => ExclusionTransition::KeepCapturing, + (false, true) => ExclusionTransition::Suspend, + (true, true) => ExclusionTransition::StaySuspended, + (true, false) => ExclusionTransition::Resume, + } +} + +fn exclusion_resume_allowed( + enabled: bool, + paused: bool, + stop_requested: bool, + recognized: bool, + exclusion_active: bool, + producer_active: bool, +) -> bool { + enabled && !paused && !stop_requested && !recognized && exclusion_active && !producer_active +} + +fn wait_for_rotation( + app: &AppHandle, + stop: &AtomicBool, + config: &ScreenMemoryConfig, +) -> RotationWait { let deadline = Instant::now() + Duration::from_secs(config.segment_seconds); - let sample_interval = Duration::from_secs(config.sample_interval_seconds.max(1)); let mut next_sample = Instant::now(); + let mut next_exclusion_poll = Instant::now(); loop { if stop.load(Ordering::Relaxed) { - return true; + return RotationWait::Stop; } if Instant::now() >= deadline { - return false; + return RotationWait::Rotate; } - if Instant::now() >= next_sample { - append_event(app, sample_active_window()); - next_sample = Instant::now() + sample_interval; - } - let remaining = deadline.saturating_duration_since(Instant::now()); + let now = Instant::now(); + if now >= next_exclusion_poll { + // Exclusion edits are privacy controls, so they must take effect on + // the next 250 ms poll rather than waiting for the current media + // segment (up to five minutes by default) to rotate. + let live_config = + normalize_screen_memory_config(crate::config::feature_config(app).screen_memory); + let (event, exclusion_reason) = sample_active_window(&live_config); + if exclusion_transition(false, exclusion_reason.is_some()) + == ExclusionTransition::Suspend + { + // A concurrent stop/fence that wins the transition lock must + // still fail closed after recognition. + mark_active_segment_exclusion_tainted(app); + append_event( + app, + coverage_gap_event(exclusion_reason.unwrap_or("privacy-exclusion")), + ); + if let Err(error) = suspend_for_exclusion(app) { + record_error(app, error); + } + if wait_for_exclusion_to_clear(app, stop) { + return RotationWait::Stop; + } + // A resumed producer begins a fresh logical segment and gets a + // fresh rotation deadline in the outer worker loop. + return RotationWait::Resumed; + } + if now >= next_sample { + append_event(app, event); + next_sample = now + Duration::from_secs(live_config.sample_interval_seconds.max(1)); + } + next_exclusion_poll = now + EXCLUSION_POLL_INTERVAL; + } + let remaining = deadline.saturating_duration_since(Instant::now()); let until_sample = next_sample.saturating_duration_since(Instant::now()); - std::thread::sleep(remaining.min(until_sample).min(ROTATOR_TICK)); + let until_exclusion = next_exclusion_poll.saturating_duration_since(Instant::now()); + std::thread::sleep( + remaining + .min(until_sample) + .min(until_exclusion) + .min(ROTATOR_TICK), + ); } } -fn sample_active_window() -> ScreenMemoryEvent { +fn mark_active_segment_exclusion_tainted(app: &AppHandle) { + if let Some(state) = app.try_state::() { + if let Ok(runtime) = state.inner.lock() { + if let Some(active) = runtime.active.as_ref() { + active.exclusion_tainted.store(true, Ordering::SeqCst); + } + } + } +} + +fn sample_active_window(config: &ScreenMemoryConfig) -> (ScreenMemoryEvent, Option<&'static str>) { #[cfg(target_os = "macos")] { - let context = crate::accessibility::macos::active_window_context_impl(); - ScreenMemoryEvent { - captured_at: now_iso(), - app_name: context.app_name, - window_title: context.window_title, - bundle_id: context.bundle_id, - source: context.source, + let (context, owner_pid) = + crate::accessibility::macos::active_window_context_with_pid_impl(); + if let Some(reason) = event_exclusion_reason( + config, + context.bundle_id.as_deref(), + context.window_title.as_deref(), + ) { + return (coverage_gap_event(reason), Some(reason)); } + ( + ScreenMemoryEvent { + captured_at: now_iso(), + app_name: context.app_name, + window_title: context.window_title, + bundle_id: context.bundle_id, + source: context.source, + coverage_gap_reason: None, + // Exclusion was checked above; this silent AX read never + // prompts and degrades to None when macOS denies/unreadable. + accessibility: owner_pid + .and_then(crate::accessibility::macos::semantic_fingerprint_for_pid_impl), + }, + None, + ) } #[cfg(not(target_os = "macos"))] { - ScreenMemoryEvent { - captured_at: now_iso(), - app_name: None, - window_title: None, - bundle_id: None, - source: "unsupported".to_string(), + ( + ScreenMemoryEvent { + captured_at: now_iso(), + app_name: None, + window_title: None, + bundle_id: None, + source: "unsupported".to_string(), + coverage_gap_reason: None, + accessibility: None, + }, + None, + ) + } +} + +fn event_exclusion_reason( + config: &ScreenMemoryConfig, + bundle_id: Option<&str>, + window_title: Option<&str>, +) -> Option<&'static str> { + let normalized_bundle_id = bundle_id.map(|value| value.trim().to_ascii_lowercase()); + if normalized_bundle_id.as_deref().is_some_and(|bundle_id| { + config + .excluded_bundle_ids + .iter() + .any(|excluded| excluded.eq_ignore_ascii_case(bundle_id)) + }) { + return Some("excluded-bundle-id"); + } + + if config.exclude_private_windows + && window_title.is_some_and(|title| { + let title = title.to_ascii_lowercase(); + [ + "incognito", + "inprivate", + "private browsing", + "private window", + ] + .iter() + .any(|marker| title.contains(marker)) + }) + { + return Some("excluded-private-window"); + } + + None +} + +fn coverage_gap_event(reason: &str) -> ScreenMemoryEvent { + ScreenMemoryEvent { + captured_at: now_iso(), + app_name: None, + window_title: None, + bundle_id: None, + source: "coverage-gap".to_string(), + coverage_gap_reason: Some(reason.to_string()), + accessibility: None, + } +} + +/// Stop the one physical producer and erase the complete logical segment that +/// may contain excluded media. The rotator worker deliberately remains alive: +/// while this state is active it does nothing except poll the recognized-window +/// signal until a safe replacement can be started under the transition lock. +fn suspend_for_exclusion(app: &AppHandle) -> Result<(), String> { + let state = app.state::(); + let _transition = state.transition.lock().map_err(|error| error.to_string())?; + let active = { + let mut runtime = state.inner.lock().map_err(|error| error.to_string())?; + if runtime.exclusion_active { + return Ok(()); } + let worker_running = runtime + .worker_stop + .as_ref() + .is_some_and(|stop| !stop.load(Ordering::Relaxed)); + let config = crate::config::feature_config(app).screen_memory; + if !worker_running || !config.enabled || config.paused { + return Ok(()); + } + let active = runtime.active.take(); + runtime.exclusion_active = true; + runtime.exclusion_gap = Some(match active.as_ref() { + Some(active) => active_exclusion_gap( + // The whole segment is discarded, so coverage ended when that + // segment began—not merely when the excluded window was noticed. + active.started_at, + active.capture_mode, + ), + None => active_exclusion_gap( + Instant::now(), + effective_mode(config.capture_mode, runtime.temporary_audio_consumers.len()), + ), + }); + active + }; + + if let Some(active) = active { + let segment_id = active.id.clone(); + discard_active_segment(active); + discard_segment_artifacts(app, &segment_id)?; } + end_rewind_lease(app); + let _ = app.emit(SCREEN_MEMORY_EVENT, ()); + Ok(()) +} + +fn wait_for_exclusion_to_clear(app: &AppHandle, stop: &AtomicBool) -> bool { + loop { + if stop.load(Ordering::Relaxed) { + return true; + } + let config = + normalize_screen_memory_config(crate::config::feature_config(app).screen_memory); + if !config.enabled || config.paused { + return true; + } + let recognized = sample_active_window(&config).1.is_some(); + if exclusion_transition(true, recognized) == ExclusionTransition::Resume { + match resume_after_exclusion(app, &config, stop) { + Ok(true) => return false, + Ok(false) => {} + Err(error) => record_error(app, error), + } + } + std::thread::sleep(EXCLUSION_POLL_INTERVAL); + } +} + +/// Returns true only after one replacement producer is installed. A second +/// recognition check is made while holding the producer transition lock so a +/// config callback, temporary-audio request, or stop cannot race a second start. +fn resume_after_exclusion( + app: &AppHandle, + _config: &ScreenMemoryConfig, + stop: &AtomicBool, +) -> Result { + let state = app.state::(); + let _transition = state.transition.lock().map_err(|error| error.to_string())?; + if crate::rewind_capture_suspension::is_active(app) { + return Ok(false); + } + let current = normalize_screen_memory_config(crate::config::feature_config(app).screen_memory); + let recognized = sample_active_window(¤t).1.is_some(); + let (exclusion_active, producer_active) = { + let runtime = state.inner.lock().map_err(|error| error.to_string())?; + (runtime.exclusion_active, runtime.active.is_some()) + }; + if !exclusion_resume_allowed( + current.enabled, + current.paused, + stop.load(Ordering::Relaxed), + recognized, + exclusion_active, + producer_active, + ) { + return Ok(producer_active); + } + + let mode = effective_capture_mode(app, current.capture_mode); + let next = start_new_segment(app, mode)?; + if let Err(error) = install_rotated_segment(app, next) { + let failed = state + .inner + .lock() + .ok() + .and_then(|mut runtime| runtime.active.take()); + if let Some(failed) = failed { + discard_active_segment(failed); + } + end_rewind_lease(app); + return Err(error); + } + + let gap = { + let mut runtime = state.inner.lock().map_err(|error| error.to_string())?; + runtime.exclusion_active = false; + runtime.exclusion_gap.take() + }; + if let Some(gap) = gap { + record_source_coverage_gap(app, gap, Instant::now()); + } + let _ = app.emit(SCREEN_MEMORY_EVENT, ()); + Ok(true) +} + +fn record_source_coverage_gap(app: &AppHandle, gap: ActiveExclusionGap, ended_at: Instant) { + let graph_state = app.state::(); + let Ok(mut graph) = graph_state.0.lock() else { + return; + }; + for source in gap.sources { + if let Err(error) = graph.record_coverage_gap_at( + source, + CoverageGapReason::PrivacyExclusion, + gap.started_at, + ended_at, + ) { + eprintln!("[clips-tray] exclusion coverage gap record failed: {error}"); + } + } +} + +fn discard_segment_artifacts(app: &AppHandle, segment_id: &str) -> Result<(), String> { + let state = app.state::(); + { + let mut runtime = state.inner.lock().map_err(|error| error.to_string())?; + runtime.ocr_queue.retain(|job| job.segment.id != segment_id); + runtime + .ocr_cancelled_segments + .insert(segment_id.to_string()); + runtime + .transcript_queue + .retain(|job| job.segment.id != segment_id); + runtime + .transcript_cancelled_segments + .insert(segment_id.to_string()); + } + let dir = screen_memory_dir(app)?; + let (transcript_rows, transcript_status) = transcript_sidecar_paths(&dir, segment_id); + remove_discarded_sidecars(&[ + segment_metadata_path(app, segment_id)?, + segment_ocr_rows_path(app, segment_id)?, + segment_ocr_status_path(app, segment_id)?, + transcript_rows, + transcript_status, + ]) +} + +fn remove_discarded_sidecars(paths: &[PathBuf]) -> Result<(), String> { + for path in paths { + remove_file_if_exists(path)?; + } + Ok(()) } fn append_event(app: &AppHandle, event: ScreenMemoryEvent) { @@ -501,6 +1732,14 @@ fn append_event_inner(app: &AppHandle, event: &ScreenMemoryEvent) -> Result<(), } fn rotate_segment(app: &AppHandle, config: &ScreenMemoryConfig) -> Result<(), String> { + let state = app.state::(); + let _transition = state.transition.lock().map_err(|error| error.to_string())?; + rotate_segment_inner(app, config) +} + +fn rotate_segment_inner(app: &AppHandle, config: &ScreenMemoryConfig) -> Result<(), String> { + let mut config = config.clone(); + config.capture_mode = effective_capture_mode(app, config.capture_mode); let active = { let state = app.state::(); let mut guard = state.inner.lock().map_err(|e| e.to_string())?; @@ -508,18 +1747,96 @@ fn rotate_segment(app: &AppHandle, config: &ScreenMemoryConfig) -> Result<(), St }; if let Some(active) = active { - match finalize_active_segment(active) { - Ok(segment) => { - write_segment_metadata(app, &segment)?; - prune_segments(app, config)?; + if active.exclusion_tainted.load(Ordering::SeqCst) { + let gap = active_exclusion_gap(active.started_at, active.capture_mode); + let segment_id = active.id.clone(); + { + let state = app.state::(); + let mut runtime = state.inner.lock().map_err(|error| error.to_string())?; + runtime.exclusion_active = true; + runtime.exclusion_gap = Some(gap); + } + discard_active_segment(active); + discard_segment_artifacts(app, &segment_id)?; + end_rewind_lease(app); + let _ = app.emit(SCREEN_MEMORY_EVENT, ()); + return Ok(()); + } + // A normal rotation on the segmented custom SCK backend only fences + // the fMP4 writer. The stream and physical producer stay alive; a + // mode change must still stop/restart because its source contract has + // changed. + if active.capture_mode == config.capture_mode { + match fence_rotate_segment(app, active) { + Ok((segment, next)) => { + // The physical stream is already writing the next file. + // Do not let an auxiliary metadata/OCR failure drop it. + if let Err(error) = write_segment_metadata(app, &segment) { + record_error(app, error); + } else { + enqueue_segment_ocr(app, segment.clone(), config.sample_interval_seconds); + if let Err(error) = prune_segments(app, &config) { + record_error(app, error); + } + } + return install_rotated_segment(app, next); + } + Err(active) => { + // Non-segmented fallback and failed fences retain the + // old stop/start behavior. A failed fence never creates + // metadata for a path that has not closed. + match finalize_active_segment(app, active) { + Ok(segment) => { + write_segment_metadata(app, &segment)?; + enqueue_segment_ocr( + app, + segment.clone(), + config.sample_interval_seconds, + ); + prune_segments(app, &config)?; + } + Err(err) => { + record_error(app, err); + end_rewind_lease(app); + } + } + } } - Err(err) => { - record_error(app, err); + } else { + // The current graph lease must describe the actual producer source + // set. End it before the safe stop/start mode transition; + // `install_rotated_segment` starts the replacement lease from the + // new effective mode. + end_rewind_lease(app); + match finalize_active_segment(app, active) { + Ok(segment) => { + write_segment_metadata(app, &segment)?; + enqueue_segment_ocr(app, segment.clone(), config.sample_interval_seconds); + prune_segments(app, &config)?; + } + Err(err) => { + record_error(app, err); + // The rolling producer has reported an error; its lease must + // not imply uninterrupted coverage across the replacement. + end_rewind_lease(app); + } } } } - let mut next = Some(start_new_segment(app)?); + let next = match start_new_segment(app, config.capture_mode) { + Ok(next) => next, + Err(err) => { + end_rewind_lease(app); + return Err(err); + } + }; + install_rotated_segment(app, next) +} + +fn install_rotated_segment(app: &AppHandle, next: ActiveScreenMemorySegment) -> Result<(), String> { + let capture_mode = next.capture_mode; + let mut next = Some(next); let installed = { let state = app.state::(); let mut guard = state.inner.lock().map_err(|e| e.to_string())?; @@ -542,28 +1859,155 @@ fn rotate_segment(app: &AppHandle, config: &ScreenMemoryConfig) -> Result<(), St } } + if installed { + let needs_lease = app + .state::() + .inner + .lock() + .ok() + .is_some_and(|runtime| runtime.rewind_lease_id.is_none()); + if needs_lease { + start_rewind_lease(app, capture_mode)?; + } + } + let _ = app.emit(SCREEN_MEMORY_EVENT, ()); Ok(()) } +/// Returns the fenced metadata plus the replacement logical segment while +/// preserving the one physical custom SCK backend. On unsupported/failing +/// backends it returns the untouched active segment for stop/start fallback. +fn fence_rotate_segment( + app: &AppHandle, + active: ActiveScreenMemorySegment, +) -> Result<(ScreenMemorySegmentMetadata, ActiveScreenMemorySegment), ActiveScreenMemorySegment> { + #[cfg(not(target_os = "macos"))] + { + let _ = app; + Err(active) + } + #[cfg(target_os = "macos")] + { + let next_id = next_segment_id(); + let Ok(next_path) = segment_media_path(app, &next_id, "mp4") else { + return Err(active); + }; + let Ok(fence) = active.backend.request_fragment_fence(next_path.clone()) else { + return Err(active); + }; + let Ok(closed) = + NativeFullscreenBackend::await_fragment_fence(fence, Duration::from_secs(15)) + else { + return Err(active); + }; + let boundary = Instant::now(); + let Ok(segment) = finalize_fenced_segment(&active, closed.path, boundary) else { + return Err(active); + }; + let next = ActiveScreenMemorySegment { + id: next_id, + path: next_path, + mime_type: MP4_RECORDING_MIME_TYPE, + backend: active.backend, + started_at: boundary, + started_at_iso: now_iso(), + width: active.width, + height: active.height, + capture_mode: active.capture_mode, + exclusion_tainted: Arc::new(AtomicBool::new(false)), + graph_epoch_id: active.graph_epoch_id, + graph_started_elapsed_ms: graph_elapsed_ms(app, boundary), + }; + Ok((segment, next)) + } +} + fn stop_active_segment(app: &AppHandle) -> Result, String> { - let active = { + let state = app.state::(); + let _transition = state.transition.lock().map_err(|error| error.to_string())?; + stop_active_segment_inner(app) +} + +/// Temporarily yields the one physical producer to an ordinary recorder while +/// leaving the persisted Rewind enabled/paused setting untouched. An active +/// privacy exclusion is already physically suspended and remains authoritative. +pub(crate) fn suspend_physical_capture(app: &AppHandle) -> Result<(), String> { + let state = app.state::(); + let _transition = state.transition.lock().map_err(|error| error.to_string())?; + let exclusion_active = state + .inner + .lock() + .map_err(|error| error.to_string())? + .exclusion_active; + if !exclusion_active { + stop_active_segment_inner(app)?; + } + Ok(()) +} + +/// Restores Rewind after an ordinary recorder releases the producer. This is a +/// no-op while disabled, paused, or privacy-excluded and therefore cannot start +/// a competing producer or bypass the exclusion poller. +pub(crate) fn resume_physical_capture(app: &AppHandle) -> Result<(), String> { + let state = app.state::(); + let _transition = state.transition.lock().map_err(|error| error.to_string())?; + let config = normalize_screen_memory_config(crate::config::feature_config(app).screen_memory); + let exclusion_active = state + .inner + .lock() + .map_err(|error| error.to_string())? + .exclusion_active; + if config.enabled && !config.paused && !exclusion_active { + ensure_running_inner(app, &config)?; + } + Ok(()) +} + +fn stop_active_segment_inner( + app: &AppHandle, +) -> Result, String> { + let (active, exclusion_gap) = { let state = app.state::(); let mut guard = state.inner.lock().map_err(|e| e.to_string())?; if let Some(stop) = guard.worker_stop.take() { stop.store(true, Ordering::Relaxed); } - guard.active.take() + guard.exclusion_active = false; + (guard.active.take(), guard.exclusion_gap.take()) }; + if let Some(gap) = exclusion_gap { + record_source_coverage_gap(app, gap, Instant::now()); + } + let Some(active) = active else { + end_rewind_lease(app); + let _ = app.emit(SCREEN_MEMORY_EVENT, ()); return Ok(None); }; - let segment = finalize_active_segment(active)?; - write_segment_metadata(app, &segment)?; - let config = normalize_screen_memory_config(crate::config::feature_config(app).screen_memory); - prune_segments(app, &config)?; + if active.exclusion_tainted.load(Ordering::SeqCst) { + let gap = active_exclusion_gap(active.started_at, active.capture_mode); + let segment_id = active.id.clone(); + discard_active_segment(active); + discard_segment_artifacts(app, &segment_id)?; + record_source_coverage_gap(app, gap, Instant::now()); + end_rewind_lease(app); + let _ = app.emit(SCREEN_MEMORY_EVENT, ()); + return Ok(None); + } + + let result = finalize_active_segment(app, active).and_then(|segment| { + write_segment_metadata(app, &segment)?; + let config = + normalize_screen_memory_config(crate::config::feature_config(app).screen_memory); + enqueue_segment_ocr(app, segment.clone(), config.sample_interval_seconds); + prune_segments(app, &config)?; + Ok(segment) + }); + end_rewind_lease(app); + let segment = result?; let _ = app.emit(SCREEN_MEMORY_EVENT, ()); Ok(Some(segment)) } @@ -584,7 +2028,10 @@ fn stop_active_segment_if_matches(app: &AppHandle, segment_id: &str) -> Result<( Ok(()) } -fn start_new_segment(app: &AppHandle) -> Result { +fn start_new_segment( + app: &AppHandle, + capture_mode: RewindCaptureMode, +) -> Result { #[cfg(not(target_os = "macos"))] { let _ = app; @@ -610,10 +2057,12 @@ fn start_new_segment(app: &AppHandle) -> Result Result { + if visuals_audio { + record_requested_source_gaps(app, capture_mode); + let _ = std::fs::remove_file(&mp4_path); + return Err(format!( + "Screen Memory visuals + audio could not start its required ScreenCaptureKit screen, system-audio, and microphone sources: {sck_err}" + )); + } eprintln!( "[clips-tray] Screen Memory ScreenCaptureKit unavailable; falling back to screencapture: {sck_err}" ); @@ -658,22 +2118,109 @@ fn start_new_segment(app: &AppHandle) -> Result Vec { + match capture_mode { + RewindCaptureMode::Visuals => vec![CaptureSource::Screen], + RewindCaptureMode::VisualsAudio => vec![ + CaptureSource::Screen, + CaptureSource::SystemAudio, + CaptureSource::Microphone, + ], + } +} + +fn start_rewind_lease(app: &AppHandle, capture_mode: RewindCaptureMode) -> Result<(), String> { + let lease = { + let graph = app.state::(); + let mut graph = graph.0.lock().map_err(|err| err.to_string())?; + graph + .start_consumer(CaptureConsumer::Rewind, requested_sources(capture_mode)) + .map_err(capture_graph_error)? + }; + let state = app.state::(); + let mut runtime = state.inner.lock().map_err(|err| err.to_string())?; + runtime.rewind_lease_id = Some(lease.id); + Ok(()) +} + +fn end_rewind_lease(app: &AppHandle) { + let lease_id = app + .state::() + .inner + .lock() + .ok() + .and_then(|mut runtime| runtime.rewind_lease_id.take()); + if let Some(lease_id) = lease_id { + let graph_state = app.state::(); + if let Ok(mut graph) = graph_state.0.lock() { + if let Err(err) = graph.end_consumer(&lease_id) { + eprintln!("[clips-tray] Screen Memory Rewind lease close failed: {err}"); + } + }; + } +} + +fn record_requested_source_gaps(app: &AppHandle, capture_mode: RewindCaptureMode) { + let now = Instant::now(); + let graph_state = app.state::(); + let Ok(mut graph) = graph_state.0.lock() else { + return; + }; + for source in requested_sources(capture_mode) { + if let Err(err) = + graph.record_coverage_gap_at(source, CoverageGapReason::ProducerUnavailable, now, now) + { + eprintln!("[clips-tray] Screen Memory coverage gap record failed: {err}"); + } + } +} + +fn capture_graph_error(error: CaptureGraphError) -> String { + format!("Screen Memory capture graph error: {error}") +} + +fn graph_elapsed_ms(app: &AppHandle, instant: Instant) -> u64 { + app.try_state::() + .and_then(|state| { + state + .0 + .lock() + .ok() + .and_then(|graph| graph.status_at(instant).ok()) + }) + .map(|status| status.graph_elapsed_ms) + .unwrap_or_default() +} + +fn graph_epoch_id(app: &AppHandle) -> String { + app.try_state::() + .and_then(|state| state.0.lock().ok().map(|graph| graph.epoch_id())) + .unwrap_or_default() +} + fn discard_active_segment(mut active: ActiveScreenMemorySegment) { let _ = native_screen::stop_native_recording(&mut active.backend, false); - let _ = std::fs::remove_file(active.path); + let _ = std::fs::remove_file(&active.path); + let _ = std::fs::remove_file(audio_sidecar_path(&active.path, "system")); + let _ = std::fs::remove_file(audio_sidecar_path(&active.path, "microphone")); } fn finalize_active_segment( + app: &AppHandle, mut active: ActiveScreenMemorySegment, ) -> Result { let stop_error = native_screen::stop_native_recording(&mut active.backend, true).err(); let ended_at = now_iso(); let duration_ms = active.started_at.elapsed().as_millis(); - let bytes = std::fs::metadata(&active.path) + let video_bytes = std::fs::metadata(&active.path) .map_err(|e| { let suffix = stop_error .as_ref() @@ -682,10 +2229,20 @@ fn finalize_active_segment( format!("Screen Memory segment missing{suffix}: {e}") })? .len(); - if bytes == 0 { + if video_bytes == 0 { let _ = std::fs::remove_file(&active.path); - return Err("Screen Memory segment produced an empty file.".to_string()); + return Err(stop_error.unwrap_or_else(|| { + "Screen Memory segment produced an empty file with no more specific backend error." + .to_string() + })); } + let (system_audio_path, microphone_path) = + existing_audio_sidecars(&active.path, active.capture_mode); + let bytes = media_unit_bytes( + &active.path, + system_audio_path.as_deref(), + microphone_path.as_deref(), + ); let corrupt = if active.mime_type == MP4_RECORDING_MIME_TYPE || active.mime_type == QUICKTIME_RECORDING_MIME_TYPE @@ -718,11 +2275,476 @@ fn finalize_active_segment( width: active.width, height: active.height, bytes, + system_audio_path, + microphone_path, corrupt, error, + capture_mode: active.capture_mode, + exclusion_tainted: active.exclusion_tainted.load(Ordering::Relaxed), + graph_epoch_id: Some(active.graph_epoch_id), + graph_started_elapsed_ms: active.graph_started_elapsed_ms, + graph_ended_elapsed_ms: graph_elapsed_ms(app, Instant::now()), }) } +fn finalize_fenced_segment( + active: &ActiveScreenMemorySegment, + closed_path: PathBuf, + ended_at: Instant, +) -> Result { + let video_bytes = std::fs::metadata(&closed_path) + .map_err(|error| format!("fenced Screen Memory segment missing: {error}"))? + .len(); + if video_bytes == 0 { + return Err("fenced Screen Memory segment produced an empty file.".to_string()); + } + let (system_audio_path, microphone_path) = + existing_audio_sidecars(&closed_path, active.capture_mode); + let bytes = media_unit_bytes( + &closed_path, + system_audio_path.as_deref(), + microphone_path.as_deref(), + ); + Ok(ScreenMemorySegmentMetadata { + id: active.id.clone(), + file_name: closed_path + .file_name() + .map(|value| value.to_string_lossy().to_string()) + .unwrap_or_default(), + path: closed_path.clone(), + mime_type: MP4_RECORDING_MIME_TYPE.to_string(), + started_at: active.started_at_iso.clone(), + ended_at: now_iso(), + duration_ms: active.started_at.elapsed().as_millis(), + width: active.width, + height: active.height, + bytes, + system_audio_path, + microphone_path, + // fMP4 finalization is asynchronous. A fence proves a fragment + // boundary, not that every player accepts the logical file yet. + corrupt: native_screen::mp4_has_moov(&closed_path) == Some(false), + error: None, + capture_mode: active.capture_mode, + exclusion_tainted: active.exclusion_tainted.load(Ordering::Relaxed), + graph_epoch_id: Some(active.graph_epoch_id.clone()), + graph_started_elapsed_ms: active.graph_started_elapsed_ms, + graph_ended_elapsed_ms: active + .graph_started_elapsed_ms + .saturating_add(ended_at.duration_since(active.started_at).as_millis() as u64), + }) +} + +fn enqueue_segment_ocr( + app: &AppHandle, + segment: ScreenMemorySegmentMetadata, + sample_interval_seconds: u64, +) { + if !segment_derivatives_eligible(&segment) { + return; + } + if let Err(error) = write_ocr_status( + app, + &segment.id, + &crate::screen_memory_ocr::ScreenMemoryOcrIndexStatus::pending(), + ) { + record_error(app, error); + return; + } + let should_start = app.try_state::().is_some_and(|state| { + state + .inner + .lock() + .map(|mut runtime| { + if runtime.ocr_cancelled_segments.contains(&segment.id) { + return false; + } + runtime.ocr_queue.push_back(ScreenMemoryOcrJob { + segment, + sample_interval_seconds, + }); + if runtime.ocr_worker_running { + false + } else { + runtime.ocr_worker_running = true; + true + } + }) + .unwrap_or(false) + }); + if should_start { + let app = app.clone(); + std::thread::spawn(move || run_ocr_queue(app)); + } +} + +fn run_ocr_queue(app: AppHandle) { + loop { + let Some(state) = app.try_state::() else { + return; + }; + let job = { + let Ok(mut runtime) = state.inner.lock() else { + return; + }; + let Some(job) = take_next_ocr_job(&mut runtime) else { + return; + }; + job + }; + let state = app.state::(); + let Ok(_indexing) = state.indexing.lock() else { + return; + }; + let _ = run_ocr_job(&app, job); + if let Ok(mut runtime) = state.inner.lock() { + if let Some(segment_id) = runtime.ocr_active_segment.take() { + runtime.ocr_cancelled_segments.remove(&segment_id); + } + }; + } +} + +fn take_next_ocr_job(runtime: &mut ScreenMemoryRuntime) -> Option { + let job = runtime.ocr_queue.pop_front(); + if let Some(job) = job.as_ref() { + runtime.ocr_active_segment = Some(job.segment.id.clone()); + } else { + runtime.ocr_worker_running = false; + } + job +} + +fn write_ocr_status( + app: &AppHandle, + segment_id: &str, + status: &crate::screen_memory_ocr::ScreenMemoryOcrIndexStatus, +) -> Result<(), String> { + let data = serde_json::to_vec_pretty(status) + .map_err(|err| format!("screen memory OCR status encode failed: {err}"))?; + let state = app.state::(); + let runtime = state.inner.lock().map_err(|error| error.to_string())?; + if runtime.ocr_cancelled_segments.contains(segment_id) { + return Ok(()); + } + std::fs::write(segment_ocr_status_path(app, segment_id)?, data) + .map_err(|err| format!("screen memory OCR status write failed: {err}")) +} + +fn write_ocr_rows( + app: &AppHandle, + segment_id: &str, + rows: &[crate::screen_memory_ocr::ScreenMemoryOcrRow], +) -> Result<(), String> { + let state = app.state::(); + let runtime = state.inner.lock().map_err(|error| error.to_string())?; + if runtime.ocr_cancelled_segments.contains(segment_id) { + return Ok(()); + } + let rows_path = segment_ocr_rows_path(app, segment_id)?; + let mut file = File::create(rows_path) + .map_err(|err| format!("screen memory OCR rows open failed: {err}"))?; + for row in rows { + serde_json::to_writer(&mut file, row) + .map_err(|err| format!("screen memory OCR row encode failed: {err}"))?; + file.write_all(b"\n") + .map_err(|err| format!("screen memory OCR row write failed: {err}"))?; + } + Ok(()) +} + +fn run_ocr_job(app: &AppHandle, job: ScreenMemoryOcrJob) -> Result<(), String> { + let started_at = now_iso(); + if !job.segment.path.exists() { + return Ok(()); + } + if job.segment.exclusion_tainted { + return write_ocr_status( + app, + &job.segment.id, + &crate::screen_memory_ocr::ScreenMemoryOcrIndexStatus::skipped(now_iso()), + ); + } + if job.segment.corrupt || job.segment.error.is_some() { + return write_ocr_status( + app, + &job.segment.id, + &crate::screen_memory_ocr::ScreenMemoryOcrIndexStatus::skipped(now_iso()), + ); + } + write_ocr_status( + app, + &job.segment.id, + &crate::screen_memory_ocr::ScreenMemoryOcrIndexStatus::indexing(started_at.clone()), + )?; + let attempted = crate::screen_memory_ocr::requested_frame_offsets( + job.segment.duration_ms as u64, + job.sample_interval_seconds, + ) + .len(); + match crate::screen_memory_ocr::recognize_segment( + &job.segment.path, + &job.segment.id, + DateTime::parse_from_rfc3339(&job.segment.started_at) + .map_err(|_| "segment start timestamp unavailable".to_string())? + .with_timezone(&Utc), + job.sample_interval_seconds, + ) { + Ok(rows) => { + write_ocr_rows(app, &job.segment.id, &rows)?; + let result = write_ocr_status( + app, + &job.segment.id, + &crate::screen_memory_ocr::ScreenMemoryOcrIndexStatus::ready( + attempted, + rows.len(), + started_at, + now_iso(), + ), + ); + let _ = refresh_rewind_chapters(app); + result + } + Err(error) => { + let result = write_ocr_status( + app, + &job.segment.id, + &crate::screen_memory_ocr::ScreenMemoryOcrIndexStatus::failed( + attempted, + 0, + started_at, + now_iso(), + error, + ), + ); + let _ = refresh_rewind_chapters(app); + result + } + } +} + +fn enqueue_segment_transcript(app: &AppHandle, segment: ScreenMemorySegmentMetadata) { + use crate::screen_memory_transcript::ScreenMemoryTranscriptStatus; + + if !segment_transcript_eligible(&segment) { + if let Err(error) = write_transcript_status( + app, + &segment.id, + &ScreenMemoryTranscriptStatus::skipped(now_iso()), + ) { + record_error(app, error); + } + return; + } + if let Err(error) = + write_transcript_status(app, &segment.id, &ScreenMemoryTranscriptStatus::pending()) + { + record_error(app, error); + return; + } + let should_start = app.try_state::().is_some_and(|state| { + state + .inner + .lock() + .map(|mut runtime| { + if runtime.transcript_cancelled_segments.contains(&segment.id) { + return false; + } + runtime + .transcript_queue + .push_back(ScreenMemoryTranscriptJob { segment }); + if runtime.transcript_worker_running { + false + } else { + runtime.transcript_worker_running = true; + true + } + }) + .unwrap_or(false) + }); + if should_start { + let app = app.clone(); + std::thread::spawn(move || run_transcript_queue(app)); + } +} + +fn segment_derivatives_eligible(segment: &ScreenMemorySegmentMetadata) -> bool { + !segment.exclusion_tainted +} + +fn segment_transcript_eligible(segment: &ScreenMemorySegmentMetadata) -> bool { + segment.capture_mode == RewindCaptureMode::VisualsAudio + && segment_derivatives_eligible(segment) + && !segment.corrupt + && segment.error.is_none() + && segment + .system_audio_path + .as_ref() + .is_some_and(|path| path.exists()) + && segment + .microphone_path + .as_ref() + .is_some_and(|path| path.exists()) +} + +fn run_transcript_queue(app: AppHandle) { + loop { + let Some(state) = app.try_state::() else { + return; + }; + let job = { + let Ok(mut runtime) = state.inner.lock() else { + return; + }; + let Some(job) = take_next_transcript_job(&mut runtime) else { + return; + }; + job + }; + let state = app.state::(); + let Ok(_indexing) = state.indexing.lock() else { + return; + }; + let _ = run_transcript_job(&app, job); + if let Ok(mut runtime) = state.inner.lock() { + if let Some(segment_id) = runtime.transcript_active_segment.take() { + runtime.transcript_cancelled_segments.remove(&segment_id); + } + }; + } +} + +fn take_next_transcript_job( + runtime: &mut ScreenMemoryRuntime, +) -> Option { + let job = runtime.transcript_queue.pop_front(); + if let Some(job) = job.as_ref() { + runtime.transcript_active_segment = Some(job.segment.id.clone()); + } else { + runtime.transcript_worker_running = false; + } + job +} + +fn write_transcript_status( + app: &AppHandle, + segment_id: &str, + status: &crate::screen_memory_transcript::ScreenMemoryTranscriptStatus, +) -> Result<(), String> { + let data = serde_json::to_vec_pretty(status) + .map_err(|err| format!("screen memory transcript status encode failed: {err}"))?; + let state = app.state::(); + let runtime = state.inner.lock().map_err(|error| error.to_string())?; + if runtime.transcript_cancelled_segments.contains(segment_id) { + return Ok(()); + } + std::fs::write(segment_transcript_status_path(app, segment_id)?, data) + .map_err(|err| format!("screen memory transcript status write failed: {err}")) +} + +fn write_transcript_rows( + app: &AppHandle, + segment_id: &str, + rows: &[crate::screen_memory_transcript::ScreenMemoryTranscriptRow], +) -> Result<(), String> { + let state = app.state::(); + let runtime = state.inner.lock().map_err(|error| error.to_string())?; + if runtime.transcript_cancelled_segments.contains(segment_id) { + return Ok(()); + } + let rows_path = segment_transcript_rows_path(app, segment_id)?; + let mut file = File::create(rows_path) + .map_err(|err| format!("screen memory transcript rows open failed: {err}"))?; + for row in rows { + serde_json::to_writer(&mut file, row) + .map_err(|err| format!("screen memory transcript row encode failed: {err}"))?; + file.write_all(b"\n") + .map_err(|err| format!("screen memory transcript row write failed: {err}"))?; + } + Ok(()) +} + +fn run_transcript_job(app: &AppHandle, job: ScreenMemoryTranscriptJob) -> Result<(), String> { + use crate::screen_memory_transcript::ScreenMemoryTranscriptStatus; + + // Re-check eligibility after dequeue: retention/deletion and legacy data + // may have changed while the serialized worker was occupied with OCR. + if !segment_transcript_eligible(&job.segment) || !job.segment.path.exists() { + return write_transcript_status( + app, + &job.segment.id, + &ScreenMemoryTranscriptStatus::skipped(now_iso()), + ); + } + let started_at = now_iso(); + write_transcript_status( + app, + &job.segment.id, + &ScreenMemoryTranscriptStatus::transcribing(started_at.clone()), + )?; + let segment_started_at = match DateTime::parse_from_rfc3339(&job.segment.started_at) { + Ok(value) => value.with_timezone(&Utc), + Err(_) => { + return write_transcript_status( + app, + &job.segment.id, + &ScreenMemoryTranscriptStatus::failed( + started_at, + now_iso(), + "segment start timestamp unavailable", + ), + ) + } + }; + let duration_ms = u64::try_from(job.segment.duration_ms) + .unwrap_or(crate::screen_memory_transcript::MAX_DURATION_MS.saturating_add(1)); + let mut rows = Vec::new(); + for (path, source) in [ + (job.segment.system_audio_path.as_ref(), "system-audio"), + (job.segment.microphone_path.as_ref(), "microphone"), + ] { + let Some(path) = path else { + return write_transcript_status( + app, + &job.segment.id, + &ScreenMemoryTranscriptStatus::failed( + started_at, + now_iso(), + format!("{source} sidecar is unavailable"), + ), + ); + }; + match crate::screen_memory_transcript::transcribe_segment( + app, + path, + &job.segment.id, + source, + 0, + segment_started_at, + duration_ms, + None, + ) { + Ok(mut source_rows) => rows.append(&mut source_rows), + Err(error) => { + return write_transcript_status( + app, + &job.segment.id, + &ScreenMemoryTranscriptStatus::failed(started_at, now_iso(), error), + ) + } + } + } + rows.sort_by_key(|row| (row.start_ms, row.end_ms, row.source.clone())); + write_transcript_rows(app, &job.segment.id, &rows)?; + let result = write_transcript_status( + app, + &job.segment.id, + &ScreenMemoryTranscriptStatus::ready(rows.len(), started_at, now_iso()), + ); + let _ = refresh_rewind_chapters(app); + result +} + fn write_segment_metadata( app: &AppHandle, segment: &ScreenMemorySegmentMetadata, @@ -730,7 +2752,33 @@ fn write_segment_metadata( let path = segment_metadata_path(app, &segment.id)?; let data = serde_json::to_vec_pretty(segment) .map_err(|e| format!("screen memory metadata encode failed: {e}"))?; - std::fs::write(path, data).map_err(|e| format!("screen memory metadata write failed: {e}")) + std::fs::write(path, data).map_err(|e| format!("screen memory metadata write failed: {e}"))?; + crate::rewind_clip::pin_finalized_segment_if_active(app, segment)?; + enqueue_segment_transcript(app, segment.clone()); + refresh_rewind_chapters(app)?; + Ok(()) +} + +fn refresh_rewind_chapters(app: &AppHandle) -> Result<(), String> { + let events = read_screen_memory_events(app)?; + crate::rewind_chapters::rebuild( + &screen_memory_dir(app)?, + recent_segments(app, None)?, + events, + ) +} + +fn read_screen_memory_events(app: &AppHandle) -> Result, String> { + let path = screen_memory_events_path(app)?; + if !path.exists() { + return Ok(Vec::new()); + } + let file = File::open(path).map_err(|e| format!("screen memory events open failed: {e}"))?; + Ok(BufReader::new(file) + .lines() + .map_while(Result::ok) + .filter_map(|line| serde_json::from_str(&line).ok()) + .collect()) } fn read_segment_metadata_path(path: &Path) -> Result { @@ -769,6 +2817,91 @@ fn recent_segments( Ok(segments) } +/// All finalized local segments, newest metadata included. This intentionally +/// exposes no mutation or upload behavior; bounded local consumers apply their +/// own coverage and privacy checks before reading adjacent indexes. +pub(crate) fn finalized_segments( + app: &AppHandle, +) -> Result, String> { + recent_segments(app, None) +} + +/// Finalized logical segments overlapping a shared capture-graph interval. +/// This is local metadata only; callers must pin before assembling an artifact. +pub(crate) fn finalized_segments_in_graph_interval( + app: &AppHandle, + started_elapsed_ms: u64, + ended_elapsed_ms: u64, +) -> Result, String> { + if ended_elapsed_ms < started_elapsed_ms { + return Ok(Vec::new()); + } + let current_epoch = graph_epoch_id(app); + recent_segments(app, None).map(|segments| { + segments + .into_iter() + .filter(|segment| { + segment_overlaps_graph_interval( + segment, + ¤t_epoch, + started_elapsed_ms, + ended_elapsed_ms, + ) + }) + .collect() + }) +} + +fn segment_overlaps_graph_interval( + segment: &ScreenMemorySegmentMetadata, + current_epoch: &str, + started_elapsed_ms: u64, + ended_elapsed_ms: u64, +) -> bool { + segment.graph_epoch_id.as_deref() == Some(current_epoch) + && segment.graph_started_elapsed_ms <= ended_elapsed_ms + && segment.graph_ended_elapsed_ms >= started_elapsed_ms +} + +/// Keep a finalized local segment from retention pruning while an in-flight +/// artifact copies it. Pin state is intentionally process-local. +pub(crate) fn pin_segment(app: &AppHandle, segment_id: &str, pin_id: &str) -> Result<(), String> { + let state = app.state::(); + let mut runtime = state.inner.lock().map_err(|error| error.to_string())?; + runtime + .segment_pins + .entry(sanitize_segment_id(segment_id)) + .or_default() + .insert(pin_id.to_owned()); + Ok(()) +} + +pub(crate) fn unpin_segment(app: &AppHandle, segment_id: &str, pin_id: &str) -> Result<(), String> { + let state = app.state::(); + let mut runtime = state.inner.lock().map_err(|error| error.to_string())?; + let segment_id = sanitize_segment_id(segment_id); + if let Some(pins) = runtime.segment_pins.get_mut(&segment_id) { + pins.remove(pin_id); + if pins.is_empty() { + runtime.segment_pins.remove(&segment_id); + } + } + Ok(()) +} + +fn segment_is_pinned(app: &AppHandle, segment_id: &str) -> bool { + app.try_state::() + .and_then(|state| { + state.inner.lock().ok().map(|runtime| { + runtime + .segment_pins + .get(segment_id) + .is_some_and(|pins| !pins.is_empty()) + }) + }) + .unwrap_or(false) +} + fn query_screen_memory( app: &AppHandle, query: Option, @@ -855,19 +2988,7 @@ fn recent_events( } fn export_recent(app: &AppHandle, minutes: u64) -> Result { - let minutes = minutes.clamp(1, 24 * 30 * 60); - let cutoff = Utc::now() - ChronoDuration::minutes(minutes as i64); - let segments = recent_segments(app, None)? - .into_iter() - .filter(|segment| { - DateTime::parse_from_rfc3339(&segment.ended_at) - .map(|value| value.with_timezone(&Utc) >= cutoff) - .unwrap_or(true) - }) - .collect::>(); - if segments.is_empty() { - return Err("No Screen Memory segments are available for that range.".to_string()); - } + let minutes = minutes.clamp(1, 5); let folder = app .path() @@ -878,29 +2999,28 @@ fn export_recent(app: &AppHandle, minutes: u64) -> Result artifact, + Err(error) => { + let _ = std::fs::remove_dir(&folder); + return Err(error); + } + }; + let bytes = std::fs::metadata(&artifact.path) + .map_err(|e| format!("screen memory export metadata failed: {e}"))? + .len(); + let files = vec![ScreenMemoryExportFile { + path: artifact.path.to_string_lossy().to_string(), + file_name, + bytes, + mime_type: artifact.mime_type.to_string(), + }]; Ok(ScreenMemoryExportResult { folder_path: folder.to_string_lossy().to_string(), @@ -918,14 +3038,125 @@ fn prune_segments(app: &AppHandle, config: &ScreenMemoryConfig) -> Result<(), St .ok(); let expired = ended_at.map(|value| value < cutoff).unwrap_or(false); kept_bytes = kept_bytes.saturating_add(segment.bytes); - if expired || kept_bytes > config.max_bytes { + if (expired || kept_bytes > config.max_bytes) && !segment_is_pinned(app, &segment.id) { let _ = delete_segment(app, &segment.id); } } + prune_events(app, cutoff)?; + refresh_rewind_chapters(app)?; Ok(()) } +fn prune_events(app: &AppHandle, cutoff: DateTime) -> Result<(), String> { + let path = screen_memory_events_path(app)?; + if !path.exists() { + return Ok(()); + } + let file = File::open(&path).map_err(|e| format!("screen memory events open failed: {e}"))?; + let retained = BufReader::new(file) + .lines() + .map_while(Result::ok) + .filter_map(|line| serde_json::from_str::(&line).ok()) + .collect::>(); + let mut coverage = recent_segments(app, None)? + .into_iter() + .filter_map(|segment| { + Some(( + DateTime::parse_from_rfc3339(&segment.started_at) + .ok()? + .with_timezone(&Utc), + DateTime::parse_from_rfc3339(&segment.ended_at) + .ok()? + .with_timezone(&Utc), + )) + }) + .collect::>(); + if let Some(active) = app + .state::() + .inner + .lock() + .ok() + .and_then(|runtime| { + runtime + .active + .as_ref() + .map(|active| active.started_at_iso.clone()) + }) + .and_then(|started_at| DateTime::parse_from_rfc3339(&started_at).ok()) + { + coverage.push((active.with_timezone(&Utc), Utc::now())); + } + let retained = retain_events_with_media_coverage(retained, cutoff, &coverage); + let temporary_path = path.with_extension("jsonl.tmp"); + let mut file = File::create(&temporary_path) + .map_err(|e| format!("screen memory events rewrite failed: {e}"))?; + for event in retained { + serde_json::to_writer(&mut file, &event) + .map_err(|e| format!("screen memory event encode failed: {e}"))?; + file.write_all(b"\n") + .map_err(|e| format!("screen memory event write failed: {e}"))?; + } + std::fs::rename(&temporary_path, &path) + .map_err(|e| format!("screen memory events replace failed: {e}")) +} + +fn retain_events_since( + events: Vec, + cutoff: DateTime, +) -> Vec { + events + .into_iter() + .filter(|event| { + DateTime::parse_from_rfc3339(&event.captured_at) + .map(|captured_at| captured_at.with_timezone(&Utc) >= cutoff) + .unwrap_or(false) + }) + .collect() +} + +fn retain_events_with_media_coverage( + events: Vec, + cutoff: DateTime, + coverage: &[(DateTime, DateTime)], +) -> Vec { + retain_events_since(events, cutoff) + .into_iter() + .filter(|event| { + if event.source == "coverage-gap" { + return true; + } + let Ok(captured_at) = DateTime::parse_from_rfc3339(&event.captured_at) else { + return false; + }; + let captured_at = captured_at.with_timezone(&Utc); + coverage.iter().any(|(started_at, ended_at)| { + captured_at >= *started_at && captured_at <= *ended_at + }) + }) + .collect() +} + fn delete_segment(app: &AppHandle, segment_id: &str) -> Result { + let state = app.try_state::(); + if let Some(state) = state.as_ref() { + if let Ok(mut runtime) = state.inner.lock() { + runtime.ocr_queue.retain(|job| job.segment.id != segment_id); + runtime.ocr_cancelled_segments.insert(segment_id.to_owned()); + runtime + .transcript_queue + .retain(|job| job.segment.id != segment_id); + runtime + .transcript_cancelled_segments + .insert(segment_id.to_owned()); + } + } + // Wait for a currently-running derivative job to observe cancellation + // before removing the source and its sidecars. This makes Clear and + // retention deletion a real boundary instead of a hopeful suggestion. + let _indexing = state + .as_ref() + .map(|state| state.indexing.lock().map_err(|error| error.to_string())) + .transpose()?; let metadata_path = segment_metadata_path(app, segment_id)?; let segment = read_segment_metadata_path(&metadata_path).ok(); let mut deleted_segments = 0_usize; @@ -933,9 +3164,26 @@ fn delete_segment(app: &AppHandle, segment_id: &str) -> Result Result Result { + let state = app.try_state::(); + if let Some(state) = state.as_ref() { + let mut runtime = state.inner.lock().map_err(|error| error.to_string())?; + let queued_ocr: Vec<_> = runtime + .ocr_queue + .iter() + .map(|job| job.segment.id.clone()) + .collect(); + let queued_transcripts: Vec<_> = runtime + .transcript_queue + .iter() + .map(|job| job.segment.id.clone()) + .collect(); + runtime.ocr_cancelled_segments.extend(queued_ocr); + runtime + .transcript_cancelled_segments + .extend(queued_transcripts); + if let Some(segment_id) = runtime.ocr_active_segment.clone() { + runtime.ocr_cancelled_segments.insert(segment_id); + } + if let Some(segment_id) = runtime.transcript_active_segment.clone() { + runtime.transcript_cancelled_segments.insert(segment_id); + } + runtime.ocr_queue.clear(); + runtime.transcript_queue.clear(); + } + let _indexing = state + .as_ref() + .map(|state| state.indexing.lock().map_err(|error| error.to_string())) + .transpose()?; let dir = screen_memory_dir(app)?; let mut deleted_segments = 0_usize; let mut deleted_bytes = 0_u64; for segment in recent_segments(app, None)? { deleted_bytes = deleted_bytes.saturating_add(segment.bytes); remove_file_if_exists(&segment.path)?; + if let Some(path) = segment.system_audio_path.as_ref() { + remove_file_if_exists(path)?; + } + if let Some(path) = segment.microphone_path.as_ref() { + remove_file_if_exists(path)?; + } remove_file_if_exists(&segment_metadata_path(app, &segment.id)?)?; deleted_segments += 1; } @@ -961,6 +3245,14 @@ fn delete_all_segments(app: &AppHandle) -> Result String { Utc::now().to_rfc3339() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn codex_agent_connection_replaces_only_its_own_mcp_footprint() { + let existing = r#"model = "gpt-5.6" + +[mcp_servers."clips-screen-memory"] +command = "old" +args = ["old"] + +[mcp_servers."clips-screen-memory".env] +OLD = "value" + +[mcp_servers.other] +command = "keep" +"#; + let block = codex_screen_memory_block(Path::new("/tmp/Clips Alpha/screen-memory")); + let next = replace_codex_screen_memory_block(existing, &block); + assert_eq!(next.matches("clips-screen-memory").count(), 1); + assert!(!next.contains("OLD")); + assert!(next.contains("[mcp_servers.other]")); + assert!(next.contains("/tmp/Clips Alpha/screen-memory")); + assert!(next.contains("@agent-native/core@latest")); + } + + #[test] + fn codex_agent_connection_appends_to_an_existing_config() { + let block = codex_screen_memory_block(Path::new("/tmp/screen-memory")); + let next = replace_codex_screen_memory_block("model = \"gpt-5.6\"\n", &block); + assert!(next.starts_with("model = \"gpt-5.6\"")); + assert!(next.contains("command = \"npx\"")); + assert!(next.contains("mcp\", \"screen-memory")); + } + + fn event(captured_at: &str) -> ScreenMemoryEvent { + ScreenMemoryEvent { + captured_at: captured_at.to_string(), + app_name: Some("Example".to_string()), + window_title: Some("Example window".to_string()), + bundle_id: Some("example.app".to_string()), + source: "test".to_string(), + coverage_gap_reason: None, + accessibility: None, + } + } + + fn segment(id: &str) -> ScreenMemorySegmentMetadata { + ScreenMemorySegmentMetadata { + id: id.into(), + path: PathBuf::from(format!("{id}.mp4")), + file_name: format!("{id}.mp4"), + mime_type: "video/mp4".into(), + started_at: "2026-07-19T10:00:00Z".into(), + ended_at: "2026-07-19T10:00:01Z".into(), + duration_ms: 1_000, + width: Some(1280), + height: Some(720), + bytes: 1, + system_audio_path: None, + microphone_path: None, + corrupt: false, + error: None, + capture_mode: RewindCaptureMode::VisualsAudio, + exclusion_tainted: false, + graph_epoch_id: Some("test-epoch".into()), + graph_started_elapsed_ms: 0, + graph_ended_elapsed_ms: 1_000, + } + } + + #[test] + fn derivative_dequeue_marks_the_job_active_in_the_same_runtime_mutation() { + let mut runtime = ScreenMemoryRuntime { + ocr_worker_running: true, + transcript_worker_running: true, + ..ScreenMemoryRuntime::default() + }; + runtime.ocr_queue.push_back(ScreenMemoryOcrJob { + segment: segment("ocr"), + sample_interval_seconds: 10, + }); + runtime + .transcript_queue + .push_back(ScreenMemoryTranscriptJob { + segment: segment("transcript"), + }); + + assert_eq!(take_next_ocr_job(&mut runtime).unwrap().segment.id, "ocr"); + assert_eq!(runtime.ocr_active_segment.as_deref(), Some("ocr")); + assert_eq!( + take_next_transcript_job(&mut runtime).unwrap().segment.id, + "transcript" + ); + assert_eq!( + runtime.transcript_active_segment.as_deref(), + Some("transcript") + ); + } + + #[test] + fn normalizes_bounds_and_excluded_bundle_ids() { + let config = normalize_screen_memory_config(ScreenMemoryConfig { + retention_hours: 100, + max_bytes: 1, + segment_seconds: 1, + sample_interval_seconds: 100, + excluded_bundle_ids: vec![ + " COM.Example.Secret ".to_string(), + "com.example.secret".to_string(), + " ".to_string(), + ], + ..ScreenMemoryConfig::default() + }); + + assert_eq!(config.retention_hours, MAX_RETENTION_HOURS); + assert_eq!(config.max_bytes, MIN_MAX_BYTES); + assert_eq!(config.segment_seconds, MIN_SEGMENT_SECONDS); + assert_eq!(config.sample_interval_seconds, MIN_SEGMENT_SECONDS); + assert_eq!(config.excluded_bundle_ids, vec!["com.example.secret"]); + } + + #[test] + fn excludes_bundle_ids_without_retaining_context() { + let config = ScreenMemoryConfig::default(); + assert_eq!( + event_exclusion_reason(&config, Some("COM.1PASSWORD.1PASSWORD"), Some("vault")), + Some("excluded-bundle-id") + ); + assert_eq!( + event_exclusion_reason( + &config, + Some("com.example.browser"), + Some("Search - Incognito") + ), + None + ); + let private_window_config = ScreenMemoryConfig { + exclude_private_windows: true, + ..ScreenMemoryConfig::default() + }; + assert_eq!( + event_exclusion_reason( + &private_window_config, + Some("com.example.browser"), + Some("Search - Incognito") + ), + Some("excluded-private-window") + ); + let gap = coverage_gap_event("privacy-exclusion"); + assert_eq!(gap.source, "coverage-gap"); + assert_eq!( + gap.coverage_gap_reason.as_deref(), + Some("privacy-exclusion") + ); + assert!(gap.app_name.is_none() && gap.window_title.is_none() && gap.bundle_id.is_none()); + assert!(gap.accessibility.is_none()); + } + + #[test] + fn old_event_jsonl_lines_parse_without_accessibility() { + let event: ScreenMemoryEvent = serde_json::from_str( + r#"{"capturedAt":"2026-07-18T12:00:00Z","appName":"Mail","windowTitle":"Inbox","bundleId":"com.apple.mail","source":"core-graphics"}"#, + ) + .expect("old event line remains readable"); + assert_eq!(event.app_name.as_deref(), Some("Mail")); + assert!(event.coverage_gap_reason.is_none()); + assert!(event.accessibility.is_none()); + } + + #[test] + fn recognized_exclusion_transitions_suspend_stay_suspended_and_resume() { + assert_eq!( + exclusion_transition(false, false), + ExclusionTransition::KeepCapturing + ); + assert_eq!( + exclusion_transition(false, true), + ExclusionTransition::Suspend + ); + assert_eq!( + exclusion_transition(true, true), + ExclusionTransition::StaySuspended + ); + assert_eq!( + exclusion_transition(true, false), + ExclusionTransition::Resume + ); + } + + #[test] + fn exclusion_resume_requires_one_stopped_producer_and_safe_live_config() { + assert!(exclusion_resume_allowed( + true, false, false, false, true, false + )); + assert!(!exclusion_resume_allowed( + true, false, false, false, true, true + )); + assert!(!exclusion_resume_allowed( + true, false, false, true, true, false + )); + assert!(!exclusion_resume_allowed( + true, true, false, false, true, false + )); + assert!(!exclusion_resume_allowed( + true, false, true, false, true, false + )); + } + + #[test] + fn exclusion_gap_covers_discarded_segment_start_and_every_effective_source() { + let started_at = Instant::now() - Duration::from_secs(7); + let gap = active_exclusion_gap(started_at, RewindCaptureMode::VisualsAudio); + assert_eq!(gap.started_at, started_at); + assert_eq!( + gap.sources, + vec![ + CaptureSource::Screen, + CaptureSource::SystemAudio, + CaptureSource::Microphone + ] + ); + } + + #[test] + fn discarded_segment_sidecars_are_removed_as_one_retention_unit() { + let dir = std::env::temp_dir().join(format!( + "clips-exclusion-discard-{}-{}", + std::process::id(), + SEGMENT_COUNTER.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&dir).unwrap(); + let paths: Vec<_> = [ + "segment.json", + "segment.ocr.jsonl", + "segment.ocr-status.json", + "segment.transcript.jsonl", + "segment.transcript-status.json", + ] + .into_iter() + .map(|name| dir.join(name)) + .collect(); + for path in &paths { + std::fs::write(path, b"private-derived-data").unwrap(); + } + + remove_discarded_sidecars(&paths).unwrap(); + + assert!(paths.iter().all(|path| !path.exists())); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn segment_metadata_preserves_exclusion_taint_and_defaults_legacy_rows() { + let tainted: ScreenMemorySegmentMetadata = serde_json::from_value(serde_json::json!({ + "id": "segment-1", "path": "/tmp/segment.mp4", "fileName": "segment.mp4", + "mimeType": "video/mp4", "startedAt": "2026-07-14T12:00:00Z", + "endedAt": "2026-07-14T12:01:00Z", "durationMs": 60000, "width": 1, + "height": 1, "bytes": 1, "corrupt": false, "error": null, + "exclusionTainted": true + })) + .unwrap(); + assert!(tainted.exclusion_tainted); + assert_eq!(tainted.graph_started_elapsed_ms, 0); + assert_eq!(tainted.graph_ended_elapsed_ms, 0); + assert_eq!(tainted.capture_mode, RewindCaptureMode::Visuals); + + let legacy: ScreenMemorySegmentMetadata = serde_json::from_value(serde_json::json!({ + "id": "segment-2", "path": "/tmp/segment.mp4", "fileName": "segment.mp4", + "mimeType": "video/mp4", "startedAt": "2026-07-14T12:00:00Z", + "endedAt": "2026-07-14T12:01:00Z", "durationMs": 60000, "width": null, + "height": null, "bytes": 1, "corrupt": false, "error": null + })) + .unwrap(); + assert!(!legacy.exclusion_tainted); + assert_eq!(legacy.capture_mode, RewindCaptureMode::Visuals); + assert_eq!(legacy.graph_epoch_id, None); + assert_eq!(legacy.graph_started_elapsed_ms, 0); + assert_eq!(legacy.graph_ended_elapsed_ms, 0); + + let mut current = legacy.clone(); + current.graph_epoch_id = Some("current-epoch".into()); + current.graph_started_elapsed_ms = 100; + current.graph_ended_elapsed_ms = 200; + assert!(segment_overlaps_graph_interval( + ¤t, + "current-epoch", + 150, + 250 + )); + assert!(!segment_overlaps_graph_interval( + ¤t, + "stale-epoch", + 150, + 250 + )); + assert!(!segment_overlaps_graph_interval( + &legacy, + "current-epoch", + 0, + 1 + )); + } + + #[test] + fn transcript_lifecycle_only_indexes_clean_visuals_audio_segments() { + let system = std::env::temp_dir().join(format!( + "clips-transcript-system-{}.wav", + std::process::id() + )); + let microphone = std::env::temp_dir().join(format!( + "clips-transcript-microphone-{}.wav", + std::process::id() + )); + std::fs::write(&system, b"audio").unwrap(); + std::fs::write(µphone, b"audio").unwrap(); + let mut segment: ScreenMemorySegmentMetadata = serde_json::from_value(serde_json::json!({ + "id": "segment-1", "path": "/tmp/segment.mp4", "fileName": "segment.mp4", + "mimeType": "video/mp4", "startedAt": "2026-07-14T12:00:00Z", + "endedAt": "2026-07-14T12:01:00Z", "durationMs": 60000, "width": 1, + "height": 1, "bytes": 1, "corrupt": false, "error": null, + "captureMode": "visuals-audio", + "systemAudioPath": system.clone(), + "microphonePath": microphone.clone() + })) + .unwrap(); + assert!(segment_transcript_eligible(&segment)); + segment.exclusion_tainted = true; + assert!(!segment_derivatives_eligible(&segment)); + assert!(!segment_transcript_eligible(&segment)); + segment.exclusion_tainted = false; + segment.corrupt = true; + assert!(!segment_transcript_eligible(&segment)); + segment.corrupt = false; + segment.error = Some("capture failed".into()); + assert!(!segment_transcript_eligible(&segment)); + segment.error = None; + segment.capture_mode = RewindCaptureMode::Visuals; + assert!(!segment_transcript_eligible(&segment)); + let _ = std::fs::remove_file(system); + let _ = std::fs::remove_file(microphone); + } + + #[test] + fn status_explains_recognized_media_exclusion_and_coverage_gap() { + let config = ScreenMemoryConfig { + enabled: true, + paused: false, + ..ScreenMemoryConfig::default() + }; + let coverage = coverage_language(&config, true, false); + assert!(coverage.contains("entire in-flight media segment was discarded")); + assert!(coverage.contains("source coverage gap")); + } + + #[test] + fn transcript_sidecars_are_deleted_together() { + let dir = std::env::temp_dir().join(format!( + "clips-transcript-sidecars-{}-{}", + std::process::id(), + SEGMENT_COUNTER.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&dir).unwrap(); + let (rows, status) = transcript_sidecar_paths(&dir, "segment/one"); + std::fs::write(&rows, b"{}\n").unwrap(); + std::fs::write(&status, b"{}").unwrap(); + remove_transcript_sidecars(&dir, "segment/one").unwrap(); + assert!(!rows.exists()); + assert!(!status.exists()); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn retains_only_parseable_events_within_the_retention_window() { + let cutoff = DateTime::parse_from_rfc3339("2026-07-14T12:00:00Z") + .unwrap() + .with_timezone(&Utc); + let events = retain_events_since( + vec![ + event("2026-07-14T11:59:59Z"), + event("2026-07-14T12:00:00Z"), + event("not-a-date"), + ], + cutoff, + ); + + assert_eq!(events.len(), 1); + assert_eq!(events[0].captured_at, "2026-07-14T12:00:00Z"); + } + + #[test] + fn app_context_is_pruned_with_media_while_coverage_gaps_remain_visible() { + let cutoff = DateTime::parse_from_rfc3339("2026-07-14T12:00:00Z") + .unwrap() + .with_timezone(&Utc); + let coverage = [( + DateTime::parse_from_rfc3339("2026-07-14T12:05:00Z") + .unwrap() + .with_timezone(&Utc), + DateTime::parse_from_rfc3339("2026-07-14T12:10:00Z") + .unwrap() + .with_timezone(&Utc), + )]; + let mut gap = event("2026-07-14T12:03:00Z"); + gap.source = "coverage-gap".into(); + let events = retain_events_with_media_coverage( + vec![ + event("2026-07-14T12:02:00Z"), + gap, + event("2026-07-14T12:06:00Z"), + ], + cutoff, + &coverage, + ); + + assert_eq!(events.len(), 2); + assert_eq!(events[0].source, "coverage-gap"); + assert_eq!(events[1].captured_at, "2026-07-14T12:06:00Z"); + } + + #[test] + fn temporary_audio_refcounts_restore_the_persisted_base_mode() { + assert_eq!( + effective_mode(RewindCaptureMode::Visuals, 0), + RewindCaptureMode::Visuals + ); + assert_eq!( + effective_mode(RewindCaptureMode::Visuals, 1), + RewindCaptureMode::VisualsAudio + ); + assert_eq!( + effective_mode(RewindCaptureMode::Visuals, 2), + RewindCaptureMode::VisualsAudio + ); + assert_eq!( + effective_mode(RewindCaptureMode::VisualsAudio, 0), + RewindCaptureMode::VisualsAudio + ); + + let mut runtime = ScreenMemoryRuntime::default(); + runtime.temporary_audio_consumers.insert( + "meeting-whisper".into(), + TemporaryAudioConsumer { + graph_lease_id: "meeting-lease".into(), + }, + ); + runtime.temporary_audio_consumers.insert( + "rewind-clip".into(), + TemporaryAudioConsumer { + graph_lease_id: "clip-lease".into(), + }, + ); + assert_eq!( + requested_sources(effective_mode( + RewindCaptureMode::Visuals, + runtime.temporary_audio_consumers.len() + )), + vec![ + CaptureSource::Screen, + CaptureSource::SystemAudio, + CaptureSource::Microphone + ] + ); + runtime.temporary_audio_consumers.remove("rewind-clip"); + assert_eq!( + effective_mode( + RewindCaptureMode::Visuals, + runtime.temporary_audio_consumers.len() + ), + RewindCaptureMode::VisualsAudio + ); + runtime.temporary_audio_consumers.remove("meeting-whisper"); + assert_eq!( + effective_mode( + RewindCaptureMode::Visuals, + runtime.temporary_audio_consumers.len() + ), + RewindCaptureMode::Visuals + ); + } + + #[test] + fn temporary_audio_requires_a_live_unpaused_rewind_producer() { + assert!(temporary_audio_lease_available( + true, false, true, true, false + )); + assert!(!temporary_audio_lease_available( + false, false, true, true, true + )); + assert!(!temporary_audio_lease_available( + true, true, true, true, true + )); + assert!(!temporary_audio_lease_available( + true, false, false, true, true + )); + assert!(!temporary_audio_lease_available( + true, false, true, false, false + )); + } + + #[test] + fn terminal_handoffs_do_not_retain_agent_reason_text() { + let mut object = serde_json::json!({ + "status": "pending", + "reason": "private words from the requested interval" + }) + .as_object() + .unwrap() + .clone(); + scrub_terminal_handoff_fields(&mut object, "processing"); + assert!(object.contains_key("reason")); + scrub_terminal_handoff_fields(&mut object, "declined"); + assert!(!object.contains_key("reason")); + } +} diff --git a/templates/clips/desktop/src-tauri/src/screen_memory_ocr.rs b/templates/clips/desktop/src-tauri/src/screen_memory_ocr.rs new file mode 100644 index 0000000000..65dd257faf --- /dev/null +++ b/templates/clips/desktop/src-tauri/src/screen_memory_ocr.rs @@ -0,0 +1,360 @@ +//! Local, sparse OCR for completed Screen Memory video segments. +//! +//! This module deliberately returns bounded rows for the caller to persist; +//! it never writes frame images and never logs recognized text. +// The capture worker consumes this foundation in the next integration slice. +#![allow(dead_code)] + +use chrono::{DateTime, Duration, Utc}; +use serde::{Deserialize, Serialize}; +use std::ffi::{CStr, CString}; +use std::path::Path; + +pub const OCR_SCHEMA_VERSION: u32 = 1; +pub const MAX_OCR_FRAMES: usize = 30; +pub const MAX_OCR_TEXT_BYTES: usize = 8 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ScreenMemoryOcrIndexState { + Pending, + Indexing, + Ready, + Failed, + Skipped, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScreenMemoryOcrIndexStatus { + pub state: ScreenMemoryOcrIndexState, + pub attempted_frames: usize, + pub completed_frames: usize, + pub started_at: Option, + pub finished_at: Option, + pub error: Option, +} + +impl ScreenMemoryOcrIndexStatus { + pub fn pending() -> Self { + Self { + state: ScreenMemoryOcrIndexState::Pending, + attempted_frames: 0, + completed_frames: 0, + started_at: None, + finished_at: None, + error: None, + } + } + + pub fn indexing(started_at: impl Into) -> Self { + Self { + state: ScreenMemoryOcrIndexState::Indexing, + attempted_frames: 0, + completed_frames: 0, + started_at: Some(started_at.into()), + finished_at: None, + error: None, + } + } + + pub fn ready( + attempted_frames: usize, + completed_frames: usize, + started_at: impl Into, + finished_at: impl Into, + ) -> Self { + Self { + state: ScreenMemoryOcrIndexState::Ready, + attempted_frames, + completed_frames, + started_at: Some(started_at.into()), + finished_at: Some(finished_at.into()), + error: None, + } + } + + pub fn failed( + attempted_frames: usize, + completed_frames: usize, + started_at: impl Into, + finished_at: impl Into, + error: impl Into, + ) -> Self { + Self { + state: ScreenMemoryOcrIndexState::Failed, + attempted_frames, + completed_frames, + started_at: Some(started_at.into()), + finished_at: Some(finished_at.into()), + error: Some(error.into()), + } + } + + pub fn skipped(finished_at: impl Into) -> Self { + Self { + state: ScreenMemoryOcrIndexState::Skipped, + attempted_frames: 0, + completed_frames: 0, + started_at: None, + finished_at: Some(finished_at.into()), + error: None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScreenMemoryOcrRow { + pub schema_version: u32, + pub segment_id: String, + pub captured_at: String, + pub offset_ms: i64, + pub source: String, + pub ocr_text: String, + pub confidence: f32, + pub frame_width: u32, + pub frame_height: u32, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct NativeOcrFrame { + offset_ms: i64, + text: String, + confidence: f32, + width: u32, + height: u32, +} + +#[derive(Debug, Deserialize)] +struct NativeOcrResponse { + ok: bool, + frames: Option>, + error: Option, +} + +/// Maps Vision's actual extracted frame time onto the segment timeline. +pub fn captured_at_for_offset(segment_started_at: DateTime, offset_ms: i64) -> String { + (segment_started_at + Duration::milliseconds(offset_ms.max(0))).to_rfc3339() +} + +/// The sparse sampling contract shared with the native helper: start at the +/// segment boundary, use the configured interval, and never request more than +/// thirty video frames. +pub fn requested_frame_offsets(duration_ms: u64, sample_interval_seconds: u64) -> Vec { + let interval_ms = sample_interval_seconds.max(1).saturating_mul(1_000); + (0..MAX_OCR_FRAMES) + .map(|index| (index as u64).saturating_mul(interval_ms)) + .take_while(|offset_ms| *offset_ms <= duration_ms) + .collect() +} + +/// Retains a bounded, UTF-8-safe prefix without carrying frame images forward. +pub fn bounded_ocr_text(text: &str) -> String { + if text.len() <= MAX_OCR_TEXT_BYTES { + return text.to_owned(); + } + let mut end = MAX_OCR_TEXT_BYTES; + while !text.is_char_boundary(end) { + end -= 1; + } + text[..end].to_owned() +} + +fn rows_from_frames( + segment_id: &str, + segment_started_at: DateTime, + frames: impl IntoIterator, +) -> Vec { + frames + .into_iter() + .filter(|frame| !frame.text.trim().is_empty()) + .take(MAX_OCR_FRAMES) + .map(|frame| ScreenMemoryOcrRow { + schema_version: OCR_SCHEMA_VERSION, + segment_id: segment_id.to_owned(), + captured_at: captured_at_for_offset(segment_started_at, frame.offset_ms), + offset_ms: frame.offset_ms.max(0), + source: "ocr".to_owned(), + ocr_text: bounded_ocr_text(&frame.text), + confidence: frame.confidence.clamp(0.0, 1.0), + frame_width: frame.width, + frame_height: frame.height, + }) + .collect() +} + +/// Runs sparse, on-device OCR on macOS. Callers own persistence and should +/// transition the associated index status from `indexing` to `ready`/`failed`. +#[cfg(target_os = "macos")] +pub fn recognize_segment( + segment_path: &Path, + segment_id: &str, + segment_started_at: DateTime, + sample_interval_seconds: u64, +) -> Result, String> { + let path = CString::new(segment_path.as_os_str().as_encoded_bytes()) + .map_err(|_| "segment path contains an interior NUL byte".to_owned())?; + let response = + unsafe { clips_screen_memory_ocr_json(path.as_ptr(), sample_interval_seconds.max(1)) }; + if response.is_null() { + return Err("macOS OCR helper returned no response".to_owned()); + } + let response_text = unsafe { + let text = CStr::from_ptr(response).to_string_lossy().into_owned(); + clips_screen_memory_ocr_free(response); + text + }; + let response: NativeOcrResponse = serde_json::from_str(&response_text) + .map_err(|_| "macOS OCR helper returned an invalid response".to_owned())?; + if !response.ok { + return Err(response + .error + .unwrap_or_else(|| "macOS OCR failed".to_owned())); + } + Ok(rows_from_frames( + segment_id, + segment_started_at, + response.frames.unwrap_or_default(), + )) +} + +/// Platforms without AVFoundation/Vision do not silently provide a different +/// OCR path. Capture remains usable, while callers can mark OCR as skipped. +#[cfg(not(target_os = "macos"))] +pub fn recognize_segment( + _segment_path: &Path, + _segment_id: &str, + _segment_started_at: DateTime, + _sample_interval_seconds: u64, +) -> Result, String> { + Err("local Screen Memory OCR is supported on macOS only".to_owned()) +} + +#[cfg(target_os = "macos")] +unsafe extern "C" { + fn clips_screen_memory_ocr_json( + video_path: *const std::ffi::c_char, + sample_interval_seconds: u64, + ) -> *mut std::ffi::c_char; + fn clips_screen_memory_ocr_free(response: *mut std::ffi::c_char); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame(offset_ms: i64, text: impl Into) -> NativeOcrFrame { + NativeOcrFrame { + offset_ms, + text: text.into(), + confidence: 1.2, + width: 1920, + height: 1080, + } + } + + #[test] + fn maps_actual_frame_time_to_segment_timestamp() { + let started_at = "2026-07-14T12:00:00Z".parse::>().unwrap(); + assert_eq!( + captured_at_for_offset(started_at, 1_250), + "2026-07-14T12:00:01.250+00:00" + ); + assert_eq!( + captured_at_for_offset(started_at, -1), + "2026-07-14T12:00:00+00:00" + ); + } + + #[test] + fn caps_rows_and_bounds_text_without_splitting_unicode() { + let started_at = "2026-07-14T12:00:00Z".parse::>().unwrap(); + let oversized = "é".repeat(MAX_OCR_TEXT_BYTES); + let rows = rows_from_frames( + "segment-1", + started_at, + (0..35).map(|index| { + frame( + index * 1_000, + if index == 0 { + oversized.clone() + } else { + "text".to_owned() + }, + ) + }), + ); + assert_eq!(rows.len(), MAX_OCR_FRAMES); + assert!(rows[0].ocr_text.len() <= MAX_OCR_TEXT_BYTES); + assert!(rows[0].ocr_text.is_char_boundary(rows[0].ocr_text.len())); + assert_eq!(rows[0].source, "ocr"); + assert_eq!(rows[0].confidence, 1.0); + } + + #[test] + fn samples_the_requested_interval_with_a_thirty_frame_cap() { + assert_eq!( + requested_frame_offsets(5 * 60 * 1_000, 10), + (0..MAX_OCR_FRAMES) + .map(|index| index as u64 * 10_000) + .collect::>() + ); + assert_eq!(requested_frame_offsets(2_500, 0), vec![0, 1_000, 2_000]); + } + + #[test] + fn status_shape_serializes_stably() { + assert_eq!( + serde_json::to_value(ScreenMemoryOcrIndexStatus::pending()).unwrap(), + serde_json::json!({ + "state": "pending", "attemptedFrames": 0, "completedFrames": 0, + "startedAt": null, "finishedAt": null, "error": null + }) + ); + assert_eq!( + serde_json::to_value(ScreenMemoryOcrIndexStatus::ready( + 4, + 3, + "2026-07-14T12:00:00Z", + "2026-07-14T12:00:01Z", + )) + .unwrap(), + serde_json::json!({ + "state": "ready", "attemptedFrames": 4, "completedFrames": 3, + "startedAt": "2026-07-14T12:00:00Z", "finishedAt": "2026-07-14T12:00:01Z", "error": null + }) + ); + assert_eq!( + serde_json::to_value(ScreenMemoryOcrIndexStatus::indexing("2026-07-14T12:00:00Z")) + .unwrap(), + serde_json::json!({ + "state": "indexing", "attemptedFrames": 0, "completedFrames": 0, + "startedAt": "2026-07-14T12:00:00Z", "finishedAt": null, "error": null + }) + ); + assert_eq!( + serde_json::to_value(ScreenMemoryOcrIndexStatus::failed( + 4, + 1, + "2026-07-14T12:00:00Z", + "2026-07-14T12:00:01Z", + "unavailable", + )) + .unwrap(), + serde_json::json!({ + "state": "failed", "attemptedFrames": 4, "completedFrames": 1, + "startedAt": "2026-07-14T12:00:00Z", "finishedAt": "2026-07-14T12:00:01Z", "error": "unavailable" + }) + ); + assert_eq!( + serde_json::to_value(ScreenMemoryOcrIndexStatus::skipped("2026-07-14T12:00:01Z")) + .unwrap(), + serde_json::json!({ + "state": "skipped", "attemptedFrames": 0, "completedFrames": 0, + "startedAt": null, "finishedAt": "2026-07-14T12:00:01Z", "error": null + }) + ); + } +} diff --git a/templates/clips/desktop/src-tauri/src/screen_memory_transcript.rs b/templates/clips/desktop/src-tauri/src/screen_memory_transcript.rs new file mode 100644 index 0000000000..19613f6304 --- /dev/null +++ b/templates/clips/desktop/src-tauri/src/screen_memory_transcript.rs @@ -0,0 +1,408 @@ +//! Local post-segment audio transcription for Screen Memory. +//! +//! MP4 audio is decoded directly into bounded memory, transcribed through the +//! process-wide Whisper context, and returned as typed rows. This module does +//! not persist PCM and never logs transcript text; the capture worker owns the +//! adjacent JSONL/status files. +#![allow(dead_code)] + +use chrono::{DateTime, Duration, Utc}; +use serde::{Deserialize, Serialize}; +use std::io::Read; +use std::path::Path; +use std::process::{Command, Stdio}; +use tauri::AppHandle; + +pub const TRANSCRIPT_SCHEMA_VERSION: u32 = 1; +pub const MAX_INPUT_BYTES: u64 = 2 * 1024 * 1024 * 1024; +pub const MAX_DURATION_MS: u64 = 30 * 60 * 1_000; +pub const MAX_TRANSCRIPT_ROWS: usize = 2_048; +pub const MAX_ROW_TEXT_BYTES: usize = 8 * 1024; +pub const MAX_TOTAL_TEXT_BYTES: usize = 1024 * 1024; +const SAMPLE_RATE: usize = 16_000; +const BYTES_PER_SAMPLE: usize = std::mem::size_of::(); +const FFMPEG_CANDIDATE_PATHS: &[&str] = &[ + "ffmpeg", + "/opt/homebrew/bin/ffmpeg", + "/usr/local/bin/ffmpeg", + "/opt/local/bin/ffmpeg", +]; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ScreenMemoryTranscriptState { + Pending, + Transcribing, + Ready, + Failed, + Skipped, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScreenMemoryTranscriptStatus { + pub state: ScreenMemoryTranscriptState, + pub row_count: usize, + pub started_at: Option, + pub finished_at: Option, + pub error: Option, +} + +impl ScreenMemoryTranscriptStatus { + pub fn pending() -> Self { + Self::new(ScreenMemoryTranscriptState::Pending) + } + + pub fn transcribing(started_at: impl Into) -> Self { + Self { + started_at: Some(started_at.into()), + ..Self::new(ScreenMemoryTranscriptState::Transcribing) + } + } + + pub fn ready( + row_count: usize, + started_at: impl Into, + finished_at: impl Into, + ) -> Self { + Self { + state: ScreenMemoryTranscriptState::Ready, + row_count: row_count.min(MAX_TRANSCRIPT_ROWS), + started_at: Some(started_at.into()), + finished_at: Some(finished_at.into()), + error: None, + } + } + + pub fn failed( + started_at: impl Into, + finished_at: impl Into, + error: impl AsRef, + ) -> Self { + Self { + state: ScreenMemoryTranscriptState::Failed, + row_count: 0, + started_at: Some(started_at.into()), + finished_at: Some(finished_at.into()), + error: Some(bounded_text(error.as_ref(), MAX_ROW_TEXT_BYTES)), + } + } + + pub fn skipped(finished_at: impl Into) -> Self { + Self { + finished_at: Some(finished_at.into()), + ..Self::new(ScreenMemoryTranscriptState::Skipped) + } + } + + fn new(state: ScreenMemoryTranscriptState) -> Self { + Self { + state, + row_count: 0, + started_at: None, + finished_at: None, + error: None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScreenMemoryTranscriptRow { + pub schema_version: u32, + pub segment_id: String, + pub source: String, + pub captured_at: String, + pub start_ms: u64, + pub end_ms: u64, + pub text: String, +} + +/// The exact background-worker entry point intended for `screen_memory.rs`. +/// `duration_ms` comes from finalized segment metadata and is checked before +/// ffmpeg is started. `source` should describe the recorded audio track (for +/// example `mixed-audio`); rows retain that source verbatim after validation. +pub fn transcribe_segment( + app: &AppHandle, + segment_path: &Path, + segment_id: &str, + source: &str, + audio_stream_index: usize, + segment_started_at: DateTime, + duration_ms: u64, + language: Option<&str>, +) -> Result, String> { + validate_request(segment_path, segment_id, source, duration_ms)?; + let samples = decode_audio(segment_path, audio_stream_index, duration_ms)?; + let raw = crate::whisper_speech::transcribe_offline_file_samples(app, &samples, language)?; + Ok(clean_rows( + segment_id, + source, + segment_started_at, + duration_ms, + raw, + )) +} + +fn validate_request( + segment_path: &Path, + segment_id: &str, + source: &str, + duration_ms: u64, +) -> Result<(), String> { + if segment_id.trim().is_empty() || segment_id.len() > 255 { + return Err("invalid transcript segment id".to_owned()); + } + if source.trim().is_empty() || source.len() > 128 { + return Err("invalid transcript source".to_owned()); + } + if duration_ms == 0 || duration_ms > MAX_DURATION_MS { + return Err(format!( + "transcript duration must be between 1 and {MAX_DURATION_MS} ms" + )); + } + let metadata = std::fs::metadata(segment_path) + .map_err(|e| format!("transcript input unavailable: {e}"))?; + if !metadata.is_file() || metadata.len() == 0 || metadata.len() > MAX_INPUT_BYTES { + return Err("transcript input size is outside the allowed bounds".to_owned()); + } + Ok(()) +} + +fn decode_audio( + segment_path: &Path, + audio_stream_index: usize, + duration_ms: u64, +) -> Result, String> { + let ffmpeg = resolve_ffmpeg_path() + .ok_or_else(|| "ffmpeg is required for local segment transcription".to_owned())?; + let max_samples = ((duration_ms as usize) + .saturating_mul(SAMPLE_RATE) + .saturating_add(999)) + / 1_000; + let max_bytes = max_samples.saturating_mul(BYTES_PER_SAMPLE); + let duration_seconds = format!("{:.3}", duration_ms as f64 / 1_000.0); + let audio_map = format!("0:a:{audio_stream_index}?"); + let mut child = Command::new(ffmpeg) + .args(["-v", "error", "-nostdin", "-i"]) + .arg(segment_path) + .args(["-map", &audio_map, "-vn", "-t", &duration_seconds]) + .args(["-ac", "1", "-ar", "16000", "-f", "f32le", "pipe:1"]) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(|e| format!("ffmpeg audio decode spawn failed: {e}"))?; + let mut bytes = Vec::with_capacity(max_bytes.min(8 * 1024 * 1024)); + child + .stdout + .take() + .ok_or_else(|| "ffmpeg audio decode stdout unavailable".to_owned())? + .take((max_bytes as u64).saturating_add(1)) + .read_to_end(&mut bytes) + .map_err(|e| format!("ffmpeg audio decode read failed: {e}"))?; + if bytes.len() > max_bytes { + let _ = child.kill(); + let _ = child.wait(); + return Err("decoded audio exceeded the declared segment duration".to_owned()); + } + let status = child + .wait() + .map_err(|e| format!("ffmpeg audio decode wait failed: {e}"))?; + if !status.success() { + return Err("ffmpeg could not decode segment audio".to_owned()); + } + if bytes.is_empty() { + return Err("segment has no decodable audio track".to_owned()); + } + if bytes.len() % BYTES_PER_SAMPLE != 0 { + return Err("ffmpeg returned malformed PCM audio".to_owned()); + } + let samples = bytes + .chunks_exact(BYTES_PER_SAMPLE) + .map(|chunk| f32::from_le_bytes(chunk.try_into().expect("four-byte PCM chunk"))) + .collect::>(); + if samples.iter().any(|sample| !sample.is_finite()) { + return Err("ffmpeg returned invalid PCM audio".to_owned()); + } + Ok(samples) +} + +fn clean_rows( + segment_id: &str, + source: &str, + segment_started_at: DateTime, + duration_ms: u64, + raw: impl IntoIterator, +) -> Vec { + let mut total_text_bytes = 0usize; + raw.into_iter() + .filter_map(|segment| { + let text = clean_text(&segment.text)?; + let remaining = MAX_TOTAL_TEXT_BYTES.saturating_sub(total_text_bytes); + if remaining == 0 { + return None; + } + let text = bounded_text(&text, MAX_ROW_TEXT_BYTES.min(remaining)); + if text.is_empty() { + return None; + } + total_text_bytes = total_text_bytes.saturating_add(text.len()); + let start_ms = segment.start_ms.max(0) as u64; + let end_ms = segment.end_ms.max(segment.start_ms).max(0) as u64; + Some(ScreenMemoryTranscriptRow { + schema_version: TRANSCRIPT_SCHEMA_VERSION, + segment_id: segment_id.to_owned(), + source: source.to_owned(), + captured_at: (segment_started_at + + Duration::milliseconds(start_ms.min(duration_ms) as i64)) + .to_rfc3339(), + start_ms: start_ms.min(duration_ms), + end_ms: end_ms.min(duration_ms), + text, + }) + }) + .filter(|row| row.end_ms > row.start_ms) + .take(MAX_TRANSCRIPT_ROWS) + .collect() +} + +fn clean_text(text: &str) -> Option { + let trimmed = text.trim(); + if trimmed.is_empty() || !trimmed.chars().any(char::is_alphanumeric) { + return None; + } + let normalized = trimmed + .trim_matches(|character: char| !character.is_alphanumeric() && character != '_') + .to_ascii_lowercase(); + if matches!( + normalized.as_str(), + "blank_audio" | "silence" | "music" | "no speech" + ) { + return None; + } + if (trimmed.starts_with('[') && trimmed.ends_with(']')) + || (trimmed.starts_with('(') && trimmed.ends_with(')')) + { + return None; + } + Some(trimmed.to_owned()) +} + +fn bounded_text(text: &str, max_bytes: usize) -> String { + if text.len() <= max_bytes { + return text.to_owned(); + } + let mut end = max_bytes; + while end > 0 && !text.is_char_boundary(end) { + end -= 1; + } + text[..end].to_owned() +} + +fn resolve_ffmpeg_path() -> Option { + if let Ok(path) = std::env::var("CLIPS_FFMPEG_PATH") { + let trimmed = path.trim(); + if !trimmed.is_empty() && command_available(trimmed) { + return Some(trimmed.to_owned()); + } + } + FFMPEG_CANDIDATE_PATHS + .iter() + .copied() + .find(|candidate| command_available(candidate)) + .map(str::to_owned) +} + +fn command_available(command: &str) -> bool { + Command::new(command) + .arg("-version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|status| status.success()) + .unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::whisper_speech::OfflineTranscriptSegment; + + fn segment(start_ms: i64, end_ms: i64, text: impl Into) -> OfflineTranscriptSegment { + OfflineTranscriptSegment { + start_ms, + end_ms, + text: text.into(), + } + } + + #[test] + fn rejects_unbounded_requests_before_decode() { + let path = Path::new("does-not-matter.mp4"); + assert!(validate_request(path, "segment", "mixed-audio", 0).is_err()); + assert!(validate_request(path, "segment", "mixed-audio", MAX_DURATION_MS + 1).is_err()); + assert!(validate_request(path, "", "mixed-audio", 1).is_err()); + assert!(validate_request(path, "segment", "", 1).is_err()); + } + + #[test] + fn cleans_rows_and_rejects_silence_placeholders() { + let rows = clean_rows( + "segment-1", + "mixed-audio", + "2026-07-14T12:00:00Z".parse().unwrap(), + 2_000, + [ + segment(0, 250, " [BLANK_AUDIO]. "), + segment(250, 500, "(silence)"), + segment(-50, 750, " hello there "), + segment(1_900, 2_500, "last words"), + ], + ); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].text, "hello there"); + assert_eq!(rows[0].captured_at, "2026-07-14T12:00:00+00:00"); + assert_eq!((rows[0].start_ms, rows[0].end_ms), (0, 750)); + assert_eq!(rows[1].source, "mixed-audio"); + assert_eq!((rows[1].start_ms, rows[1].end_ms), (1_900, 2_000)); + assert_eq!(rows[1].captured_at, "2026-07-14T12:00:01.900+00:00"); + } + + #[test] + fn bounds_unicode_row_and_total_output() { + let text = "é".repeat(MAX_ROW_TEXT_BYTES); + let rows = clean_rows( + "segment-1", + "mixed-audio", + "2026-07-14T12:00:00Z".parse().unwrap(), + MAX_DURATION_MS, + (0..MAX_TRANSCRIPT_ROWS + 10) + .map(|index| segment(index as i64, index as i64 + 1, text.clone())), + ); + assert!(rows.len() <= MAX_TRANSCRIPT_ROWS); + assert!(rows.iter().all(|row| row.text.len() <= MAX_ROW_TEXT_BYTES)); + assert!(rows.iter().map(|row| row.text.len()).sum::() <= MAX_TOTAL_TEXT_BYTES); + assert!(rows + .iter() + .all(|row| row.text.is_char_boundary(row.text.len()))); + } + + #[test] + fn status_helpers_have_stable_lifecycle_fields() { + assert_eq!( + ScreenMemoryTranscriptStatus::pending().state, + ScreenMemoryTranscriptState::Pending + ); + let active = ScreenMemoryTranscriptStatus::transcribing("start"); + assert_eq!(active.started_at.as_deref(), Some("start")); + let ready = ScreenMemoryTranscriptStatus::ready(MAX_TRANSCRIPT_ROWS + 1, "start", "end"); + assert_eq!(ready.row_count, MAX_TRANSCRIPT_ROWS); + assert_eq!(ready.finished_at.as_deref(), Some("end")); + let failed = ScreenMemoryTranscriptStatus::failed("start", "end", "x".repeat(20_000)); + assert_eq!(failed.error.unwrap().len(), MAX_ROW_TEXT_BYTES); + assert_eq!( + ScreenMemoryTranscriptStatus::skipped("end").state, + ScreenMemoryTranscriptState::Skipped + ); + } +} diff --git a/templates/clips/desktop/src-tauri/src/shortcuts.rs b/templates/clips/desktop/src-tauri/src/shortcuts.rs index 3f9fe6dc45..9d2f06fcd2 100644 --- a/templates/clips/desktop/src-tauri/src/shortcuts.rs +++ b/templates/clips/desktop/src-tauri/src/shortcuts.rs @@ -1,12 +1,12 @@ use std::str::FromStr; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Mutex, OnceLock}; use std::thread; use std::time::Duration; use tauri::{AppHandle, Emitter, Listener, Manager, PhysicalPosition, PhysicalSize}; use tauri_plugin_global_shortcut::{Code, GlobalShortcutExt, Modifiers, Shortcut}; -use crate::clips::{remember_voice_target, toggle_popover}; +use crate::clips::{force_show_popover, remember_voice_target, toggle_popover}; use crate::dlog; use crate::state::{DictationActive, VoiceWakePopover}; use crate::util::{ @@ -33,6 +33,8 @@ static FN_TAP_ENABLED: AtomicBool = AtomicBool::new(false); static FN_TAP_INSTALL_STARTED: AtomicBool = AtomicBool::new(false); static POPOVER_DISMISS_SHORTCUT_ACTIVE: AtomicBool = AtomicBool::new(false); static COUNTDOWN_SHORTCUTS_ACTIVE: AtomicBool = AtomicBool::new(false); +static COUNTDOWN_SHORTCUTS_GENERATION: AtomicU64 = AtomicU64::new(0); +static COUNTDOWN_SHORTCUTS_LOCK: OnceLock> = OnceLock::new(); // P1: tracks whether Escape is currently registered *for dictation-cancel* // specifically (independent of POPOVER_DISMISS_SHORTCUT_ACTIVE, which tracks // the popover's own reason to want Escape registered). Escape should stay @@ -324,54 +326,130 @@ fn sync_dictation_escape_shortcut(app: AppHandle, active: bool) { }); } -pub fn install_countdown_shortcut_handler(app: &tauri::App) { - let handle = app.handle().clone(); - app.listen("clips:countdown-shortcuts-active", move |event| { - let payload = event.payload().to_string(); - let handle = handle.clone(); - let active: bool = serde_json::from_str(&payload).unwrap_or(false); - set_countdown_shortcuts_active(handle, active); - }); -} - -fn set_countdown_shortcuts_active(app: AppHandle, active: bool) { - COUNTDOWN_SHORTCUTS_ACTIVE.store(active, Ordering::SeqCst); - thread::spawn(move || { +/// Register the countdown shortcuts before its window becomes visible. +/// +/// The event-driven synchronizer above intentionally hops off Carbon's hotkey +/// callback stack, but that also means it cannot be used for the initial +/// activation: a fast Return at the visible `3` can otherwise beat the worker +/// thread and leave the recorder waiting for the full timer. The countdown +/// command awaits this worker before showing the window, preserving the +/// non-reentrant teardown path while making the first visible frame actionable. +pub(crate) async fn prepare_countdown_shortcuts(app: AppHandle) -> Result { + tauri::async_runtime::spawn_blocking(move || { + let _guard = COUNTDOWN_SHORTCUTS_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let generation = COUNTDOWN_SHORTCUTS_GENERATION.fetch_add(1, Ordering::SeqCst) + 1; + COUNTDOWN_SHORTCUTS_ACTIVE.store(true, Ordering::SeqCst); let gs = app.global_shortcut(); - let escape = escape_shortcut(); - let enter = enter_shortcut(); - let numpad_enter = numpad_enter_shortcut(); - if active { - for shortcut in [escape, enter, numpad_enter] { - if !gs.is_registered(shortcut) { - if let Err(err) = gs.register(shortcut) { - eprintln!("[clips-tray] failed to register countdown shortcut: {err}"); + let mut newly_registered = Vec::new(); + for shortcut in [escape_shortcut(), enter_shortcut(), numpad_enter_shortcut()] { + if !gs.is_registered(shortcut) { + if let Err(error) = gs.register(shortcut) { + for registered in newly_registered { + let _ = gs.unregister(registered); } + COUNTDOWN_SHORTCUTS_ACTIVE.store(false, Ordering::SeqCst); + return Err(format!("failed to register countdown shortcut: {error}")); } + newly_registered.push(shortcut); } - return; } + Ok::(generation) + }) + .await + .map_err(|error| format!("countdown shortcut worker stopped unexpectedly: {error}"))? +} - for shortcut in [enter, numpad_enter] { +/// Tear down only the countdown generation that the caller prepared. A late +/// cleanup from an older overlay must not unregister Return beneath the next +/// visible countdown. +pub(crate) async fn finish_countdown_shortcuts( + app: AppHandle, + generation: u64, +) -> Result<(), String> { + tauri::async_runtime::spawn_blocking(move || { + let _guard = COUNTDOWN_SHORTCUTS_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if COUNTDOWN_SHORTCUTS_GENERATION.load(Ordering::SeqCst) != generation { + return; + } + COUNTDOWN_SHORTCUTS_ACTIVE.store(false, Ordering::SeqCst); + let gs = app.global_shortcut(); + for shortcut in [enter_shortcut(), numpad_enter_shortcut()] { if gs.is_registered(shortcut) { let _ = gs.unregister(shortcut); } } + let escape = escape_shortcut(); if !POPOVER_DISMISS_SHORTCUT_ACTIVE.load(Ordering::SeqCst) && !DICTATION_ESCAPE_SHORTCUT_ACTIVE.load(Ordering::SeqCst) && gs.is_registered(escape) { let _ = gs.unregister(escape); } + }) + .await + .map_err(|error| format!("countdown shortcut cleanup worker stopped unexpectedly: {error}")) +} + +/// Observe key-down events only while Clips itself is the active application. +/// AppKit local monitors need no Input Monitoring permission and never receive +/// keystrokes destined for another app. The active-generation gate keeps this +/// listener inert outside the three-second countdown. +#[cfg(target_os = "macos")] +pub fn install_countdown_local_key_monitor(app: &tauri::App) { + use std::ptr::NonNull; + + use block2::RcBlock; + use objc2_app_kit::{NSEvent, NSEventMask}; + + let app = app.handle().clone(); + let handler = RcBlock::new(move |event: NonNull| -> *mut NSEvent { + if COUNTDOWN_SHORTCUTS_ACTIVE.load(Ordering::SeqCst) { + let key_code = unsafe { event.as_ref().keyCode() }; + let countdown_event = match key_code { + 36 | 76 => Some("clips:countdown-done"), + 53 => Some("clips:countdown-cancel"), + _ => None, + }; + if let Some(countdown_event) = countdown_event { + let app = app.clone(); + thread::spawn(move || finish_countdown_from_shortcut(&app, countdown_event)); + } + } + event.as_ptr() }); + let monitor = unsafe { + NSEvent::addLocalMonitorForEventsMatchingMask_handler(NSEventMask::KeyDown, &handler) + }; + if monitor.is_none() { + eprintln!("[clips-tray] failed to install app-local countdown key monitor"); + } } +#[cfg(not(target_os = "macos"))] +pub fn install_countdown_local_key_monitor(_app: &tauri::App) {} + fn finish_countdown_from_shortcut(app: &AppHandle, event: &'static str) { - let _ = app.emit(event, ()); + dlog!("[clips-tray] countdown shortcut accepted: {event}"); + let cause = if event == "clips:countdown-cancel" { + "escape" + } else { + "return" + }; + let _ = app.emit(event, serde_json::json!({ "cause": cause })); if let Some(window) = app.get_webview_window("countdown") { let _ = window.close(); } - set_countdown_shortcuts_active(app.clone(), false); + let app = app.clone(); + let generation = COUNTDOWN_SHORTCUTS_GENERATION.load(Ordering::SeqCst); + tauri::async_runtime::spawn(async move { + let _ = finish_countdown_shortcuts(app, generation).await; + }); } /// Build the global shortcut plugin with its handler. Called from `run()` to @@ -411,7 +489,11 @@ pub fn build_shortcut_plugin() -> tauri_plugin_global_shortcut::Builder tauri_plugin_global_shortcut::Builder>); pub struct PopoverShownAt(pub Mutex>); /// Whether a recording is currently in progress. Set from JS via -/// `set_recording_state`. Used to re-purpose the tray icon click as a -/// stop-recording shortcut while recording, matching Loom. +/// `set_recording_state`. Keeps the parked popover reachable while recording +/// and enables the explicit Stop item in the tray menu. #[derive(Default)] pub struct RecordingActive(pub Mutex); diff --git a/templates/clips/desktop/src-tauri/src/system_audio.rs b/templates/clips/desktop/src-tauri/src/system_audio.rs index 31371540c2..bee1ba8d51 100644 --- a/templates/clips/desktop/src-tauri/src/system_audio.rs +++ b/templates/clips/desktop/src-tauri/src/system_audio.rs @@ -119,7 +119,8 @@ pub async fn audio_transcription_start( owner: Option, ) -> Result<(), String> { let _ = meeting_id; - crate::whisper_speech::whisper_transcription_start( + crate::logfile::diagnostic("[meeting-audio] transcription engine start requested"); + let result = crate::whisper_speech::whisper_transcription_start( app, locale, mic_device_id, @@ -134,7 +135,16 @@ pub async fn audio_transcription_start( emit_partials.unwrap_or(true), owner, ) - .await + .await; + if let Err(error) = &result { + crate::logfile::diagnostic(&format!( + "[meeting-audio] transcription engine start failed: {error}" + )); + eprintln!("[clips-tray] meeting transcription engine start failed: {error}"); + } else { + crate::logfile::diagnostic("[meeting-audio] transcription engine started"); + } + result } #[tauri::command] @@ -221,7 +231,7 @@ pub(crate) mod macos { /// open System Settings manually). pub fn open_screen_recording_settings() -> Result<(), String> { use std::process::Command; - let url = "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture"; + let url = "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_ScreenCapture"; Command::new("open") .arg(url) .status() diff --git a/templates/clips/desktop/src-tauri/src/tray.rs b/templates/clips/desktop/src-tauri/src/tray.rs index 23a45f1b02..7a8df4f335 100644 --- a/templates/clips/desktop/src-tauri/src/tray.rs +++ b/templates/clips/desktop/src-tauri/src/tray.rs @@ -227,7 +227,10 @@ pub fn build_tray(app: &tauri::App) -> Result<(), Box> { meeting_active ); if active && !meeting_active { - let _ = app.emit("clips:recorder-stop", ()); + // Opening Clips must never double as an implicit Stop. + // Keep the recording alive and restore the parked popover; + // its active-recording view exposes an explicit Stop button. + force_show_popover(app); } else { toggle_popover(app); } diff --git a/templates/clips/desktop/src-tauri/src/util.rs b/templates/clips/desktop/src-tauri/src/util.rs index aa21d2ee6b..e91d7af245 100644 --- a/templates/clips/desktop/src-tauri/src/util.rs +++ b/templates/clips/desktop/src-tauri/src/util.rs @@ -15,10 +15,10 @@ const POPOVER_DEFAULT_HEIGHT_LOGICAL: f64 = 520.0; // // Clips-owned recording chrome (toolbar / countdown / finalizing / // recording-pill) gets `NSWindow.sharingType = NSWindowSharingNone` so it does -// not leak into the recorded video. The main popover is different: users expect -// to screenshot it for feedback, so it stays shareable during normal idle / -// settings use and is flipped to NSWindowSharingNone only while it is parked as -// the hidden recording controller. +// not leak into the recorded video. The main popover follows the same user +// preference: private by default, shareable only when "Show Clips in screen +// captures" is enabled. Reopening it during a recording must not silently +// override that choice. // Recording-time exclusion has two effects on macOS: screen pickers don't list // excluded windows, and full-screen captures omit them from the compositor // output. This is the same mechanism Loom, 1Password, and CleanShot use to keep @@ -183,12 +183,9 @@ pub fn set_capture_included(_window: &WebviewWindow) { /// immediately on anything currently on screen. Called from /// `set_feature_config` when the toggle flips. /// -/// The popover is intentionally skipped — its sharing-type is dynamic -/// (`set_capture_included` while shown, `set_capture_excluded` while -/// parked at 2x2 px during recording). Both of those helpers now consult -/// the toggle, so the next `show_popover` / `park_popover_offscreen` call -/// picks up the new setting. Rewriting it here would clobber the parked -/// state mid-recording. +/// The popover follows the same preference as ordinary Clips chrome. Applying +/// it here makes the toggle take effect immediately whether the popover is +/// visible or parked as the 2x2 recording controller. /// /// Region-guide overlays are private recorder aids, not Clips chrome demos, so /// they stay excluded even when the debug toggle makes the rest visible. @@ -198,9 +195,6 @@ pub fn reapply_capture_exclusion_to_overlays(app: &tauri::AppHandle) { let visible = crate::config::show_in_screen_capture(app); let windows = app.webview_windows(); for (label, window) in &windows { - if label.as_str() == "popover" { - continue; - } // The meeting reminder is a notification, not Clips recording // chrome — keep it visible in captures regardless of the debug // toggle so it never gets re-excluded on a config change. diff --git a/templates/clips/desktop/src-tauri/src/whisper_speech.rs b/templates/clips/desktop/src-tauri/src/whisper_speech.rs index 813aaad6ba..4893af63f5 100644 --- a/templates/clips/desktop/src-tauri/src/whisper_speech.rs +++ b/templates/clips/desktop/src-tauri/src/whisper_speech.rs @@ -16,6 +16,36 @@ //! use tauri::AppHandle; +/// One timestamped segment produced by bounded, offline-file transcription. +/// Timestamps are relative to the supplied audio, never to wall-clock time. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct OfflineTranscriptSegment { + pub start_ms: i64, + pub end_ms: i64, + pub text: String, +} + +/// Transcribe already-decoded 16 kHz mono samples with the process-wide +/// Whisper context. This is the reusable file-transcription entry point: the +/// caller owns container decoding and the samples are never persisted here. +/// +/// Blocking work -- call from a dedicated worker thread, not the async runtime. +pub(crate) fn transcribe_offline_file_samples( + app: &AppHandle, + samples: &[f32], + language: Option<&str>, +) -> Result, String> { + #[cfg(target_os = "macos")] + { + macos::transcribe_offline_file_samples(app, samples, language) + } + #[cfg(not(target_os = "macos"))] + { + let _ = (app, samples, language); + Err("local Whisper file transcription is supported on macOS only".to_owned()) + } +} + #[tauri::command] pub async fn whisper_transcription_start( app: AppHandle, @@ -112,6 +142,9 @@ mod macos { use tauri::{AppHandle, Emitter}; use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters}; + use crate::capture_audio_bus::{ + try_subscribe, AudioSources, AudioSubscription, SubscriptionAttempt, + }; use crate::native_speech::macos::{ start_raw_mic_capture, MicVoiceProcessingMode, RawMicCapture, }; @@ -121,6 +154,8 @@ mod macos { }; use crate::whisper_model::{ensure_model, model_file}; + const MEETING_AUDIO_OWNER: &str = "meeting-whisper"; + /// One transcript segment with real timestamps from whisper, already /// offset onto the meeting timeline (ms since capture start). #[derive(Serialize, Clone)] @@ -185,6 +220,31 @@ mod macos { Ok(()) } + pub(super) fn transcribe_offline_file_samples( + app: &AppHandle, + samples: &[f32], + language: Option<&str>, + ) -> Result, String> { + const CHUNK_SAMPLES: usize = 25 * 16_000; + + let ctx = context(app)?; + let mut state = ctx + .create_state() + .map_err(|e| format!("whisper state init failed: {e}"))?; + let mut segments = Vec::new(); + for (chunk_index, chunk) in samples.chunks(CHUNK_SAMPLES).enumerate() { + let chunk_offset_ms = chunk_index as i64 * 25_000; + segments.extend(infer(&mut state, chunk, language).into_iter().map( + |(start_ms, end_ms, text)| super::OfflineTranscriptSegment { + start_ms: chunk_offset_ms.saturating_add(start_ms), + end_ms: chunk_offset_ms.saturating_add(end_ms), + text, + }, + )); + } + Ok(segments) + } + // ---- resampling ------------------------------------------------------- /// One linearly-interpolated 16 kHz output sample at output index `i`, @@ -383,6 +443,12 @@ mod macos { /// deliberately deferred to the worker so we never allocate/compute on /// the realtime audio thread. fn push(&self, frames: &[f32]) { + // A bus publish may already have cloned this callback when the + // subscription is dropped. Refuse that final in-flight delivery + // once teardown has signalled the worker to stop. + if !self.running.load(Ordering::SeqCst) { + return; + } if let Ok(mut buf) = self.buf.lock() { buf.extend_from_slice(frames); } @@ -826,18 +892,31 @@ mod macos { } struct Session { + app: AppHandle, // macOS 15+ meetings use a combined SCK capture, so there is no // competing AVAudioEngine / VoiceProcessingIO mic input to stop. mic_cap: Option, // System capture is optional — skipped when the user turns system // audio off, so neither the recording nor the transcript include it. sys_cap: Option, + // When Rewind already owns the requested SCK audio sources, meeting + // Whisper receives fan-out PCM through this subscription and opens no + // physical mic/system recorder of its own. + _shared_audio: Option, + temporary_audio: Option, mic: Arc, sys: Option>, /// Who started this session — see `SessionOwner`. owner: SessionOwner, } + impl Drop for Session { + fn drop(&mut self) { + drop(self._shared_audio.take()); + release_temporary_audio(&self.app, self.temporary_audio.take()); + } + } + // SAFETY: the capture handles hold refcounted ObjC objects (already // `Send`); the streams are `Arc` over `Send + Sync` interiors. We only move // the session through the `Mutex`, never alias across threads. @@ -936,10 +1015,80 @@ mod macos { as Arc }); - let combined_cap = if should_use_combined_sck_capture( - owner, - supports_sck_microphone_capture(), - ) { + // If Rewind is running visuals-only, upgrade that one physical + // producer before touching the bus. Off/paused Rewind returns no lease + // and preserves the legacy meeting capture path. + let temporary_audio = if owner == SessionOwner::Meeting { + match crate::screen_memory::acquire_temporary_audio_consumer( + &app, + MEETING_AUDIO_OWNER, + crate::capture_graph::CaptureConsumer::Meeting, + true, + capture_system, + ) { + Ok(lease) => lease, + Err(error) => { + mic_stream.stop(); + if let Some(sys_stream) = &sys_stream { + sys_stream.stop(); + } + return Err(error); + } + } + } else { + None + }; + + // Make the shared-vs-physical choice atomically inside the bus. If a + // Rewind producer exists but lacks a requested source, `try_subscribe` + // returns a typed error and we fail closed: no combined SCK fallback, + // RawMicCapture, or raw system capture is opened. + let shared_audio = if owner == SessionOwner::Meeting { + let mic_for_shared = mic_stream.clone(); + let shared_mic = Arc::new(move |samples: &[f32], sample_rate: f64| { + mic_for_shared.set_src_rate(sample_rate); + mic_for_shared.push(samples); + }); + let shared_system = sys_stream.as_ref().map(|stream| { + let stream = stream.clone(); + Arc::new(move |samples: &[f32], sample_rate: f64| { + stream.set_src_rate(sample_rate); + stream.push(samples); + }) as crate::capture_audio_bus::AudioCallback + }); + match try_subscribe( + AudioSources::new(true, capture_system), + Some(shared_mic), + shared_system, + ) { + Ok(SubscriptionAttempt::Subscribed(subscription)) => Some(subscription), + Ok(SubscriptionAttempt::NoProducer) => { + if temporary_audio.is_some() { + release_temporary_audio(&app, temporary_audio); + mic_stream.stop(); + if let Some(sys_stream) = &sys_stream { + sys_stream.stop(); + } + return Err("shared-audio-producer-unavailable-after-upgrade".into()); + } + None + } + Err(error) => { + release_temporary_audio(&app, temporary_audio); + mic_stream.stop(); + if let Some(sys_stream) = &sys_stream { + sys_stream.stop(); + } + return Err(error.to_string()); + } + } + } else { + None + }; + + let combined_cap = if shared_audio.is_none() + && should_use_combined_sck_capture(owner, supports_sck_microphone_capture()) + { match start_raw_meeting_capture( app.clone(), mic_device_id.clone(), @@ -963,7 +1112,10 @@ mod macos { None }; - let (mic_cap, sys_cap) = if let Some(combined_cap) = combined_cap { + let (mic_cap, sys_cap) = if shared_audio.is_some() { + eprintln!("[whisper] using shared Rewind microphone/system PCM producer"); + (None, None) + } else if let Some(combined_cap) = combined_cap { // Both SCK outputs are configured at 48 kHz. mic_stream.set_src_rate(48000.0); (None, Some(combined_cap)) @@ -1004,10 +1156,30 @@ mod macos { (Some(mic_cap), sys_cap) }; - let mut slot = session_slot().lock().map_err(|e| e.to_string())?; + let mut slot = match session_slot().lock() { + Ok(slot) => slot, + Err(error) => { + drop(shared_audio); + release_temporary_audio(&app, temporary_audio); + mic_stream.stop(); + if let Some(sys_stream) = &sys_stream { + sys_stream.stop(); + } + if let Some(mic_cap) = mic_cap { + mic_cap.stop(); + } + if let Some(sys_cap) = sys_cap { + sys_cap.stop(); + } + return Err(error.to_string()); + } + }; *slot = Some(Session { + app: app.clone(), mic_cap, sys_cap, + _shared_audio: shared_audio, + temporary_audio, mic: mic_stream, sys: sys_stream, owner, @@ -1039,12 +1211,23 @@ mod macos { eprintln!("[whisper] transcription timeline reset"); } - pub fn stop(_app: &AppHandle) { + fn release_temporary_audio( + app: &AppHandle, + lease: Option, + ) { + if let Some(lease) = lease { + if let Err(error) = crate::screen_memory::release_temporary_audio_consumer(app, lease) { + eprintln!("[whisper] Rewind audio lease release failed: {error}"); + } + } + } + + pub fn stop(app: &AppHandle) { let session = match session_slot().lock() { Ok(mut slot) => slot.take(), - Err(_) => return, + Err(poisoned) => poisoned.into_inner().take(), }; - let Some(session) = session else { + let Some(mut session) = session else { return; }; // Signal workers to stop. They flush a final transcript after the loop. @@ -1053,12 +1236,16 @@ mod macos { sys.stop(); } // Stop captures so no more samples arrive while workers flush. - if let Some(mic_cap) = session.mic_cap { + if let Some(mic_cap) = session.mic_cap.take() { mic_cap.stop(); } - if let Some(sys_cap) = session.sys_cap { + if let Some(sys_cap) = session.sys_cap.take() { sys_cap.stop(); } + // Unsubscribe before a last-consumer release rotates the Rewind + // producer back to its persisted visuals-only mode. + drop(session._shared_audio.take()); + release_temporary_audio(app, session.temporary_audio.take()); // Wait up to 4 s for both workers to finish their final flush so // trailing speech is not lost when the frontend unregisters listeners. let deadline = Instant::now() + Duration::from_secs(4); diff --git a/templates/clips/desktop/src-tauri/tauri.alpha.conf.json b/templates/clips/desktop/src-tauri/tauri.alpha.conf.json new file mode 100644 index 0000000000..4fef1bdcf0 --- /dev/null +++ b/templates/clips/desktop/src-tauri/tauri.alpha.conf.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://schema.tauri.app/config/2", + "productName": "Clips Alpha", + "identifier": "com.clips.tray.alpha", + "bundle": { + "createUpdaterArtifacts": false, + "targets": ["app"], + "macOS": { + "minimumSystemVersion": "26.0" + } + }, + "plugins": { + "updater": { + "active": false + } + } +} diff --git a/templates/clips/desktop/src/app.tsx b/templates/clips/desktop/src/app.tsx index af7958244b..609ea6ae06 100644 --- a/templates/clips/desktop/src/app.tsx +++ b/templates/clips/desktop/src/app.tsx @@ -3,13 +3,18 @@ import { IconArrowLeft, IconCalendarEvent, IconCircleCheck, + IconCopy, IconDownload, IconExternalLink, IconFolderOpen, IconPencil, IconInfoCircle, + IconHistory, IconMicrophone2, + IconPlayerPlay, IconRefresh, + IconSearch, + IconShieldLock, IconTrash, IconUpload, IconX, @@ -67,7 +72,9 @@ import { import { isMacPlatform, isWindowsPlatform } from "./lib/platform"; import { dismissBrowserRecordingBackup, + createPrivateAgentRewindRecording, exportBrowserRecordingBackup, + getRewindClipOrigin, listBrowserRecordingBackups, retryBrowserRecordingBackup, scheduleNativeBackupCleanupAfterProcessing, @@ -78,6 +85,7 @@ import { type RecorderHandle, type RecorderStopResult, } from "./lib/recorder"; +import { REWIND_AGENT_PROMPT } from "./lib/rewind-agent-prompt"; import { loadBool, loadString, @@ -102,6 +110,7 @@ import { } from "./lib/voice-dictation"; import { useFeatureConfig, + type FeatureConfig, type LocalRecordingMode, type ScreenMemoryStatus, } from "./shared/config"; @@ -126,7 +135,13 @@ interface PendingNativeUpload { type PendingDesktopUpload = PendingNativeUpload | PendingBrowserRecordingUpload; -type PopoverView = "recorder" | "settings" | "meetings" | "dictation"; +type PopoverView = + | "recorder" + | "memory" + | "rewind-settings" + | "settings" + | "meetings" + | "dictation"; interface PopoverMeeting { id: string; @@ -138,6 +153,52 @@ interface PopoverMeeting { transcriptStatus: string | null; } +interface RewindMeetingHistoryAvailability { + available: boolean; + reason?: string | null; + coveredFrom?: string | null; +} + +interface RewindAgentHandoffRequest { + requestId: string; + status: "pending" | "processing" | "ready" | "declined" | "failed"; + requestedAt: string; + startAt: string; + endAt: string; + durationMs: number; + reason: string; + includeMicrophone: boolean; + includeSystemAudio: boolean; + reviewRequired: boolean; + agentClipRetention: "forever" | "24-hours" | "7-days" | "30-days"; + recordingId?: string; + agentUrl?: string; + contextUrl?: string; + expiresAt?: string; + error?: string; +} + +interface RewindExtensionRequest { + requestId: string; + recordingId: string; + seconds: 30 | 300; + status: "pending" | "processing" | "ready" | "failed"; + updatedAt: string; + preRollRecordingId?: string; + actualDurationMs?: number; + error?: string; +} + +interface NativeRewindUploadResult { + recordingId: string; + durationMs: number; +} + +interface DueRewindAgentHandoff { + requestId: string; + recordingId: string; +} + interface LocalRecordingNotice { folderPath?: string; files: LocalExportedFile[]; @@ -153,6 +214,74 @@ interface ScreenMemoryExportResult { }>; } +interface RewindEgressEvent { + requestId: string; + occurredAt: string; + state: + | "prepared" + | "completed" + | "failed" + | "local-evidence-read" + | "handoff-requested"; + evidenceCount: number; + packetBytes: number; + error?: string | null; + operation?: string | null; + receipt?: { + evidence?: Array<{ + id: string; + momentId: string; + sourceType: "app-context" | "transcript" | "ocr" | "chapter"; + capturedAt?: string | null; + }>; + frames?: Array<{ timestamp: string; segmentId: string }>; + mediaInterval?: { startAt: string; endAt: string }; + reviewRequired?: boolean; + } | null; +} + +interface RewindLocalAskResult { + query: string; + answerSummary: string; + evidence: Array<{ + id: string; + sourceType: "app-context" | "transcript" | "ocr"; + capturedAt: string; + excerpt: string; + confidence?: number | null; + segmentId: string; + offsetMs: number; + }>; + coverage: { + segmentsConsidered: number; + transcriptIndexesReady: number; + ocrIndexesReady: number; + gaps: Array<{ + kind: string; + source: string; + startedAt?: string | null; + endedAt?: string | null; + detail: string; + }>; + }; + confidence: string; + truncated: boolean; +} + +interface RewindExcludedApplication { + bundleId: string; + name: string; + path?: string | null; + installed: boolean; +} + +interface RewindAgentConnectionStatus { + client: "codex" | "claude-code"; + configured: boolean; + configPath: string; + storeDir: string; +} + type MeetingTranscriptionMode = "manual" | "ask" | "auto"; type CaptureMode = "screen" | "screen-camera" | "camera"; @@ -165,12 +294,35 @@ const STORAGE_SETUP_FAILURE_RE = const DEFAULT_SCREEN_MEMORY_CONFIG = { enabled: false, paused: false, - retentionHours: 24, + retentionHours: 8, maxBytes: 20 * 1024 * 1024 * 1024, segmentSeconds: 5 * 60, sampleIntervalSeconds: 10, + captureMode: "visuals" as const, + reviewBeforeSending: true, + autoPreviewBeforeSending: true, + agentClipRetention: "forever" as const, + excludedBundleIds: [ + "com.1password.1password", + "com.agilebits.onepassword7", + "com.bitwarden.desktop", + "com.dashlane.dashlane", + "com.lastpass.lastpass", + ], + excludePrivateWindows: false, }; +function parseExcludedBundleIds(value: string): string[] { + return [ + ...new Set( + value + .split(/[\n,]/) + .map((id) => id.trim()) + .filter(Boolean), + ), + ]; +} + function isStorageSetupFailureMessage(message: string | null | undefined) { return STORAGE_SETUP_FAILURE_RE.test(message ?? ""); } @@ -367,17 +519,17 @@ type MacosPrivacyPane = const MACOS_PRIVACY_URLS: Record = { camera: - "x-apple.systempreferences:com.apple.preference.security?Privacy_Camera", + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_Camera", microphone: - "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone", + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_Microphone", screen: - "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture", + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_ScreenCapture", speech: - "x-apple.systempreferences:com.apple.preference.security?Privacy_SpeechRecognition", + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_SpeechRecognition", accessibility: - "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility", + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_Accessibility", "input-monitoring": - "x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent", + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_ListenEvent", }; const WINDOWS_PRIVACY_URLS: Partial> = { @@ -552,6 +704,17 @@ function measurePopoverHeight(el: HTMLElement): number { const candidates = [rect.height, el.scrollHeight + borderY]; + // When a direct child is the scrollable content pane, the shell's own + // scrollHeight already includes the fixed footer but excludes content that + // is hidden inside that child. Add only that hidden delta to the shell + // baseline. Measuring the child's full height alone would omit the footer + // and leave the resumed recorder window partially clipped. + const directChildOverflow = Array.from(el.children).reduce((total, child) => { + if (!(child instanceof HTMLElement)) return total; + return total + Math.max(0, child.scrollHeight - child.clientHeight); + }, 0); + candidates.push(el.scrollHeight + borderY + directChildOverflow); + // ResizeObserver on `.app` alone misses scroll-only and absolutely // positioned growth. Measure descendant bounds so menus, banners, and // settings sections can grow the native window even when `.app` is capped @@ -563,6 +726,19 @@ function measurePopoverHeight(el: HTMLElement): number { const childRect = child.getBoundingClientRect(); if (childRect.width === 0 && childRect.height === 0) continue; lowestBottom = Math.max(lowestBottom, childRect.bottom); + + // Recorder Home deliberately keeps its footer fixed and lets the content + // region scroll when the native window is short. A newly inserted row can + // therefore grow `child.scrollHeight` without changing any descendant's + // visible bounding box. Include that hidden overflow in the desired + // window height so a state transition (for example paused -> remembering) + // can grow the popover back to its natural size. + const childBorderY = + Number.parseFloat(childStyle.borderTopWidth || "0") + + Number.parseFloat(childStyle.borderBottomWidth || "0"); + candidates.push( + childRect.top - rect.top + child.scrollHeight + childBorderY, + ); } candidates.push(lowestBottom - rect.top); @@ -711,12 +887,33 @@ export function App() { const [localRecordingNotice, setLocalRecordingNotice] = useState(null); const [popoverView, setPopoverView] = useState("recorder"); + const [rewindSettingsReturnView, setRewindSettingsReturnView] = useState< + "recorder" | "settings" + >("recorder"); + const [homeScreenMemoryStatus, setHomeScreenMemoryStatus] = + useState(null); + const [homeScreenMemoryBusy, setHomeScreenMemoryBusy] = useState(false); + const [rewindAgentPromptCopied, setRewindAgentPromptCopied] = useState(false); + const [agentHandoff, setAgentHandoff] = + useState(null); + const agentHandoffProcessingRef = useRef(null); + const agentHandoffPreviewedRef = useRef>(new Set()); + const rewindExtensionProcessingRef = useRef>(new Set()); + const [agentHandoffPreviewBusy, setAgentHandoffPreviewBusy] = useState(false); + const [agentHandoffPreviewError, setAgentHandoffPreviewError] = useState< + string | null + >(null); + const [promptRewindEnable, setPromptRewindEnable] = useState(false); const [meetings, setMeetings] = useState([]); const [meetingsLoading, setMeetingsLoading] = useState(false); const [meetingsError, setMeetingsError] = useState(null); const [meetingStartMessage, setMeetingStartMessage] = useState( null, ); + const [ + rewindMeetingHistoryAvailability, + setRewindMeetingHistoryAvailability, + ] = useState>({}); const [activeMeetingId, setActiveMeetingId] = useState(null); const [readinessOpen, setReadinessOpen] = useState( () => !loadBool(READINESS_REVIEWED_KEY, false), @@ -749,6 +946,26 @@ export function App() { const isRecording = recorder !== null; // Whether the popover window is shown; driven by the visibility effect below. const [popoverVisible, setPopoverVisible] = useState(false); + useEffect(() => { + if (featureConfig?.screenMemory?.enabled !== true) { + setHomeScreenMemoryStatus(null); + return; + } + let cancelled = false; + const refresh = () => { + invoke("screen_memory_status") + .then((status) => { + if (!cancelled) setHomeScreenMemoryStatus(status); + }) + .catch(() => {}); + }; + refresh(); + const timer = window.setInterval(refresh, popoverVisible ? 5_000 : 30_000); + return () => { + cancelled = true; + window.clearInterval(timer); + }; + }, [featureConfig?.screenMemory?.enabled, popoverVisible]); const recordShortcutHandlerRef = useRef<() => void>(() => {}); // Mirrors `bubbleActive` (assigned below once it is computed) so device // probes can synchronously tell whether the camera bubble owns the grant. @@ -1105,6 +1322,327 @@ export function App() { [serverUrl], ); + const updateAgentHandoff = useCallback( + async ( + requestId: string, + status: RewindAgentHandoffRequest["status"], + result?: Record, + error?: string, + ) => { + await invoke("screen_memory_update_agent_handoff", { + requestId, + status, + result: result ?? null, + error: error ?? null, + }); + }, + [], + ); + + const previewAgentHandoff = useCallback( + async (request: RewindAgentHandoffRequest) => { + setAgentHandoffPreviewBusy(true); + setAgentHandoffPreviewError(null); + try { + await invoke("rewind_agent_handoff_preview", { + requestId: request.requestId, + startedAt: request.startAt, + endedAt: request.endAt, + includeMic: request.includeMicrophone, + includeSystemAudio: request.includeSystemAudio, + }); + } catch (error) { + setAgentHandoffPreviewError( + error instanceof Error ? error.message : String(error), + ); + } finally { + setAgentHandoffPreviewBusy(false); + } + }, + [], + ); + + const processAgentHandoff = useCallback( + async (request: RewindAgentHandoffRequest) => { + if (agentHandoffProcessingRef.current === request.requestId) return; + agentHandoffProcessingRef.current = request.requestId; + const processing = { ...request, status: "processing" as const }; + setAgentHandoff(processing); + let recordingId: string | null = null; + try { + await updateAgentHandoff(request.requestId, "processing"); + const hasAudio = + request.includeMicrophone || request.includeSystemAudio; + const recording = await createPrivateAgentRewindRecording( + serverUrl, + hasAudio, + request.startAt, + ); + recordingId = recording.id; + await invoke("rewind_agent_handoff_upload", { + requestId: request.requestId, + startedAt: request.startAt, + endedAt: request.endAt, + serverUrl, + recordingId, + authToken: loadDesktopAuthToken(serverUrl), + cookie: typeof document !== "undefined" ? document.cookie || "" : "", + uploadMode: recording.uploadMode, + includeMic: request.includeMicrophone, + includeSystemAudio: request.includeSystemAudio, + }); + + const retentionHours = { + forever: null, + "24-hours": 24, + "7-days": 7 * 24, + "30-days": 30 * 24, + }[request.agentClipRetention]; + const autoDeleteAt = retentionHours + ? new Date( + Date.now() + retentionHours * 60 * 60 * 1_000, + ).toISOString() + : null; + await callClipsAction("update-recording", { + id: recordingId, + ...(autoDeleteAt ? { expiresAt: autoDeleteAt } : {}), + }); + const link = await callClipsAction<{ + recordingId: string; + url: string; + contextUrl: string; + expiresAt: string; + }>("create-recording-agent-link", { recordingId }); + const ready: RewindAgentHandoffRequest = { + ...processing, + status: "ready", + recordingId, + agentUrl: link.url, + contextUrl: link.contextUrl, + expiresAt: link.expiresAt, + }; + await updateAgentHandoff(request.requestId, "ready", { + recordingId, + agentUrl: link.url, + contextUrl: link.contextUrl, + expiresAt: link.expiresAt, + ...(autoDeleteAt ? { autoDeleteAt } : {}), + }); + setAgentHandoff(ready); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (recordingId) { + await callClipsAction("trash-recording", { + id: recordingId, + skipIfReady: true, + }).catch(() => {}); + } + await updateAgentHandoff( + request.requestId, + "failed", + undefined, + message, + ).catch(() => {}); + setAgentHandoff({ ...processing, status: "failed", error: message }); + } finally { + agentHandoffProcessingRef.current = null; + } + }, + [callClipsAction, serverUrl, updateAgentHandoff], + ); + + const processRewindExtension = useCallback( + async (request: RewindExtensionRequest) => { + if (rewindExtensionProcessingRef.current.has(request.requestId)) return; + rewindExtensionProcessingRef.current.add(request.requestId); + let preRollRecordingId: string | null = null; + try { + const origin = getRewindClipOrigin(request.recordingId); + if (!origin) { + throw new Error( + "Clips Alpha no longer has the local start time for this Clip.", + ); + } + const endedAtMs = Date.parse(origin.startedAt); + if (!Number.isFinite(endedAtMs)) { + throw new Error("The original Clip start time is invalid."); + } + await callClipsAction("update-rewind-extension-request", { + recordingId: request.recordingId, + requestId: request.requestId, + status: "processing", + }); + const startedAt = new Date( + endedAtMs - request.seconds * 1_000, + ).toISOString(); + const recording = await createPrivateAgentRewindRecording( + serverUrl, + origin.includeMicrophone || origin.includeSystemAudio, + startedAt, + ); + preRollRecordingId = recording.id; + const upload = await invoke( + "rewind_agent_handoff_upload", + { + requestId: `handoff-${request.requestId}`, + startedAt, + endedAt: origin.startedAt, + serverUrl, + recordingId: recording.id, + authToken: loadDesktopAuthToken(serverUrl), + cookie: + typeof document !== "undefined" ? document.cookie || "" : "", + uploadMode: recording.uploadMode, + includeMic: origin.includeMicrophone, + includeSystemAudio: origin.includeSystemAudio, + }, + ); + await callClipsAction("update-rewind-extension-request", { + recordingId: request.recordingId, + requestId: request.requestId, + status: "ready", + preRollRecordingId: recording.id, + actualDurationMs: Math.round(upload.durationMs), + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (preRollRecordingId) { + await callClipsAction("trash-recording", { + id: preRollRecordingId, + skipIfReady: true, + }).catch(() => {}); + } + await callClipsAction("update-rewind-extension-request", { + recordingId: request.recordingId, + requestId: request.requestId, + status: "failed", + error: message, + }).catch(() => {}); + } finally { + rewindExtensionProcessingRef.current.delete(request.requestId); + } + }, + [callClipsAction, serverUrl], + ); + + useEffect(() => { + if ( + authStatus !== "authed" || + featureConfig?.screenMemory?.enabled !== true + ) { + return; + } + let cancelled = false; + const poll = async () => { + const result = await callClipsAction<{ + requests?: RewindExtensionRequest[]; + }>("list-rewind-extension-requests", {}, { method: "GET" }).catch( + () => null, + ); + if (cancelled) return; + for (const request of result?.requests ?? []) { + void processRewindExtension(request); + } + }; + void poll(); + const timer = window.setInterval(() => void poll(), 3_000); + return () => { + cancelled = true; + window.clearInterval(timer); + }; + }, [ + authStatus, + callClipsAction, + featureConfig?.screenMemory?.enabled, + processRewindExtension, + ]); + + useEffect(() => { + if (featureConfig?.screenMemory?.enabled !== true || agentHandoff) return; + let cancelled = false; + const poll = () => { + invoke( + "screen_memory_next_agent_handoff", + ) + .then((request) => { + if (cancelled || !request) return; + setAgentHandoff(request); + getCurrentWindow() + .show() + .catch(() => {}); + getCurrentWindow() + .setFocus() + .catch(() => {}); + if (!request.reviewRequired) { + void processAgentHandoff(request); + } else if ( + featureConfig?.screenMemory?.autoPreviewBeforeSending !== false && + !agentHandoffPreviewedRef.current.has(request.requestId) + ) { + // Polling and config refreshes can rerender this surface. Mark the + // request before preparing QuickTime so one approval request opens + // one local preview, while leaving the manual control available. + agentHandoffPreviewedRef.current.add(request.requestId); + void previewAgentHandoff(request); + } + }) + .catch(() => {}); + }; + poll(); + const timer = window.setInterval(poll, 2_000); + return () => { + cancelled = true; + window.clearInterval(timer); + }; + }, [ + agentHandoff, + featureConfig?.screenMemory?.autoPreviewBeforeSending, + featureConfig?.screenMemory?.enabled, + previewAgentHandoff, + processAgentHandoff, + ]); + + useEffect(() => { + if (featureConfig?.screenMemory?.enabled !== true) return; + let cancelled = false; + const sweep = async () => { + const due = await invoke( + "screen_memory_due_agent_handoffs", + ).catch(() => []); + if (cancelled) return; + for (const item of due) { + try { + const cleanup = await callClipsAction<{ + deleted: boolean; + reason: string; + }>("delete-agent-recording-if-unpromoted", { + id: item.recordingId, + }); + if (cleanup.deleted) { + await invoke("screen_memory_mark_agent_handoff_deleted", { + requestId: item.requestId, + }); + } else if (cleanup.reason === "promoted") { + await invoke("screen_memory_cancel_agent_handoff_cleanup", { + requestId: item.requestId, + }); + } + } catch (error) { + console.warn( + "[clips-tray] agent-created Clip cleanup failed:", + error, + ); + } + } + }; + void sweep(); + const timer = window.setInterval(() => void sweep(), 60_000); + return () => { + cancelled = true; + window.clearInterval(timer); + }; + }, [callClipsAction, featureConfig?.screenMemory?.enabled]); + const fetchUpcomingMeetings = useCallback(async () => { if (authStatus !== "authed") { setMeetings([]); @@ -1145,22 +1683,73 @@ export function App() { } }, [authStatus, callClipsAction]); - const startMeetingNotes = useCallback((meeting: PopoverMeeting) => { - setActiveMeetingId(meeting.id); - setMeetingStartMessage(`Starting notes for ${meeting.title}…`); - emit("meetings:start-transcription", { - meetingId: meeting.id, - joinUrl: meeting.joinUrl, - reason: "user", - }).catch((err) => { - console.error("[clips-popover] start meeting notes failed:", err); - setActiveMeetingId(null); - setMeetingStartMessage("Could not start notes. Try again from Meetings."); + useEffect(() => { + let cancelled = false; + if (popoverView !== "meetings" || meetings.length === 0) { + setRewindMeetingHistoryAvailability({}); + return () => { + cancelled = true; + }; + } + Promise.all( + meetings.map(async (meeting) => { + if (!meeting.scheduledStart) + return [meeting.id, { available: false }] as const; + try { + const availability = await invoke( + "rewind_meeting_history_status", + { scheduledStart: meeting.scheduledStart }, + ); + return [meeting.id, availability] as const; + } catch (error) { + return [ + meeting.id, + { + available: false, + reason: + error instanceof Error + ? error.message + : "Earlier local meeting audio is unavailable.", + }, + ] as const; + } + }), + ).then((entries) => { + if (!cancelled) + setRewindMeetingHistoryAvailability(Object.fromEntries(entries)); }); - }, []); + return () => { + cancelled = true; + }; + }, [meetings, popoverView]); + + const startMeetingNotes = useCallback( + (meeting: PopoverMeeting, includeFromMeetingStart = false) => { + setActiveMeetingId(meeting.id); + setMeetingStartMessage( + includeFromMeetingStart + ? `Including earlier local audio for ${meeting.title}…` + : `Starting notes for ${meeting.title}…`, + ); + emit("meetings:start-transcription", { + meetingId: meeting.id, + joinUrl: meeting.joinUrl, + reason: "user", + scheduledStart: meeting.scheduledStart, + includeFromMeetingStart, + }).catch((err) => { + console.error("[clips-popover] start meeting notes failed:", err); + setActiveMeetingId(null); + setMeetingStartMessage( + "Could not start notes. Try again from Meetings.", + ); + }); + }, + [], + ); const startMeetingNotesAndJoin = useCallback( - (meeting: PopoverMeeting) => { + (meeting: PopoverMeeting, includeFromMeetingStart = false) => { if (meeting.joinUrl) { openExternal(resolveDesktopMeetingJoinUrl(meeting.joinUrl)).catch( (err) => { @@ -1168,7 +1757,7 @@ export function App() { }, ); } - startMeetingNotes(meeting); + startMeetingNotes(meeting, includeFromMeetingStart); hidePopover(); }, [startMeetingNotes], @@ -1186,7 +1775,7 @@ export function App() { useEffect(() => { invoke("get_active_meeting_id") .then((meetingId) => { - if (meetingId) setActiveMeetingId(meetingId); + if (meetingId) setActiveMeetingId((current) => current ?? meetingId); }) .catch(() => {}); @@ -1209,6 +1798,7 @@ export function App() { (event) => { if (event.payload?.meetingId) { setActiveMeetingId(event.payload.meetingId); + setMeetingStartMessage("Meeting notes are live and staying local."); } }, ), @@ -1222,11 +1812,12 @@ export function App() { ? null : current, ); + setMeetingStartMessage(null); }, ), ); track( - listen<{ meetingId?: string | null }>( + listen<{ meetingId?: string | null; error?: string }>( "meetings:transcription-error", (event) => { setActiveMeetingId((current) => @@ -1234,6 +1825,20 @@ export function App() { ? null : current, ); + setMeetingStartMessage( + event.payload?.error || "Could not start meeting notes.", + ); + }, + ), + ); + track( + listen<{ meetingId?: string | null; error?: string }>( + "meetings:history-error", + (event) => { + setMeetingStartMessage( + event.payload?.error || + "Meeting notes are live, but the earlier local audio could not be included.", + ); }, ), ); @@ -1797,7 +2402,7 @@ export function App() { const appRef = useRef(null); usePopoverAutoSize(appRef, { disabled: !popoverVisible || isRecording || recordingFlowActive, - width: popoverView === "settings" ? 440 : 360, + width: popoverView === "settings" || popoverView === "memory" ? 440 : 360, }); const loadPendingUploads = useCallback(async () => { @@ -2527,12 +3132,43 @@ export function App() { const showCameraRow = mode !== "screen"; // screen-only has no camera const showSourceRow = mode !== "camera"; // camera-only has no screen source - // During recording the popover is normally hidden — the tray click and the - // global shortcut both emit `clips:recorder-stop` directly, and the - // floating left-edge toolbar has the canonical Stop button. If the popover - // does somehow end up visible (dock reopen, global-shortcut race, etc.), - // we just render the normal pre-record panel so the user at least knows - // where they are. No recording-only UI lives here. + async function setHomeRewindRemembering(remembering: boolean) { + if (homeScreenMemoryBusy) return; + if (remembering && featureConfig?.screenMemory?.enabled !== true) { + setRewindSettingsReturnView("recorder"); + setPromptRewindEnable(true); + setPopoverView("rewind-settings"); + return; + } + setHomeScreenMemoryBusy(true); + try { + const current = await invoke("get_feature_config"); + await invoke("set_feature_config", { + config: { + ...current, + screenMemory: { + ...DEFAULT_SCREEN_MEMORY_CONFIG, + ...current.screenMemory, + // Once Rewind has been set up, the everyday Home switch is a + // pause/resume control. Full disable and capture permissions live + // in Rewind Settings, so an accidental Home click never tears + // down the configured memory system or asks for consent again. + enabled: true, + paused: !remembering, + }, + }, + }); + const status = await invoke("screen_memory_status"); + setHomeScreenMemoryStatus(status); + } catch (err) { + console.error( + "[clips-tray] update Rewind remembering state failed:", + err, + ); + } finally { + setHomeScreenMemoryBusy(false); + } + } const pendingUploadBanner = recordingStopFinalizing ? ( @@ -2550,11 +3186,179 @@ export function App() { /> ) : null; - if (popoverView === "settings") { + async function copyRewindAgentPrompt() { + try { + await navigator.clipboard.writeText(REWIND_AGENT_PROMPT); + setRewindAgentPromptCopied(true); + window.setTimeout(() => setRewindAgentPromptCopied(false), 1_500); + } catch (err) { + console.error("[clips-tray] copy Rewind agent prompt failed:", err); + } + } + + if (agentHandoff) { + const durationSeconds = Math.max( + 1, + Math.round( + (new Date(agentHandoff.endAt).getTime() - + new Date(agentHandoff.startAt).getTime()) / + 1_000, + ), + ); + return ( +
+
+
+

+ {agentHandoff.status === "pending" + ? "Review before sending" + : agentHandoff.status === "processing" + ? "Making a private Clip" + : agentHandoff.status === "ready" + ? "Clip sent to your agent" + : "Clip handoff needs attention"} +

+
+
+
+ +
+
+ {agentHandoff.reason} +

+ {new Date(agentHandoff.startAt).toLocaleString()} –{" "} + {new Date(agentHandoff.endAt).toLocaleTimeString()} ·{" "} + {durationSeconds} seconds +

+
+
+
+ +

+ Only this interval becomes a private Clip. The + rolling Rewind archive and its local paths stay on this Mac. + {agentHandoff.agentClipRetention === "forever" + ? " This Clip will be kept in your Library." + : ` It uses your ${agentHandoff.agentClipRetention.replace("-", " ")} agent-Clip retention setting.`} +

+
+ {agentHandoff.status === "pending" ? ( + <> +
+
+ Microphone + + setAgentHandoff({ ...agentHandoff, includeMicrophone }) + } + label="Include microphone audio" + /> +
+
+ Mac audio + + setAgentHandoff({ ...agentHandoff, includeSystemAudio }) + } + label="Include Mac audio" + /> +
+
+ + {agentHandoffPreviewError ? ( +

+ {agentHandoffPreviewError} +

+ ) : null} + + + + ) : agentHandoff.status === "processing" ? ( +

+ Materializing the bounded range, uploading it privately, and + preparing transcript and frame access for your agent… +

+ ) : agentHandoff.status === "ready" ? ( + <> +

+ The agent received a temporary access link. The private Clip + itself remains in your Library according to your retention + setting. +

+ + + + ) : ( + <> +

+ {agentHandoff.error || "The bounded Clip could not be sent."} +

+ + + )} +
+
+ ); + } + + if (popoverView === "memory" || popoverView === "rewind-settings") { return (
{pendingUploadBanner} + {isRecording ? : null} setPopoverView("recorder")} + onOpenRewind={() => setPopoverView("rewind-settings")} + onOpenMemory={() => setPopoverView("memory")} + onCancel={() => { + setPromptRewindEnable(false); + setPopoverView( + popoverView === "memory" + ? "rewind-settings" + : rewindSettingsReturnView, + ); + }} />
); } - if (popoverView === "meetings") { + if (popoverView === "settings") { return ( -
+
{pendingUploadBanner} - setPopoverView("recorder")} - onRefresh={fetchUpcomingMeetings} - onOpenMeetings={() => openInBrowser("/meetings")} + {isRecording ? : null} + { + saveString(STORAGE_KEY, url.replace(/\/+$/, "")); + setServerUrl(url.replace(/\/+$/, "")); + setPopoverView("recorder"); + }} + onOpenRewind={() => { + setPromptRewindEnable(false); + setRewindSettingsReturnView("settings"); + setPopoverView("rewind-settings"); + }} + onCancel={() => setPopoverView("recorder")} + /> +
+ ); + } + + if (popoverView === "meetings") { + return ( +
+ {pendingUploadBanner} + {isRecording ? : null} + setPopoverView("recorder")} + onRefresh={fetchUpcomingMeetings} + onOpenMeetings={() => openInBrowser("/meetings")} onOpenMeeting={(meetingId) => openInBrowser(`/meetings/${encodeURIComponent(meetingId)}`) } @@ -2615,6 +3472,7 @@ export function App() { return (
{pendingUploadBanner} + {isRecording ? : null} -
- - {pendingUploadBanner} +
+
+
+ - {localRecordingMode !== "off" ? ( - - ) : null} + {pendingUploadBanner} - {localRecordingNotice ? ( - setLocalRecordingNotice(null)} - onOpenFolder={() => { - if (!localRecordingNotice.folderPath) return; - invoke("open_local_recording_folder", { - path: localRecordingNotice.folderPath, - }).catch((err) => { - console.error("[clips-tray] open local folder failed:", err); - }); - }} - /> - ) : null} + {isRecording ? : null} + + {localRecordingMode !== "off" ? ( + + ) : null} -
- {showSourceRow ? ( - setLocalRecordingNotice(null)} + onOpenFolder={() => { + if (!localRecordingNotice.folderPath) return; + invoke("open_local_recording_folder", { + path: localRecordingNotice.folderPath, + }).catch((err) => { + console.error("[clips-tray] open local folder failed:", err); + }); + }} /> ) : null} - {showCameraRow ? ( +
+ {showSourceRow ? ( + + ) : null} + + {showCameraRow ? ( + { + setCameraId(id); + setCameraLabel(label); + }} + onRefresh={() => requestDeviceAccess("camera")} + on={cameraOn} + onToggle={setCameraOn} + /> + ) : null} + { - setCameraId(id); - setCameraLabel(label); + setMicId(id); + setMicLabel(label); }} - onRefresh={() => requestDeviceAccess("camera")} - on={cameraOn} - onToggle={setCameraOn} + onRefresh={() => requestDeviceAccess("mic")} + on={micOn} + onToggle={setMicOn} + systemAudio={systemAudioOn} + onSystemAudioToggle={setSystemAudioOn} + meterActive={popoverVisible && !isRecording && !recordingFlowActive} /> - ) : null} +
- { - setMicId(id); - setMicLabel(label); - }} - onRefresh={() => requestDeviceAccess("mic")} - on={micOn} - onToggle={setMicOn} - systemAudio={systemAudioOn} - onSystemAudioToggle={setSystemAudioOn} - meterActive={popoverVisible && !isRecording && !recordingFlowActive} + -
- +
+
+ +
+ + {featureConfig?.screenMemory?.enabled !== true + ? "Rewind is off" + : featureConfig.screenMemory.paused === true + ? "Rewind is paused" + : homeScreenMemoryStatus?.exclusionActive + ? "Rewind is protecting a private moment" + : "Rewind is remembering"} + + {featureConfig?.screenMemory?.enabled !== true || + featureConfig.screenMemory.paused || + homeScreenMemoryStatus?.exclusionActive ? ( +

+ {featureConfig?.screenMemory?.enabled !== true + ? "Private memory for moments you may need later." + : featureConfig.screenMemory.paused + ? "Existing local memory is still available." + : "An excluded app is being skipped."} +

+ ) : null} +
+
+ + + void setHomeRewindRemembering(remembering) + } + label={ + featureConfig?.screenMemory?.enabled === true + ? "Remember with Rewind" + : "Set up Rewind" + } + /> +
+
+
- - {recError ? ( - recError === MACOS_UPDATE_RESTART_MESSAGE ? ( - - ) : recError === MACOS_CAPTURE_PERMISSION_MESSAGE || - recError === MACOS_SCREEN_PERMISSION_MESSAGE || - recError === DESKTOP_CAPTURE_PERMISSION_MESSAGE ? ( - - ) : recError === MACOS_SPEECH_PERMISSION_MESSAGE ? ( - - ) : isStorageSetupFailureMessage(recError) ? ( - openVideoStorageSetup()} /> - ) : ( -
{recError}
- ) - ) : null} - {cameraError && !recError ? ( - cameraError === MACOS_CAPTURE_PERMISSION_MESSAGE || - cameraError === DESKTOP_CAPTURE_PERMISSION_MESSAGE ? ( - - ) : ( -
{cameraError}
- ) - ) : null} + onClick={() => { + void handleStartRecording(); + }} + > + {localRecordingMode === "off" && videoStorageStatus === "checking" + ? "Checking storage..." + : localRecordingMode === "off" + ? "Start recording" + : "Start local recording"} + + ) : null} + {recError ? ( + recError === MACOS_UPDATE_RESTART_MESSAGE ? ( + + ) : recError === MACOS_CAPTURE_PERMISSION_MESSAGE || + recError === MACOS_SCREEN_PERMISSION_MESSAGE || + recError === DESKTOP_CAPTURE_PERMISSION_MESSAGE ? ( + + ) : recError === MACOS_SPEECH_PERMISSION_MESSAGE ? ( + + ) : isStorageSetupFailureMessage(recError) ? ( + openVideoStorageSetup()} + /> + ) : ( +
{recError}
+ ) + ) : null} + {cameraError && !recError ? ( + cameraError === MACOS_CAPTURE_PERMISSION_MESSAGE || + cameraError === DESKTOP_CAPTURE_PERMISSION_MESSAGE ? ( + + ) : ( +
{cameraError}
+ ) + ) : null} +
; onBack: () => void; onRefresh: () => void; onOpenMeetings: () => void; onOpenMeeting: (meetingId: string) => void; onOpenSettings: () => void; - onStartNotes: (meeting: PopoverMeeting) => void; - onStartNotesAndJoin: (meeting: PopoverMeeting) => void; + onStartNotes: ( + meeting: PopoverMeeting, + includeFromMeetingStart?: boolean, + ) => void; + onStartNotesAndJoin: ( + meeting: PopoverMeeting, + includeFromMeetingStart?: boolean, + ) => void; onShowActiveMeeting: (meetingId: string) => void; }) { + const [includeHistoryFor, setIncludeHistoryFor] = useState>( + () => new Set(), + ); + + const consumeIncludeHistoryChoice = (meeting: PopoverMeeting): boolean => { + const include = includeHistoryFor.has(meeting.id); + // This is intentionally a one-shot choice. It never follows the next + // meeting, an automatic start, or a tray action around like a lost duck. + setIncludeHistoryFor((current) => { + const next = new Set(current); + next.delete(meeting.id); + return next; + }); + return include; + }; + return (
@@ -3604,6 +4577,24 @@ function MeetingsPopoverView({ {formatMeetingWhen(meeting)} {meeting.platform ? ` · ${meeting.platform}` : ""}
+ {canStart && rewindHistory?.available ? ( + + ) : null}
{isActive ? ( + + ); +} + // ---- inline icons (Tabler-style, monochrome, stroke=1.75) ----------------- // --------------------------------------------------------------------------- @@ -3815,6 +4836,9 @@ function desktopUpdateStatusText(status: UpdateStatus): string { } function Setup({ + surface = "settings", + promptRewindEnable = false, + recordingActive = false, initial, serverUrl, signedInAs, @@ -3834,9 +4858,14 @@ function Setup({ onVoiceProviderChange, onVoiceInstructionsChange, onConnect, + onOpenRewind, + onOpenMemory, onCancel, onSignOut, }: { + surface?: "settings" | "memory" | "rewind"; + promptRewindEnable?: boolean; + recordingActive?: boolean; initial?: string | null; serverUrl?: string; signedInAs?: string | null; @@ -3856,6 +4885,8 @@ function Setup({ onVoiceProviderChange: (value: VoiceProvider) => void; onVoiceInstructionsChange: (value: string) => void; onConnect: (url: string) => void; + onOpenRewind?: () => void; + onOpenMemory?: () => void; onCancel?: () => void; onSignOut?: () => void; }) { @@ -3869,8 +4900,26 @@ function Setup({ const autoHidePopoverEnabled = featureConfig?.autoHidePopoverEnabled === true; const showInScreenCapture = featureConfig?.showInScreenCapture === true; const localRecordingMode = featureConfig?.localRecordingMode ?? "off"; - const screenMemory = + const observedScreenMemory = featureConfig?.screenMemory ?? DEFAULT_SCREEN_MEMORY_CONFIG; + const [screenMemory, setScreenMemory] = useState(observedScreenMemory); + const [rewindConsentOpen, setRewindConsentOpen] = useState( + promptRewindEnable && observedScreenMemory.enabled !== true, + ); + const [rewindConsentMode, setRewindConsentMode] = useState< + "visuals" | "visuals-audio" + >(observedScreenMemory.captureMode ?? "visuals"); + const screenMemoryRef = useRef(observedScreenMemory); + const screenMemoryMutationRef = useRef(0); + const screenMemoryMutationVersionRef = useRef(0); + const screenMemoryMutationTailRef = useRef>(Promise.resolve()); + const [screenMemoryConfigBusy, setScreenMemoryConfigBusy] = useState(false); + + useEffect(() => { + if (screenMemoryMutationRef.current > 0) return; + screenMemoryRef.current = observedScreenMemory; + setScreenMemory(observedScreenMemory); + }, [observedScreenMemory]); const regionGuides = featureConfig?.regionGuides ?? { enabled: false, rects: [], @@ -3900,14 +4949,74 @@ function Setup({ kind: "ok" | "error"; text: string; } | null>(null); + const [screenMemoryExportResult, setScreenMemoryExportResult] = + useState(null); const [clipDraftsError, setClipDraftsError] = useState(null); const [screenMemoryBusy, setScreenMemoryBusy] = useState(false); + const [rewindEgressEvents, setRewindEgressEvents] = useState< + RewindEgressEvent[] + >([]); + const [rewindEgressOpen, setRewindEgressOpen] = useState(false); + const [rewindLocalQuery, setRewindLocalQuery] = useState(""); + const [rewindLocalResult, setRewindLocalResult] = + useState(null); + const [rewindLocalBusy, setRewindLocalBusy] = useState(false); + const [rewindLocalError, setRewindLocalError] = useState(null); + const [rewindReplayId, setRewindReplayId] = useState(null); + const [excludedBundleIdsInput, setExcludedBundleIdsInput] = useState(""); + const [excludedApps, setExcludedApps] = useState( + [], + ); + const [excludedAppsBusy, setExcludedAppsBusy] = useState(false); + const [agentConnectionBusy, setAgentConnectionBusy] = useState< + "codex" | "claude-code" | null + >(null); + const [agentConnectionMessage, setAgentConnectionMessage] = useState<{ + kind: "ok" | "error"; + text: string; + } | null>(null); const screenMemorySegments = screenMemoryStatus?.recentSegments ?? []; const screenMemoryTotalBytes = screenMemorySegments.reduce( (sum, segment) => sum + segment.bytes, 0, ); const screenMemoryRecording = screenMemoryStatus?.state === "recording"; + const captureControlsLocked = recordingActive; + + useEffect(() => { + setExcludedBundleIdsInput( + (screenMemory.excludedBundleIds ?? []).join(", "), + ); + invoke("resolve_rewind_excluded_apps", { + bundleIds: screenMemory.excludedBundleIds ?? [], + }) + .then(setExcludedApps) + .catch(() => { + setExcludedApps( + (screenMemory.excludedBundleIds ?? []).map((bundleId) => ({ + bundleId, + name: bundleId.split(".").pop() || "Application", + installed: false, + })), + ); + }); + }, [screenMemory.excludedBundleIds]); + + const excludedAppGroups = excludedApps.reduce< + Array + >((groups, app) => { + const existing = groups.find( + (candidate) => candidate.name.toLowerCase() === app.name.toLowerCase(), + ); + if (existing) { + existing.bundleIds.push(app.bundleId); + existing.installed ||= app.installed; + existing.path ||= app.path; + } else { + groups.push({ ...app, bundleIds: [app.bundleId] }); + } + return groups; + }, []); const [providerStatus, setProviderStatus] = useState(null); @@ -3982,27 +5091,89 @@ function Setup({ ); } - function setScreenMemoryConfig( - patch: Partial, + async function setScreenMemoryConfig( + patch: Partial, ) { if (!featureConfig) return; setScreenMemoryMessage(null); - invoke("set_feature_config", { - config: { - ...featureConfig, - screenMemory: { - ...DEFAULT_SCREEN_MEMORY_CONFIG, - ...screenMemory, - ...patch, - }, - }, - }).catch((err) => { + const next = { + ...DEFAULT_SCREEN_MEMORY_CONFIG, + ...screenMemoryRef.current, + ...patch, + }; + const version = ++screenMemoryMutationVersionRef.current; + screenMemoryMutationRef.current += 1; + screenMemoryRef.current = next; + setScreenMemory(next); + setScreenMemoryConfigBusy(true); + let operation!: Promise; + operation = screenMemoryMutationTailRef.current + .catch(() => {}) + .then(async () => { + // Read the latest complete config at execution time so a queued Rewind + // edit cannot overwrite an unrelated setting changed while it waited. + const current = await invoke("get_feature_config"); + await invoke("set_feature_config", { + config: { ...current, screenMemory: next }, + }); + const committed = await invoke("get_feature_config"); + if (version === screenMemoryMutationVersionRef.current) { + screenMemoryRef.current = committed.screenMemory; + setScreenMemory(committed.screenMemory); + } + }); + screenMemoryMutationTailRef.current = operation; + try { + await operation; + } catch (err) { console.error("[settings] set_feature_config failed", err); + if (version === screenMemoryMutationVersionRef.current) { + const committed = await invoke( + "get_feature_config", + ).catch(() => null); + if (committed) { + screenMemoryRef.current = committed.screenMemory; + setScreenMemory(committed.screenMemory); + } + } setScreenMemoryMessage({ kind: "error", - text: (err as Error)?.message ?? "Could not update Screen Memory.", + text: (err as Error)?.message ?? "Could not update Rewind.", }); - }); + } finally { + screenMemoryMutationRef.current = Math.max( + 0, + screenMemoryMutationRef.current - 1, + ); + if (screenMemoryMutationRef.current === 0) { + setScreenMemoryConfigBusy(false); + } + } + } + + async function installRewindAgentConnection(client: "codex" | "claude-code") { + setAgentConnectionBusy(client); + setAgentConnectionMessage(null); + try { + const status = await invoke( + "screen_memory_install_agent_connection", + { client }, + ); + setAgentConnectionMessage({ + kind: "ok", + text: `${client === "codex" ? "Codex" : "Claude Code"} is connected to this Rewind store. Restart the agent app once to load it.`, + }); + console.info( + `[clips-tray] configured ${status.client} Screen Memory MCP at ${status.configPath}`, + ); + } catch (err) { + setAgentConnectionMessage({ + kind: "error", + text: err instanceof Error ? err.message : String(err), + }); + } finally { + setAgentConnectionBusy(null); + } } function refreshScreenMemoryStatus() { @@ -4011,22 +5182,71 @@ function Setup({ .catch(() => {}); } + function refreshRewindEgressLog() { + invoke("rewind_list_evidence_egress", { limit: 20 }) + .then(setRewindEgressEvents) + .catch((err) => { + setScreenMemoryMessage({ + kind: "error", + text: + (err as Error)?.message ?? "Could not read the Rewind access log.", + }); + }); + } + + async function askRewindLocally() { + const query = rewindLocalQuery.trim(); + if (!query) return; + setRewindLocalBusy(true); + setRewindLocalError(null); + try { + const result = await invoke("rewind_local_ask", { + query, + limit: 12, + }); + setRewindLocalResult(result); + } catch (err) { + setRewindLocalError( + (err as Error)?.message ?? "Could not search local Rewind evidence.", + ); + } finally { + setRewindLocalBusy(false); + } + } + + async function replayRewindMoment( + evidence: RewindLocalAskResult["evidence"][number], + ) { + setRewindReplayId(evidence.id); + setRewindLocalError(null); + try { + await invoke("rewind_replay_moment", { + segmentId: evidence.segmentId, + offsetMs: evidence.offsetMs, + }); + } catch (err) { + setRewindLocalError( + (err as Error)?.message ?? "Could not replay this local moment.", + ); + } finally { + setRewindReplayId(null); + } + } + async function exportScreenMemoryRecent() { setScreenMemoryBusy(true); setScreenMemoryMessage(null); + setScreenMemoryExportResult(null); try { const result = await invoke( "screen_memory_export_recent", { minutes: 5 }, ); - setScreenMemoryMessage({ - kind: "ok", - text: `Saved ${result.files.length} segment${result.files.length === 1 ? "" : "s"} to ${result.folderPath}.`, - }); + setScreenMemoryExportResult(result); } catch (err) { setScreenMemoryMessage({ kind: "error", - text: (err as Error)?.message ?? "Could not export Screen Memory.", + text: err instanceof Error ? err.message : String(err), }); } finally { setScreenMemoryBusy(false); @@ -4042,11 +5262,11 @@ function Setup({ "screen_memory_delete_all", ); setScreenMemoryStatus(status); - setScreenMemoryMessage({ kind: "ok", text: "Screen Memory cleared." }); + setScreenMemoryMessage({ kind: "ok", text: "Rewind cleared." }); } catch (err) { setScreenMemoryMessage({ kind: "error", - text: (err as Error)?.message ?? "Could not clear Screen Memory.", + text: (err as Error)?.message ?? "Could not clear Rewind.", }); } finally { setScreenMemoryBusy(false); @@ -4057,8 +5277,42 @@ function Setup({ invoke("screen_memory_open_folder").catch((err) => { setScreenMemoryMessage({ kind: "error", - text: (err as Error)?.message ?? "Could not open Screen Memory folder.", + text: (err as Error)?.message ?? "Could not open Rewind folder.", + }); + }); + } + + async function chooseExcludedApplications() { + setExcludedAppsBusy(true); + setScreenMemoryMessage(null); + try { + const chosen = await invoke( + "choose_rewind_excluded_apps", + ); + if (chosen.length === 0) return; + const bundleIds = [ + ...new Set([ + ...(screenMemory.excludedBundleIds ?? []), + ...chosen.map((app) => app.bundleId), + ]), + ]; + await setScreenMemoryConfig({ excludedBundleIds: bundleIds }); + } catch (err) { + setScreenMemoryMessage({ + kind: "error", + text: (err as Error)?.message ?? "Could not choose applications.", }); + } finally { + setExcludedAppsBusy(false); + } + } + + function removeExcludedApplications(bundleIds: string[]) { + const removed = new Set(bundleIds); + void setScreenMemoryConfig({ + excludedBundleIds: (screenMemory.excludedBundleIds ?? []).filter( + (candidate) => !removed.has(candidate), + ), }); } @@ -4295,8 +5549,7 @@ function Setup({ }; }, [serverUrl, initial]); - function handleSubmit(e: React.FormEvent) { - e.preventDefault(); + function handleConnect() { const trimmed = url.trim(); if (!trimmed) return; onConnect(trimmed); @@ -4460,38 +5713,741 @@ function Setup({ }); } - return ( -
-
- {onCancel ? ( + if (surface !== "settings" && rewindConsentOpen) { + return ( +
+
+ +
+

Rewind

+

Turn on Rewind?

+

+ Rewind remembers recent moments so you can ask an agent about them or + add earlier context to a Clip. +

+
- ) : null} -

Settings

-
- -
General
- -
- - setRewindConsentMode("visuals-audio")} + > + Visuals + audio + Also remember your microphone and sound from the Mac + +
+
+ +

+ Raw Rewind recordings stay on this Mac. When you + ask an agent to search, Clips returns bounded matching context. A + media range uploads only through a private Clip handoff. +

+
+ + +
+ ); + } + + if (surface === "memory") { + return ( +
+
+ +

Manual search

+
+ {screenMemory.enabled ? ( + <> +

+ This local fallback helps you verify what Rewind remembers. For + everyday retrieval, ask Codex or your connected agent instead. +

+ { + event.preventDefault(); + void askRewindLocally(); + }} + > + + setRewindLocalQuery(event.target.value)} + placeholder="Search words you saw or heard…" + aria-label="Search memory" + maxLength={500} + /> + + + {rewindLocalError ? ( +

{rewindLocalError}

+ ) : null} + {rewindLocalResult ? ( +
+
+ + {rewindLocalResult.evidence.length} moment + {rewindLocalResult.evidence.length === 1 ? "" : "s"} found + + + {rewindLocalResult.coverage.segmentsConsidered} local + segments searched + +
+ {rewindLocalResult.evidence.length === 0 ? ( +
+ No matching moments +

+ Try an app name, a phrase you heard, or text that appeared + on screen. +

+
+ ) : ( + rewindLocalResult.evidence.map((evidence) => ( +
+
+ {new Date(evidence.capturedAt).toLocaleString()} ·{" "} + {evidence.sourceType === "ocr" + ? "On-screen text" + : evidence.sourceType === "transcript" + ? "Audio" + : "App context"} +
+

{evidence.excerpt}

+ +
+ )) + )} +
+ + Coverage and gaps + +
+

+ {rewindLocalResult.coverage.transcriptIndexesReady} audio + and {rewindLocalResult.coverage.ocrIndexesReady} visual + indexes were ready. Confidence:{" "} + {rewindLocalResult.confidence}. +

+ {rewindLocalResult.coverage.gaps.length === 0 ? ( +

+ No known capture or index gaps. +

+ ) : ( + rewindLocalResult.coverage.gaps.map((gap, index) => ( +

+ {gap.source} · {gap.detail} +

+ )) + )} +
+
+
+ ) : ( +
+ + Find the source moment +

+ Results stay grounded in retained local evidence and always + lead back to Replay. +

+
+ )} + + ) : ( +
+ + Rewind is off +

+ Turn on a private rolling memory before there is anything to + search. +

+ +
+ )} +
+ ); + } + + if (surface === "rewind") { + return ( +
+
+ +

Rewind settings

+
+
+ + { + if (enabled) setRewindConsentOpen(true); + else + void setScreenMemoryConfig({ enabled: false, paused: false }); + }} + label="Enable Rewind" + /> +
+ {captureControlsLocked ? ( +

+ Rewind capture settings unlock when this Clip ends. This keeps one + screen and audio recorder running at a time. +

+ ) : null} + {screenMemory.enabled ? ( + <> +
+ + +
+
+ + +
+
+ + +
+
+ + + {screenMemoryStatus?.exclusionActive + ? screenMemoryStatus.coverage + : screenMemoryRecording + ? `${screenMemorySegments.length} retained segment${screenMemorySegments.length === 1 ? "" : "s"} · ${formatStorageBytes(screenMemoryTotalBytes)}` + : screenMemory.paused + ? "Paused. Existing memory remains available." + : screenMemoryStatus?.lastError || + "Waiting for screen-recording permission."} + +
+
Privacy
+
+
+ + +
+ {excludedAppGroups.length > 0 ? ( +
+ {excludedAppGroups.map((app) => ( +
+ +
+ {app.name} + + {app.installed + ? "Excluded" + : "Not currently installed · still excluded"} + +
+ +
+ ))} +
+ ) : ( +

No additional apps are excluded.

+ )} +
+
Agent handoff
+
+
+ +
+
+ Set up your agent once +

+ Copy the setup prompt into any compatible agent. It installs + Rewind's reusable instructions and repairs the local + connection, so later you can simply say “Look at Rewind.” +

+
+
+
+ + Repair an agent connection + +
+

+ Usually your agent can install Rewind from the copied prompt. + These buttons are a manual repair for known clients. +

+
+ + +
+ {agentConnectionMessage ? ( +

+ {agentConnectionMessage.text} +

+ ) : null} +
+
+
+ + + void setScreenMemoryConfig({ + reviewBeforeSending: enabled, + }) + } + label="Review visual and audio ranges before sending" + /> +
+
+ + + void setScreenMemoryConfig({ + autoPreviewBeforeSending: enabled, + }) + } + label="Open a local preview automatically before sending" + /> +
+

+ Asking your agent authorizes bounded matching text. Raw Rewind + files stay local. If this review is off, an agent-requested media + range becomes a private Clip immediately and leaves a receipt. +

+
+ + +
+
+
+ +
+
+ Ask your agent +

+ Try “What was the Terminal error?” or “Replay Tuesday’s design + review.” +

+ +
+
+
{ + const open = event.currentTarget.open; + setRewindEgressOpen(open); + if (open) refreshRewindEgressLog(); + }} + > + + Agent activity + +
+

+ See when an agent searched bounded local evidence. Raw Rewind + media remains on this Mac unless you explicitly create a + private Clip. +

+ {rewindEgressEvents.length === 0 ? ( +

No matching-text requests yet.

+ ) : ( + rewindEgressEvents.slice(0, 10).map((event) => ( +

+ + {new Date(event.occurredAt).toLocaleString()} + + {` · ${event.state} · ${event.evidenceCount} item${event.evidenceCount === 1 ? "" : "s"}`} +

+ )) + )} +
+
+
+ + Manage local memory + +
+
+
+
+ Save a local Clip +

+ Export the previous five minutes as a video on this Mac. + Nothing is uploaded. +

+ {screenMemoryExportResult ? ( +
+ Saved locally + +
+ ) : null} +
+ +
+
+
+ View Rewind files +

+ Open the private folder where Rewind keeps its temporary + local memory. +

+
+ +
+
+
+ Erase Rewind memory +

+ Permanently delete all retained media and indexes from + this Mac. +

+
+ +
+
+
+
+ {screenMemoryMessage ? ( +

+ {screenMemoryMessage.text} +

+ ) : null} + + ) : ( +
+ + Nothing is being remembered +

+ Turning Rewind on begins a private rolling memory after you choose + what it may remember. +

+ +
+ )} +
+ ); + } + + return ( +
+
+ {onCancel ? ( + + ) : null} +

Settings

+
+ +
General
+ +
+ + setUrl(e.target.value)} placeholder="http://localhost:8080" /> -
@@ -4528,7 +6484,7 @@ function Setup({
Recording
+
+
+ +
+ Rewind +

+ {!screenMemory.enabled + ? "Off" + : screenMemory.paused + ? "Paused · existing local memory is still available" + : "Remembering locally"} +

+
+
+ +
+
-
+
- setScreenMemoryConfig({ enabled, paused: false }) + void setScreenMemoryConfig({ enabled, paused: false }) } - label="Enable Screen Memory" + label="Enable Rewind" + disabled={screenMemoryConfigBusy || captureControlsLocked} />
+

+ Local-only by default. Media uploads and sharing happen only when you + make a normal Clip. +

{screenMemory.enabled ? ( <>
setScreenMemoryConfig({ paused })} - label="Pause Screen Memory" + onChange={(paused) => void setScreenMemoryConfig({ paused })} + label="Pause Rewind" + disabled={screenMemoryConfigBusy || captureControlsLocked} />
@@ -4645,6 +6628,7 @@ function Setup({ Retention
+
+ + +
+

+ {screenMemory.captureMode === "visuals-audio" + ? "Visuals + audio is configured to retain microphone and system audio as separate local tracks." + : "Visuals retains screen and app context without selecting audio capture."} +

+

+ Agents receive bounded matching text when you ask. Raw Rewind + media remains local unless a bounded range becomes a private Clip. +

+ + +

+ Bundle IDs, comma-separated. Password managers are excluded by + default. Recognized bundle IDs stop media capture and discard the + entire in-flight segment; apps that do not expose a recognized + bundle ID cannot be detected. +

{screenMemoryRecording ? ( @@ -4681,22 +6748,134 @@ function Setup({ )} - {screenMemoryRecording - ? `Recording locally - ${screenMemorySegments.length} segment${screenMemorySegments.length === 1 ? "" : "s"}, ${formatStorageBytes(screenMemoryTotalBytes)}.` - : screenMemory.paused - ? "Paused. Existing local history is still available." - : screenMemoryStatus?.lastError - ? screenMemoryStatus.lastError - : "Starting when screen recording permission is available."} + {screenMemoryStatus?.exclusionActive + ? screenMemoryStatus.coverage + : screenMemoryRecording + ? `Rewind is retaining local coverage: ${screenMemorySegments.length} segment${screenMemorySegments.length === 1 ? "" : "s"}, ${formatStorageBytes(screenMemoryTotalBytes)}.` + : screenMemory.paused + ? "Capture paused. Existing local coverage remains available." + : screenMemoryStatus?.lastError + ? screenMemoryStatus.lastError + : "Capture status is waiting for screen-recording permission."}
{screenMemorySegments[0] ? (

- Latest segment:{" "} + Latest local coverage:{" "} {new Date(screenMemorySegments[0].endedAt).toLocaleTimeString()} .

) : null} +
+ +
+ setRewindLocalQuery(event.target.value)} + placeholder="What was Lilian saying about the audience?" + aria-label="Ask Rewind locally" + maxLength={500} + onKeyDown={(event) => { + if (event.key !== "Enter") return; + event.preventDefault(); + event.stopPropagation(); + void askRewindLocally(); + }} + /> + +
+ {rewindLocalError ? ( +

{rewindLocalError}

+ ) : null} + {rewindLocalResult ? ( +
+

+ Local answer:{" "} + {rewindLocalResult.answerSummary} +

+

+ Confidence: {rewindLocalResult.confidence} +

+

+ Coverage:{" "} + {rewindLocalResult.coverage.segmentsConsidered} retained + segment + {rewindLocalResult.coverage.segmentsConsidered === 1 + ? "" + : "s"} + ; {rewindLocalResult.coverage.transcriptIndexesReady}{" "} + transcript and {rewindLocalResult.coverage.ocrIndexesReady}{" "} + visual indexes ready. + {rewindLocalResult.coverage.gaps.length > 0 + ? ` ${rewindLocalResult.coverage.gaps.length} capture or index gap${rewindLocalResult.coverage.gaps.length === 1 ? "" : "s"} may hide matches.` + : " No known capture or index gaps."} +

+ {rewindLocalResult.coverage.gaps.length > 0 ? ( +
+ + Coverage gaps + +
+ {rewindLocalResult.coverage.gaps + .slice(0, 10) + .map((gap, index) => ( +

+ {gap.source} · {gap.detail} +

+ ))} +
+
+ ) : null} + {rewindLocalResult.evidence.length === 0 ? ( +

No matching evidence to show.

+ ) : ( + rewindLocalResult.evidence.map((evidence) => ( +
+

+ {evidence.sourceType} ·{" "} + {new Date(evidence.capturedAt).toLocaleString()} + {typeof evidence.confidence === "number" + ? ` · ${Math.round(evidence.confidence * 100)}% OCR confidence` + : ""} +
+ {evidence.excerpt} +

+ +
+ )) + )} + {rewindLocalResult.truncated ? ( +

+ More local matches exist; results are bounded to the + strongest 12. +

+ ) : null} +
+ ) : null} +
+
{ + const open = event.currentTarget.open; + setRewindEgressOpen(open); + if (open) refreshRewindEgressLog(); + }} + > + + Agent access log + +
+

+ Every bounded Rewind evidence request is recorded on this Mac + before matching text is returned. Raw media appears only as a + separate private Clip handoff. +

+ {rewindEgressEvents.length === 0 ? ( +

No evidence requests yet.

+ ) : ( + rewindEgressEvents.slice(0, 10).map((event) => ( +
+ + + {new Date(event.occurredAt).toLocaleString()} ·{" "} + {event.state} + + + {event.evidenceCount} item + {event.evidenceCount === 1 ? "" : "s"} + + +
+

+ {event.operation ?? "Agent access"} + {" · "}Request {event.requestId} +

+ {event.receipt?.evidence?.map((evidence) => ( +

+ {evidence.sourceType} + {evidence.capturedAt + ? ` · ${new Date(evidence.capturedAt).toLocaleTimeString()}` + : ""} +
+ Evidence {evidence.id} · moment {evidence.momentId} +

+ ))} + {event.receipt?.frames?.map((frame) => ( +

+ Local frame + {` · ${new Date(frame.timestamp).toLocaleTimeString()}`} +
+ Segment {frame.segmentId} +

+ ))} + {event.receipt?.mediaInterval ? ( +

+ Private Clip range + {` · ${new Date(event.receipt.mediaInterval.startAt).toLocaleTimeString()}–${new Date(event.receipt.mediaInterval.endAt).toLocaleTimeString()}`} +

+ ) : null} + {!event.receipt ? ( +

+ This completion record refers to the prepared + receipt with request ID {event.requestId}. +

+ ) : null} + {event.error ? ( +

{event.error}

+ ) : null} +
+
+ )) + )} +
+
{screenMemoryMessage ? (

) : null} - +
); } diff --git a/templates/clips/desktop/src/components/ReadinessPanel.tsx b/templates/clips/desktop/src/components/ReadinessPanel.tsx index 75d3d51452..76dc43b9b1 100644 --- a/templates/clips/desktop/src/components/ReadinessPanel.tsx +++ b/templates/clips/desktop/src/components/ReadinessPanel.tsx @@ -157,6 +157,26 @@ export function ReadinessPanel({ } }, []); + const requestOrOpenPermission = useCallback( + async (pane: MacosPrivacyPane) => { + if (mac && pane === "screen") { + try { + const granted = await invoke( + "system_audio_request_permission", + ); + await checkStatuses(); + if (granted) return; + } catch { + // Fall through to the dedicated privacy pane. The request API may + // be unavailable on an older macOS build or the user may already + // have denied the prompt once. + } + } + onOpenPermission(pane); + }, + [checkStatuses, mac, onOpenPermission], + ); + useEffect(() => { if (open && !statuses && mac) checkStatuses(); }, [open, statuses, mac, checkStatuses]); @@ -219,7 +239,7 @@ export function ReadinessPanel({ diff --git a/templates/clips/desktop/src/components/Switch.tsx b/templates/clips/desktop/src/components/Switch.tsx index 0098a8301e..42dbe4a4a1 100644 --- a/templates/clips/desktop/src/components/Switch.tsx +++ b/templates/clips/desktop/src/components/Switch.tsx @@ -2,10 +2,12 @@ export function Switch({ on, onChange, label, + disabled = false, }: { on: boolean; onChange: (v: boolean) => void; label: string; + disabled?: boolean; }) { return ( @@ -75,7 +75,7 @@ export function Countdown() { type="button" className="countdown-control countdown-control-skip" aria-label="Skip countdown and start recording now" - onClick={() => closeWithEvent("clips:countdown-done")} + onClick={() => closeWithEvent("clips:countdown-done", "button")} > diff --git a/templates/clips/desktop/src/overlays/onboarding.tsx b/templates/clips/desktop/src/overlays/onboarding.tsx index 92450caa96..475892490d 100644 --- a/templates/clips/desktop/src/overlays/onboarding.tsx +++ b/templates/clips/desktop/src/overlays/onboarding.tsx @@ -56,13 +56,13 @@ const PERMISSION_CARDS: Array<{ // fails — same URLs the main app window uses. const MACOS_PRIVACY_URLS: Record = { microphone: - "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone", + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_Microphone", speech: - "x-apple.systempreferences:com.apple.preference.security?Privacy_SpeechRecognition", + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_SpeechRecognition", accessibility: - "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility", + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_Accessibility", "input-monitoring": - "x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent", + "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_ListenEvent", }; function openPermissionSettings(pane: PermissionPane): void { diff --git a/templates/clips/desktop/src/overlays/preparing.tsx b/templates/clips/desktop/src/overlays/preparing.tsx new file mode 100644 index 0000000000..01562b3939 --- /dev/null +++ b/templates/clips/desktop/src/overlays/preparing.tsx @@ -0,0 +1,13 @@ +/** Visible readiness state shown while the shared Rewind Clip writer and + * resumable upload session are prepared. The numeric countdown replaces it + * only after zero can be an immediate media boundary. */ +export function Preparing() { + return ( +
+
+ +
+ ); +} diff --git a/templates/clips/desktop/src/overlays/toolbar.tsx b/templates/clips/desktop/src/overlays/toolbar.tsx index 6241ab075b..8e9e63a538 100644 --- a/templates/clips/desktop/src/overlays/toolbar.tsx +++ b/templates/clips/desktop/src/overlays/toolbar.tsx @@ -6,6 +6,7 @@ import { IconRefresh, IconTrash, } from "@tabler/icons-react"; +import { invoke } from "@tauri-apps/api/core"; import { LogicalSize } from "@tauri-apps/api/dpi"; import { emit, listen } from "@tauri-apps/api/event"; import { getCurrentWindow } from "@tauri-apps/api/window"; @@ -58,6 +59,8 @@ export function Toolbar() { // Stop / Pause are disabled until the recorder actually begins, at which // point `clips:toolbar-enabled` fires with `true` from the recorder. const [enabled, setEnabled] = useState(false); + const [preparing, setPreparing] = useState(false); + const [popoverVisible, setPopoverVisible] = useState(true); const [diskSpaceLevel, setDiskSpaceLevel] = useState< "ok" | "warning" | "critical" >("ok"); @@ -121,6 +124,7 @@ export function Toolbar() { trackListen( listen("clips:toolbar-enabled", (ev) => { setEnabled(!!ev.payload); + setPreparing(false); setPendingAction(null); if (!ev.payload) { setDiskSpaceLevel("ok"); @@ -129,6 +133,16 @@ export function Toolbar() { } }), ); + trackListen( + listen("clips:toolbar-preparing", (ev) => { + setPreparing(!!ev.payload); + }), + ); + trackListen( + listen("clips:popover-visible", (ev) => { + setPopoverVisible(!!ev.payload); + }), + ); trackListen( listen<{ freeMb: number }>("clips:disk-space-warning", () => { setDiskSpaceLevel((prev) => @@ -205,6 +219,8 @@ export function Toolbar() { // left with a zombie pill floating over their screen. The recorder // closing us first is a no-op on the already-closed window. } + + const isPreparing = preparing || (!enabled && !popoverVisible); function togglePause() { if (!enabled || pendingAction) return; const transition = paused ? "resume" : "pause"; @@ -319,13 +335,22 @@ export function Toolbar() { } data-no-drag > - {pendingAction === "stop" ? ( + {pendingAction === "stop" || isPreparing ? ( ) : ( )} -
{formatTime(elapsed)}
+ {diskSpaceLevel !== "ok" && (
.bottom-row { + flex: 0 0 auto; + padding: 7px 14px 10px; + border-top: 1px solid var(--border); + background: var(--bg); +} + /* Header — logo + mode toggle + close */ .header { @@ -709,6 +738,12 @@ body[data-clips-route="recording-pill"] #root { box-shadow: 0 0 0 2px var(--brand-ring); } +.switch:disabled { + opacity: 0.42; + filter: grayscale(0.55); + cursor: not-allowed; +} + .switch-on { background: #16a34a; } @@ -997,201 +1032,925 @@ body[data-clips-route="recording-pill"] #root { box-shadow: 0 0 0 3px var(--brand-ring); } -.readiness-item-actions { - display: flex; - align-items: center; - gap: 6px; - flex-shrink: 0; +.readiness-item-actions { + display: flex; + align-items: center; + gap: 6px; + flex-shrink: 0; +} + +.readiness-status { + display: inline-flex; + align-items: center; + justify-content: center; +} + +.readiness-status-ok { + color: var(--status-ok); +} + +.readiness-status-warn { + color: var(--status-warn); +} + +.readiness-refresh { + display: inline-flex; + align-items: center; + justify-content: center; + width: 24px; + height: 24px; + border: none; + background: transparent; + color: var(--fg-muted); + cursor: pointer; + border-radius: var(--radius-sm); + transition: + color 120ms, + background 120ms; + flex-shrink: 0; +} + +.readiness-refresh:hover { + color: var(--fg); + background: var(--surface-hover); +} + +.readiness-refresh:disabled { + opacity: 0.6; + cursor: default; +} + +.readiness-refresh-spinning svg { + animation: readiness-spin 0.8s linear infinite; +} + +@keyframes readiness-spin { + to { + transform: rotate(360deg); + } +} + +.readiness-empty { + padding: 8px; + color: var(--fg-muted); + font-size: 12px; + line-height: 1.3; +} + +.rec-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: #ef4444; + box-shadow: 0 0 0 2px rgba(239, 68, 68, 0.3); +} + +.active-recording-card { + display: grid; + grid-template-columns: auto minmax(0, 1fr); + align-items: start; + gap: 10px; + padding: 16px; + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--surface); +} + +.active-recording-dot { + margin-top: 4px; +} + +.active-recording-copy strong { + display: block; + font-size: 13px; +} + +.active-recording-copy p { + margin: 3px 0 0; + color: var(--fg-muted); + font-size: 11px; + line-height: 1.4; +} + +.active-recording-stop { + grid-column: 1 / -1; + margin-top: 4px; +} + +.active-recording-card-compact { + grid-template-columns: auto minmax(0, 1fr) auto; + align-items: center; + gap: 8px; + padding: 8px 10px; +} + +.active-recording-live { + display: inline-flex; + align-items: center; + gap: 5px; + min-height: 22px; + padding: 0 7px; + border-radius: 999px; + background: #dc2626; + color: #fff; + font-size: 9px; + font-weight: 800; + letter-spacing: 0.08em; + line-height: 1; +} + +.active-recording-live-dot { + position: relative; + width: 6px; + height: 6px; + border-radius: 50%; + background: #fff; +} + +.active-recording-live-dot::after { + content: ""; + position: absolute; + inset: -2px; + border: 1px solid rgba(255, 255, 255, 0.7); + border-radius: inherit; + animation: active-recording-pulse 1.2s ease-out infinite; +} + +@keyframes active-recording-pulse { + 0% { + transform: scale(0.8); + opacity: 1; + } + 75%, + 100% { + transform: scale(1.8); + opacity: 0; + } +} + +.active-recording-card-compact .active-recording-stop { + grid-column: auto; + margin-top: 0; + min-height: 28px; + height: 28px; + padding: 5px 10px; + font-size: 12px; +} + +@media (prefers-reduced-motion: reduce) { + .active-recording-live-dot::after { + animation: none; + opacity: 0.65; + transform: none; + } +} + +/* Active-recording state on the primary button: softened orange-red + (Tailwind orange-700) — semantic "recording is live" without the harsh + pure-red assault. White square icon + live pulsing dot still read as + "stop". Hover stays the same shade (no brighten-on-hover) to keep the + affordance calm. */ +.primary.start.rec-active { + background: #c2410c; + color: #fff; +} +.primary.start.rec-active:hover { + background: #9a3412; +} + +.rec-dot-live { + position: relative; + background: white; +} + +.rec-dot-live::after { + content: ""; + position: absolute; + inset: -1px; + border-radius: inherit; + border: 1px solid rgba(255, 255, 255, 0.6); + animation: rec-live-pulse 1.2s ease-out infinite; + pointer-events: none; +} + +@keyframes rec-live-pulse { + 0% { + transform: scale(1); + opacity: 1; + } + 70% { + transform: scale(2.4); + opacity: 0; + } + 100% { + transform: scale(2.4); + opacity: 0; + } +} + +@media (prefers-reduced-motion: reduce) { + .rec-dot-live::after { + animation: none; + opacity: 0.6; + transform: none; + } +} + +/* ------------------------------------------------------------------------- */ +/* Bottom quick-actions row */ +/* ------------------------------------------------------------------------- */ + +.bottom-row { + display: grid; + grid-template-columns: repeat(4, 1fr); + gap: 4px; +} + +.bottom-btn { + display: flex; + flex-direction: column; + align-items: center; + gap: 4px; + padding: 8px 4px; + border: none; + background: transparent; + border-radius: var(--radius-sm); + color: var(--fg-muted); + cursor: pointer; + transition: + background 120ms, + color 120ms; +} + +.bottom-btn:hover { + background: var(--surface-hover); + color: var(--fg); +} + +.bottom-icon { + position: relative; + display: inline-flex; + align-items: center; + justify-content: center; + color: var(--fg-muted); +} + +.bottom-btn:hover .bottom-icon { + color: var(--fg); +} + +.bottom-label { + font-size: 11px; + font-weight: 500; +} + +/* ------------------------------------------------------------------------- */ +/* Popover drill-in views */ +/* ------------------------------------------------------------------------- */ + +.app-popover-view { + gap: 0; + padding: 0; +} + +.popover-view { + min-height: 260px; +} + +.popover-view-header { + margin-bottom: 2px; +} + +.popover-view-header-spacer { + flex: 1; +} + +.popover-view-link { + font-size: 11px; +} + +/* Rewind is a calm everyday status first; configuration is a drill-in. */ +.rewind-home-card { + display: grid; + gap: 8px; + padding: 10px 11px; + border: 1px solid var(--border); + border-radius: var(--radius); + background: color-mix(in srgb, var(--surface) 72%, var(--bg)); +} + +.rewind-home-status { + display: grid; + grid-template-columns: auto minmax(0, 1fr) auto; + align-items: center; + gap: 9px; +} + +.rewind-settings-entry-main { + display: grid; + grid-template-columns: auto minmax(0, 1fr); + align-items: start; + gap: 9px; +} + +.rewind-home-status p { + margin: 2px 0 0; + color: var(--fg-muted); + font-size: 11px; + line-height: 1.4; +} + +.rewind-home-status strong { + font-size: 13px; + font-weight: 500; +} + +.rewind-home-lock-note, +.rewind-capture-lock-note { + margin: 0; + color: var(--fg-muted); + font-size: 10.5px; + line-height: 1.4; +} + +.rewind-capture-lock-note { + padding: 9px 10px; + border-radius: var(--radius-sm); + background: var(--surface); +} + +.rewind-home-controls { + display: inline-flex; + align-items: center; + gap: 8px; +} + +.rewind-agent-prompt-copy { + width: 28px; + height: 28px; + display: grid; + place-items: center; + padding: 0; + border: none; + border-radius: 7px; + background: transparent; + color: var(--fg-muted); + cursor: pointer; +} + +.rewind-agent-prompt-copy:hover { + background: var(--surface-hover); + color: var(--fg); +} + +.rewind-agent-prompt-copy:focus-visible { + outline: none; + box-shadow: 0 0 0 3px var(--brand-ring); +} + +.rewind-home-dot { + width: 9px; + height: 9px; + margin-top: 4px; + border-radius: 50%; + background: var(--fg-subtle); + box-shadow: 0 0 0 4px color-mix(in srgb, var(--fg-subtle) 13%, transparent); + flex-shrink: 0; +} + +.rewind-home-dot.is-live { + background: var(--status-ok); + box-shadow: 0 0 0 4px color-mix(in srgb, var(--status-ok) 15%, transparent); +} + +.rewind-home-actions { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 6px; +} + +.rewind-home-actions button { + min-width: 0; + min-height: 31px; + display: inline-flex; + align-items: center; + justify-content: center; + gap: 5px; + border: 1px solid var(--border-strong); + border-radius: var(--radius-sm); + background: var(--bg); + color: var(--fg); + cursor: pointer; + font-size: 10.5px; + font-weight: 600; +} + +.rewind-home-actions button:hover:not(:disabled) { + background: var(--surface-hover); +} + +.rewind-home-actions button:disabled { + opacity: 0.42; + cursor: default; +} + +.rewind-home-actions button:focus-visible, +.rewind-consent-choices button:focus-visible, +.rewind-consent-primary:focus-visible, +.rewind-quiet-button:focus-visible, +.rewind-search-row input:focus-visible, +.rewind-search-row button:focus-visible, +.rewind-setting-control:focus-visible, +.rewind-settings-entry .secondary:focus-visible, +.rewind-evidence-card .secondary:focus-visible, +.rewind-memory-empty .secondary:focus-visible, +.setup-back:focus-visible { + outline: none; + box-shadow: 0 0 0 3px var(--brand-ring); +} + +.rewind-home-actions .icon { + width: 14px; + height: 14px; +} + +.rewind-consent { + min-height: 430px; + align-items: center; + padding: 24px; + text-align: center; +} + +.rewind-consent-mark { + width: 48px; + height: 48px; + display: grid; + place-items: center; + border-radius: 15px; + background: var(--surface); +} + +.rewind-kicker { + margin: -4px 0 -10px; + color: var(--fg-muted); + font-size: 10px; + font-weight: 700; + letter-spacing: 0.09em; + text-transform: uppercase; +} + +.rewind-consent h2 { + margin: 0; + font-size: 20px; + letter-spacing: -0.02em; +} + +.rewind-consent-copy, +.rewind-surface-lede { + margin: 0; + color: var(--fg-muted); + font-size: 12px; + line-height: 1.5; +} + +.rewind-consent-choices { + width: 100%; + display: grid; + gap: 8px; +} + +.rewind-consent-choices button { + display: grid; + gap: 3px; + padding: 12px 13px; + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--bg); + color: var(--fg); + cursor: pointer; + text-align: left; +} + +.rewind-consent-choices button.selected { + border-color: var(--fg); + box-shadow: 0 0 0 1px var(--fg); +} + +.rewind-consent-choices span { + color: var(--fg-muted); + font-size: 11px; +} + +.rewind-local-promise { + display: grid; + grid-template-columns: auto minmax(0, 1fr); + gap: 9px; + width: 100%; + padding: 11px; + border-radius: var(--radius); + background: var(--surface); + color: var(--fg-muted); + text-align: left; +} + +.rewind-local-promise p { + margin: 0; + font-size: 11px; + line-height: 1.5; +} + +.rewind-local-promise strong { + color: var(--fg); +} + +.rewind-consent-primary { + width: 100%; + min-height: 42px; +} + +.rewind-quiet-button { + border: none; + background: transparent; + color: var(--fg-muted); + cursor: pointer; + font-size: 12px; + font-weight: 600; +} + +.rewind-memory-surface, +.rewind-settings-surface { + gap: 14px; + min-width: 0; + max-width: 100%; + overflow-x: hidden; +} + +.rewind-settings-surface { + padding-bottom: 18px; +} + +.rewind-search-row { + display: grid; + grid-template-columns: auto minmax(0, 1fr) auto; + align-items: center; + gap: 8px; + min-height: 42px; + padding-left: 11px; + border: 1px solid var(--border-strong); + border-radius: 11px; + background: var(--bg); +} + +.rewind-search-row input { + min-width: 0; + border: none; + outline: none; + background: transparent; + color: var(--fg); +} + +.rewind-search-row button { + align-self: stretch; + border: none; + border-left: 1px solid var(--border); + border-radius: 0 10px 10px 0; + background: var(--surface); + padding: 0 12px; + cursor: pointer; + font-size: 11px; + font-weight: 600; +} + +.rewind-search-row button:disabled { + opacity: 0.45; + cursor: default; +} + +.rewind-search-results { + display: grid; + gap: 8px; +} + +.rewind-result-summary { + display: flex; + justify-content: space-between; + gap: 10px; + color: var(--fg-muted); + font-size: 10.5px; +} + +.rewind-result-summary strong { + color: var(--fg); +} + +.rewind-evidence-card { + display: grid; + gap: 9px; + padding: 12px; + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--bg); +} + +.rewind-evidence-card p { + margin: 0; + line-height: 1.45; +} + +.rewind-evidence-meta { + color: var(--fg-muted); + font-size: 10.5px; + font-weight: 600; +} + +.rewind-memory-empty { + margin-top: 4px; +} + +.rewind-coverage-details { + margin-top: 2px; +} + +.rewind-setting-row { + min-height: 62px; + display: grid; + grid-template-columns: minmax(0, 1fr) 142px; + align-items: center; + gap: 14px; + border-bottom: 1px solid var(--border); +} + +.rewind-setting-row > * { + min-width: 0; +} + +.rewind-setting-control { + width: 142px; + max-width: 100%; + padding-right: 28px; +} + +.rewind-settings-status, +.rewind-boundary-note { + display: flex; + align-items: flex-start; + gap: 9px; + padding: 10px; + border-radius: var(--radius-sm); + background: var(--surface); + color: var(--fg-muted); + font-size: 11px; + line-height: 1.45; +} + +.rewind-boundary-note { + margin: -4px 0 0; +} + +.rewind-agent-row { + border-bottom: none; +} + +.rewind-excluded-apps { + display: grid; + gap: 9px; + padding: 12px 0; + border-bottom: 1px solid var(--border); +} + +.rewind-excluded-apps-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + min-width: 0; +} + +.rewind-choose-apps.secondary { + width: auto; + flex: 0 0 auto; + padding: 7px 9px; + font-size: 10.5px; +} + +.rewind-excluded-app-list { + display: grid; + gap: 5px; +} + +.rewind-excluded-app { + display: grid; + grid-template-columns: 28px minmax(0, 1fr) 26px; + align-items: center; + gap: 8px; + min-width: 0; + padding: 6px 7px; + border: 1px solid var(--border); + border-radius: var(--radius-sm); + background: var(--surface); +} + +.rewind-excluded-app.is-missing { + opacity: 0.62; +} + +.rewind-excluded-app-mark { + width: 28px; + height: 28px; + display: grid; + place-items: center; + border-radius: 7px; + background: var(--surface-strong); + color: var(--fg-muted); + font-size: 11px; + font-weight: 700; +} + +.rewind-excluded-app-copy { + display: grid; + min-width: 0; } -.readiness-status { - display: inline-flex; - align-items: center; - justify-content: center; +.rewind-excluded-app-copy strong, +.rewind-excluded-app-copy span { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; } -.readiness-status-ok { - color: var(--status-ok); +.rewind-excluded-app-copy strong { + font-size: 11px; } -.readiness-status-warn { - color: var(--status-warn); +.rewind-excluded-app-copy span { + color: var(--fg-muted); + font-size: 9.5px; } -.readiness-refresh { - display: inline-flex; - align-items: center; - justify-content: center; - width: 24px; - height: 24px; +.rewind-excluded-app-remove { + width: 26px; + height: 26px; + display: grid; + place-items: center; border: none; + border-radius: 7px; background: transparent; color: var(--fg-muted); cursor: pointer; - border-radius: var(--radius-sm); - transition: - color 120ms, - background 120ms; - flex-shrink: 0; } -.readiness-refresh:hover { - color: var(--fg); +.rewind-excluded-app-remove:hover:not(:disabled) { background: var(--surface-hover); + color: var(--fg); } -.readiness-refresh:disabled { - opacity: 0.6; - cursor: default; +.rewind-agent-guide { + display: grid; + grid-template-columns: auto minmax(0, 1fr); + gap: 9px; + padding: 11px; + border: 1px solid var(--border); + border-radius: var(--radius); + background: color-mix(in srgb, var(--surface) 78%, var(--bg)); } -.readiness-refresh-spinning svg { - animation: readiness-spin 0.8s linear infinite; +.rewind-agent-guide-icon { + width: 28px; + height: 28px; + display: grid; + place-items: center; + border-radius: 9px; + background: var(--bg); } -@keyframes readiness-spin { - to { - transform: rotate(360deg); - } +.rewind-agent-guide p { + margin: 2px 0 5px; + line-height: 1.45; } -.readiness-empty { - padding: 8px; +.rewind-text-button { + padding: 0; + border: none; + background: transparent; color: var(--fg-muted); - font-size: 12px; - line-height: 1.3; + cursor: pointer; + font-size: 10.5px; + font-weight: 600; + text-decoration: underline; + text-underline-offset: 2px; } -.rec-dot { - width: 8px; - height: 8px; - border-radius: 50%; - background: #ef4444; - box-shadow: 0 0 0 2px rgba(239, 68, 68, 0.3); +.rewind-text-button:hover { + color: var(--fg); } -/* Active-recording state on the primary button: softened orange-red - (Tailwind orange-700) — semantic "recording is live" without the harsh - pure-red assault. White square icon + live pulsing dot still read as - "stop". Hover stays the same shade (no brighten-on-hover) to keep the - affordance calm. */ -.primary.start.rec-active { - background: #c2410c; - color: #fff; -} -.primary.start.rec-active:hover { - background: #9a3412; +.rewind-detail-label { + font-size: 11px; } -.rec-dot-live { - position: relative; - background: white; +.rewind-memory-actions { + display: grid; } -.rec-dot-live::after { - content: ""; - position: absolute; - inset: -1px; - border-radius: inherit; - border: 1px solid rgba(255, 255, 255, 0.6); - animation: rec-live-pulse 1.2s ease-out infinite; - pointer-events: none; +.rewind-memory-action { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(120px, 156px); + align-items: center; + gap: 12px; + min-width: 0; + padding: 11px 0; + border-bottom: 1px solid var(--border); } -@keyframes rec-live-pulse { - 0% { - transform: scale(1); - opacity: 1; - } - 70% { - transform: scale(2.4); - opacity: 0; - } - 100% { - transform: scale(2.4); - opacity: 0; - } +.rewind-memory-action:first-child { + padding-top: 2px; } -@media (prefers-reduced-motion: reduce) { - .rec-dot-live::after { - animation: none; - opacity: 0.6; - transform: none; - } +.rewind-memory-action:last-child { + padding-bottom: 2px; + border-bottom: none; } -/* ------------------------------------------------------------------------- */ -/* Bottom quick-actions row */ -/* ------------------------------------------------------------------------- */ +.rewind-memory-action p { + margin: 3px 0 0; + color: var(--fg-muted); + font-size: 10.5px; + line-height: 1.4; +} -.bottom-row { - display: grid; - grid-template-columns: repeat(4, 1fr); - gap: 4px; +.rewind-memory-action-copy { + min-width: 0; } -.bottom-btn { +.rewind-inline-receipt { display: flex; - flex-direction: column; align-items: center; - gap: 4px; - padding: 8px 4px; - border: none; - background: transparent; - border-radius: var(--radius-sm); - color: var(--fg-muted); - cursor: pointer; - transition: - background 120ms, - color 120ms; + gap: 8px; + margin-top: 7px; + color: #15803d; + font-size: 10.5px; + font-weight: 600; } -.bottom-btn:hover { - background: var(--surface-hover); - color: var(--fg); +.rewind-inline-receipt .rewind-text-button { + color: inherit; } -.bottom-icon { - position: relative; - display: inline-flex; - align-items: center; - justify-content: center; - color: var(--fg-muted); +.rewind-agent-repair .rewind-memory-actions { + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 8px; + margin-top: 8px; } -.bottom-btn:hover .bottom-icon { - color: var(--fg); +.rewind-agent-repair .secondary { + min-width: 0; + white-space: normal; } -.bottom-label { - font-size: 11px; - font-weight: 500; +@media (max-width: 420px) { + .rewind-setting-row, + .rewind-memory-action { + grid-template-columns: minmax(0, 1fr); + align-items: stretch; + } + + .rewind-setting-control { + width: 100%; + } + + .rewind-agent-repair .rewind-memory-actions { + grid-template-columns: minmax(0, 1fr); + } } -/* ------------------------------------------------------------------------- */ -/* Popover drill-in views */ -/* ------------------------------------------------------------------------- */ +.rewind-memory-action .secondary { + width: 100%; + min-width: 0; + white-space: normal; + line-height: 1.25; +} -.app-popover-view { - gap: 0; - padding: 0; +.rewind-memory-action.is-danger strong { + color: #991b1b; } -.popover-view { - min-height: 260px; +.rewind-danger-button { + color: #b91c1c; } -.popover-view-header { - margin-bottom: 2px; +.setup .rewind-settings-entry { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + align-items: center; + gap: 12px; } -.popover-view-header-spacer { - flex: 1; +.rewind-settings-entry .secondary { + width: auto; } -.popover-view-link { - font-size: 11px; +.setup .setup-rewind-legacy { + display: none; } .popover-empty-card { @@ -1493,7 +2252,7 @@ body[data-clips-route="recording-pill"] #root { content isn't flush with the panel edge. Must follow `.setup` so it wins over the shorthand padding above. */ .setup.popover-view { - padding-bottom: 16px; + padding-bottom: 32px; } .setup h2 { @@ -1835,6 +2594,13 @@ body[data-clips-route="recording-pill"] #root { background-repeat: no-repeat; } +.setup-select:disabled { + opacity: 0.48; + color: var(--fg-muted); + background-color: var(--surface); + cursor: not-allowed; +} + .setup-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); @@ -2572,6 +3338,45 @@ body[data-clips-route="recording-pill"] #root { /* Overlay: finalizing spinner */ /* ------------------------------------------------------------------------- */ +.preparing-root { + position: fixed; + inset: 0; + display: flex; + align-items: flex-end; + justify-content: flex-start; + padding: 10px; + background: transparent; +} + +.preparing-card { + display: flex; + align-items: center; + gap: 10px; + width: 100%; + padding: 12px 14px; + border-radius: 16px; + background: rgba(20, 22, 28, 0.94); + backdrop-filter: blur(20px); + -webkit-backdrop-filter: blur(20px); + border: 1px solid rgba(255, 255, 255, 0.08); + box-shadow: + 0 14px 40px rgba(0, 0, 0, 0.45), + 0 2px 6px rgba(0, 0, 0, 0.3); + color: white; + font-size: 13px; + font-weight: 600; +} + +.preparing-spinner { + width: 18px; + height: 18px; + flex: 0 0 auto; + border-radius: 50%; + border: 2px solid rgba(255, 255, 255, 0.18); + border-top-color: #ffffff; + animation: finalizing-spin 0.9s linear infinite; +} + /* * Compact transparent window shown after the user clicks Stop. Keeping the OS * window close to the card's bounds lets its open/dismiss controls stay @@ -2962,11 +3767,28 @@ body[data-clips-route="recording-pill"] #root { .toolbar-v-time { flex: 0 0 auto; + appearance: none; + padding: 2px 4px; + border: 0; + border-radius: 6px; + background: transparent; font-variant-numeric: tabular-nums; font-size: 14px; font-weight: 600; color: white; letter-spacing: 0.3px; + cursor: pointer; +} + +.toolbar-v-time:hover, +.toolbar-v-time:focus-visible { + background: rgba(255, 255, 255, 0.1); + outline: none; +} + +.toolbar-v-time-preparing { + font-size: 9px; + letter-spacing: 0; } .toolbar-v-pause {