feat(clips): add local Rewind memory and semantic chapters

.changeset/bright-chapters-search.md

@@ -0,0 +1,6 @@
+---
+"@agent-native/core": patch
+"@agent-native/skills": patch
+---
+
+Add r5 semantic chapter metadata and provenance-aware local Screen Memory search, publish a reusable Rewind skill whose installer also repairs the local Screen Memory MCP connection, and keep the standalone skills installer aligned.

packages/core/src/cli/mcp-config-writers.spec.ts

@@ -330,6 +330,20 @@ describe("writeHttpEntryForClient", () => {
 });
 
 describe("buildLocalMcpEntryForClient", () => {
+  it("can configure a resolvable npx command for a self-installing MCP", () => {
+    expect(
+      buildLocalMcpEntryForClient(
+        "claude-code",
+        ["-y", "@agent-native/core@latest", "mcp", "screen-memory"],
+        {},
+        "npx",
+      ),
+    ).toEqual({
+      command: "npx",
+      args: ["-y", "@agent-native/core@latest", "mcp", "screen-memory"],
+    });
+  });
+
   it("uses the OpenCode local command-array shape", () => {
     expect(
       buildLocalMcpEntryForClient("opencode", ["mcp", "serve"], {

packages/core/src/cli/mcp-config-writers.ts

@@ -123,26 +123,27 @@ export function buildLocalMcpEntryForClient(
   client: ClientId,
   args: string[],
   env?: Record<string, string>,
+  command = "agent-native",
 ): Record<string, unknown> {
   const cleanEnv = env ? Object.fromEntries(Object.entries(env)) : {};
   if (client === "opencode") {
     return {
       type: "local",
-      command: ["agent-native", ...args],
+      command: [command, ...args],
       enabled: true,
       ...(Object.keys(cleanEnv).length ? { environment: cleanEnv } : {}),
     };
   }
   if (client === "github-copilot") {
     return {
       type: "stdio",
-      command: "agent-native",
+      command,
       args,
       ...(Object.keys(cleanEnv).length ? { env: cleanEnv } : {}),
     };
   }
   return {
-    command: "agent-native",
+    command,
     args,
     ...(Object.keys(cleanEnv).length ? { env: cleanEnv } : {}),
   };
@@ -543,9 +544,10 @@ export function buildCodexLocalBlock(
   name: string,
   args: string[],
   env?: Record<string, string>,
+  command = "agent-native",
 ): string {
   const lines: string[] = [codexMcpHeader(name)];
-  lines.push(`command = "agent-native"`);
+  lines.push(`command = ${tomlQuote(command)}`);
   lines.push(`args = [${args.map(tomlQuote).join(", ")}]`);
   const cleanEnv = env ? Object.fromEntries(Object.entries(env)) : {};
   if (Object.keys(cleanEnv).length) {

packages/core/src/cli/mcp-screen-memory-store.spec.ts

@@ -0,0 +1,63 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { afterEach, describe, expect, it } from "vitest";
+
+import { resolveScreenMemoryStoreDir } from "./mcp.js";
+
+const roots: string[] = [];
+
+afterEach(() => {
+  for (const root of roots.splice(0)) {
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+});
+
+function home(): string {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "clips-store-resolve-"));
+  roots.push(root);
+  return root;
+}
+
+describe("resolveScreenMemoryStoreDir", () => {
+  it("prefers an explicit directory, then the environment override", () => {
+    expect(
+      resolveScreenMemoryStoreDir({
+        explicitDir: "/tmp/explicit-clips-store",
+        env: { CLIPS_SCREEN_MEMORY_DIR: "/tmp/env-clips-store" },
+      }),
+    ).toBe("/tmp/explicit-clips-store");
+    expect(
+      resolveScreenMemoryStoreDir({
+        env: { AGENT_NATIVE_SCREEN_MEMORY_DIR: "/tmp/env-clips-store" },
+      }),
+    ).toBe("/tmp/env-clips-store");
+  });
+
+  it("discovers whichever Clips store was most recently active", () => {
+    const root = home();
+    const appData = path.join(root, "Library", "Application Support");
+    const stable = path.join(appData, "com.clips.tray", "screen-memory");
+    const alpha = path.join(appData, "com.clips.tray.alpha", "screen-memory");
+    fs.mkdirSync(stable, { recursive: true });
+    fs.mkdirSync(alpha, { recursive: true });
+    const now = new Date();
+    fs.utimesSync(
+      stable,
+      new Date(now.getTime() - 60_000),
+      new Date(now.getTime() - 60_000),
+    );
+    fs.utimesSync(alpha, now, now);
+
+    expect(
+      resolveScreenMemoryStoreDir({ platform: "darwin", homeDir: root }),
+    ).toBe(alpha);
+  });
+
+  it("returns undefined when Rewind has not created a store", () => {
+    expect(
+      resolveScreenMemoryStoreDir({ platform: "darwin", homeDir: home() }),
+    ).toBeUndefined();
+  });
+});

packages/core/src/cli/mcp.ts

@@ -15,6 +15,7 @@
 
 import crypto from "node:crypto";
 import fs from "node:fs";
+import os from "node:os";
 import path from "node:path";
 
 import { MCP_PUBLIC_ROUTE_PREFIX } from "../mcp/route-paths.js";
@@ -42,6 +43,7 @@ import {
 } from "./mcp-config-writers.js";
 
 const SERVER_NAME_PREFIX = "agent-native";
+const SCREEN_MEMORY_SERVER_NAME = "clips-screen-memory";
 
 interface ParsedArgs {
   _: string[];
@@ -83,6 +85,58 @@ function logOut(msg: string): void {
   process.stdout.write(`${msg}\n`);
 }
 
+export interface ScreenMemoryStoreResolutionOptions {
+  explicitDir?: string;
+  env?: NodeJS.ProcessEnv;
+  platform?: NodeJS.Platform;
+  homeDir?: string;
+}
+
+/**
+ * Resolve the active Clips store without asking people to find an app-data
+ * path. Environment overrides remain the unambiguous escape hatch; otherwise
+ * the most recently touched installed Clips/Clips Alpha store wins.
+ */
+export function resolveScreenMemoryStoreDir(
+  options: ScreenMemoryStoreResolutionOptions = {},
+): string | undefined {
+  if (options.explicitDir) return path.resolve(options.explicitDir);
+  const env = options.env ?? process.env;
+  const override =
+    env.CLIPS_SCREEN_MEMORY_DIR || env.AGENT_NATIVE_SCREEN_MEMORY_DIR;
+  if (override) return path.resolve(override);
+
+  const platform = options.platform ?? process.platform;
+  const home = options.homeDir ?? os.homedir();
+  const appDataRoot =
+    platform === "darwin"
+      ? path.join(home, "Library", "Application Support")
+      : platform === "win32"
+        ? env.APPDATA || path.join(home, "AppData", "Roaming")
+        : env.XDG_DATA_HOME || path.join(home, ".local", "share");
+  return ["com.clips.tray", "com.clips.tray.alpha"]
+    .map((bundleId) => path.join(appDataRoot, bundleId, "screen-memory"))
+    .filter((candidate) => fs.existsSync(candidate))
+    .map((candidate) => ({
+      candidate,
+      modifiedAt: Math.max(
+        fs.statSync(candidate).mtimeMs,
+        ...["feature-config.json", "chapters.json"]
+          .map((name) =>
+            name === "feature-config.json"
+              ? path.join(path.dirname(candidate), name)
+              : path.join(candidate, name),
+          )
+          .filter((file) => fs.existsSync(file))
+          .map((file) => fs.statSync(file).mtimeMs),
+      ),
+    }))
+    .sort(
+      (a, b) =>
+        b.modifiedAt - a.modifiedAt || a.candidate.localeCompare(b.candidate),
+    )[0]?.candidate;
+}
+
 // ---------------------------------------------------------------------------
 // .env token provisioning (local dev) — hand-rolled idempotent upsert
 // ---------------------------------------------------------------------------
@@ -343,6 +397,38 @@ function installForClient(
   return file;
 }
 
+export function installScreenMemoryForClient(
+  client: ClientId,
+  storeDir: string,
+  cwd: string,
+  scope: string | undefined,
+): string {
+  const file = configPathFor(client, cwd, scope);
+  const args = [
+    "-y",
+    "@agent-native/core@latest",
+    "mcp",
+    "screen-memory",
+    "--dir",
+    path.resolve(storeDir),
+  ];
+  if (client === "codex") {
+    writeCodexBlock(
+      file,
+      SCREEN_MEMORY_SERVER_NAME,
+      buildCodexLocalBlock(SCREEN_MEMORY_SERVER_NAME, args, {}, "npx"),
+    );
+  } else {
+    writeJsonMcpEntryForClient(
+      client,
+      file,
+      SCREEN_MEMORY_SERVER_NAME,
+      buildLocalMcpEntryForClient(client, args, {}, "npx"),
+    );
+  }
+  return file;
+}
+
 function uninstallForClient(
   client: ClientId,
   appId: string,
@@ -454,6 +540,36 @@ async function cmdInstall(p: ParsedArgs): Promise<void> {
   logOut(`  Restart ${client} to pick up the new MCP server.`);
 }
 
+function cmdInstallScreenMemory(p: ParsedArgs): void {
+  const client = normalizeClientId(p.client);
+  if (!client) {
+    logErr(
+      `Usage: npx @agent-native/core@latest mcp install-screen-memory --client ${SELECTABLE_CLIENTS.join("|")} [--dir <path>] [--scope user|project]`,
+    );
+    process.exit(1);
+  }
+  const screenMemoryDir = resolveScreenMemoryStoreDir({
+    explicitDir: p.screenMemoryDir,
+  });
+  if (!screenMemoryDir) {
+    logErr(
+      "No local Clips Screen Memory store was found. Turn Rewind on in Clips, or pass --dir <path>.",
+    );
+    process.exit(1);
+  }
+  const file = installScreenMemoryForClient(
+    client,
+    screenMemoryDir,
+    process.cwd(),
+    p.scope,
+  );
+  logOut(`Installed \"${SCREEN_MEMORY_SERVER_NAME}\" for ${client} → ${file}`);
+  logOut("  Store: current local Clips Rewind memory");
+  logOut(
+    `  Restart ${client} to pick up the repaired Screen Memory MCP server.`,
+  );
+}
+
 function cmdUninstall(p: ParsedArgs): void {
   const client = normalizeClientId(p.client);
   if (!client) {
@@ -544,6 +660,11 @@ Usage:
       Run the local Clips Screen Memory stdio server.
       Defaults to the Clips app-data screen-memory folder.
 
+  npx @agent-native/core@latest mcp install-screen-memory --client <c> [--dir <path>] [--scope user|project]
+      Install or repair the dedicated local Screen Memory MCP. The active Clips
+      or Clips Alpha store is discovered automatically; --dir is an override.
+      Clients: claude-code, codex, cowork, cursor, opencode, github-copilot
+
   npx @agent-native/core@latest mcp install --client <c> [--app <id>] [--scope user|project]
       Provision a token and write the client's MCP config (idempotent).
       Clients: claude-code, codex, cowork, cursor, opencode, github-copilot
@@ -571,6 +692,9 @@ export async function runMcp(args: string[]): Promise<void> {
     case "install":
       await cmdInstall(p);
       return;
+    case "install-screen-memory":
+      cmdInstallScreenMemory(p);
+      return;
     case "uninstall":
       cmdUninstall(p);
       return;

packages/core/src/cli/skills-content/help.ts

@@ -2,13 +2,14 @@ export const HELP = `npx @agent-native/core@latest skills
 
 Usage:
   npx @agent-native/core@latest skills list
-  npx @agent-native/core@latest skills status [assets|content|design-exploration|visual-edit|visual-plan|visual-recap|visualize-repo|context-xray|scaffold] [--client codex|claude-code|pi|all] [--scope user|project] [--json]
-  npx @agent-native/core@latest skills update [assets|content|design-exploration|visual-edit|visual-plan|visual-recap|visualize-repo|context-xray|scaffold] [--client codex|claude-code|pi|all] [--scope user|project] [--dry-run] [--json]
-  npx @agent-native/core@latest skills add assets|content|design-exploration|visual-edit|visual-plan|visual-recap|visualize-repo|context-xray [--client codex|claude-code|cowork|cursor|opencode|github-copilot|all] [--scope user|project] [--mode hosted|local-files|self-hosted] [--mcp-url <url>] [--no-connect] [--with-github-action] [--yes] [--dry-run] [--json]
+  npx @agent-native/core@latest skills status [assets|content|rewind|design-exploration|visual-edit|visual-plan|visual-recap|visualize-repo|context-xray|scaffold] [--client codex|claude-code|pi|all] [--scope user|project] [--json]
+  npx @agent-native/core@latest skills update [assets|content|rewind|design-exploration|visual-edit|visual-plan|visual-recap|visualize-repo|context-xray|scaffold] [--client codex|claude-code|pi|all] [--scope user|project] [--dry-run] [--json]
+  npx @agent-native/core@latest skills add assets|content|rewind|design-exploration|visual-edit|visual-plan|visual-recap|visualize-repo|context-xray [--client codex|claude-code|cowork|cursor|opencode|github-copilot|all] [--scope user|project] [--mode hosted|local-files|self-hosted] [--mcp-url <url>] [--no-connect] [--with-github-action] [--yes] [--dry-run] [--json]
   npx @agent-native/core@latest skills add <manifest-or-app-dir|skill-repo> [--skill <name>] [--client ...] [--yes]
 
 Examples:
   npx @agent-native/core@latest skills add assets
+  npx @agent-native/core@latest skills add rewind --client codex --scope user --yes
   npx @agent-native/core@latest skills add content --mode local-files
   npx @agent-native/core@latest skills add design-exploration
   npx @agent-native/core@latest skills add visual-edit

packages/core/src/cli/skills-content/index.ts

@@ -8,6 +8,7 @@
 export * from "./help.js";
 export * from "./assets-skill.js";
 export * from "./content-skill.js";
+export * from "./rewind-skill.js";
 export * from "./design-exploration-skill.js";
 export * from "./design-visual-edit-skill.js";
 export * from "./plan-setup-auth.js";

packages/core/src/cli/skills-content/rewind-skill.ts

@@ -0,0 +1,52 @@
+export const REWIND_SKILL_MD = `---
+name: rewind
+description: >-
+  Retrieve recent local Clips Rewind context when the user says "Look at
+  Rewind," asks what just happened, or refers to something they recently said
+  or saw.
+metadata:
+  visibility: exported
+---
+
+# Rewind
+
+Use Clips Rewind as local screen memory. Start broad enough to find the right
+moment, then read only the smallest relevant range.
+
+## Retrieval Flow
+
+1. Call \`screen_memory_status\` first. If the newest segment is still open,
+   wait for it to finalize rather than substituting an older segment.
+2. Search \`screen_memory_search_chapters\` before requesting raw recent
+   context. Use the user's words, the visible app or project, and the likely
+   time range as clues.
+3. If several chapters plausibly match, show the candidates and ask which one
+   the user means. Do not blend separate workstreams together.
+4. Read the smallest useful range with \`screen_memory_recent_context\`.
+   Restate what you recovered before acting, and flag transcription or coverage
+   uncertainty.
+5. Use \`screen_memory_frame_at\` for one exact visual moment or
+   \`screen_memory_contact_sheet\` to scan a bounded range. Prefer local frames
+   before escalating to cloud processing.
+6. Request the smallest relevant timestamp range through Clips' bounded private
+   Clip handoff only when local text and frames are insufficient, such as
+   garbled speech, important motion, dense analysis, or a Clip the user wants
+   to keep and query later.
+
+## Boundaries
+
+- Rewind recordings, screenshots, audio, transcripts, OCR, and indexes remain
+  local unless the user explicitly asks for a bounded Clip handoff.
+- Do not reveal archive filesystem paths, crawl Clips' app-data folders, or
+  bypass the Screen Memory MCP broker.
+- Do not upload frames returned by local Screen Memory tools.
+- Treat foreground apps and chapter labels as evidence, not proof of intent.
+- If the Screen Memory MCP is missing, explain that the one-time setup needs to
+  be repaired with:
+
+  \`npx -y @agent-native/core@latest skills add rewind --client <client> --scope user --yes\`
+
+  Replace \`<client>\` with the current compatible host: \`codex\`,
+  \`claude-code\`, \`cursor\`, \`opencode\`, \`github-copilot\`, or \`cowork\`.
+  Ask the user to restart the host if it cannot reload MCP servers in place.
+`;