Add supabase/mcp as a pinned submodule + SUPABASE_MCP_SERVER_PATH local-build override

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "submodules/agent-skills"]
 	path = submodules/agent-skills
 	url = git@github.com:supabase/agent-skills.git
+[submodule "submodules/mcp"]
+	path = submodules/mcp
+	url = git@github.com:supabase/mcp.git

README.md

@@ -58,6 +58,35 @@ pnpm eval -- \
 
 `--suite`, `--experiment-suite`, `--experiment`, and `--eval` accept multiple inputs via repeated flags as well as comma-separated values.
 
+### Running against an exact MCP server revision
+
+Evals launch `@supabase/mcp-server-supabase` via npx, pinned to the version in
+`MCP_SERVER_VERSION` (`packages/core/src/index.ts`). To eval an exact source
+revision instead — an unpublished branch, a PR, or the pinned submodule —
+build it and run against the build:
+
+```bash
+pnpm mcp:build   # init submodules/mcp + pnpm install + build (rerun after source edits)
+pnpm eval:local-mcp -- --eval <id> --experiment <exp>
+```
+
+`eval:local-mcp` is `pnpm eval` with `SUPABASE_MCP_SERVER_PATH` prefilled to
+the submodule's server package. To test a different revision, check out any
+commit inside `submodules/mcp` and `pnpm mcp:build` again — the gitlink pin
+only moves when a bump is committed here. Containerized agents (Claude Code,
+Codex) see the build through a read-only bind mount of the checkout, so a
+host-side rebuild is picked up by the next run with no extra copying.
+
+For a build outside the submodule, set `SUPABASE_MCP_SERVER_PATH` yourself: it
+accepts a package dir (launches `dist/transports/stdio.js`) or a direct
+`.js`/`.mjs`/`.cjs` entrypoint path; relative paths resolve against this
+repo's root. The path is existence-checked at config time, so an unbuilt
+checkout fails fast with a pointer back here.
+
+Note the pin tracks the published npm package's source (via its release tag),
+not the hosted production deployment — the prod deploy commit is separate
+provenance that this repo does not record.
+
 Run all benchmark and no-skills experiments across all benchmark evals:
 
 ```bash

apps/framework/harness/run-eval.ts

@@ -32,6 +32,7 @@ import {
   buildSkillResult,
   rehydrateTruncatedDocsResults,
   getExperimentDisplayMetadata,
+  supabaseMcpServerMounts,
 } from '@supabase-evals/core';
 import type {
   ExperimentConfig,
@@ -439,6 +440,7 @@ async function runOne(
           // (the session folds the discovery listing into its promptAddendum),
           // so no skill text is injected into the prompt here.
           skills: skillSources,
+          mounts: supabaseMcpServerMounts(),
         })
       );
 
@@ -507,7 +509,12 @@ async function runOne(
     // platform-lite via host.docker.internal (so platform-lite binds 0.0.0.0).
     // An in-process agent runs host-side with no sandbox.
     await using cliSandbox = agentRunsInSandbox
-      ? disposable(await createBareSandbox({ skills: skillSources }))
+      ? disposable(
+          await createBareSandbox({
+            skills: skillSources,
+            mounts: supabaseMcpServerMounts(),
+          })
+        )
       : undefined;
     await using session = disposable(
       await exp.runtime.startSession({

package.json

@@ -9,6 +9,8 @@
     "eval:dry": "pnpm --filter @supabase-evals/framework eval:dry",
     "eval:smoke": "pnpm --filter @supabase-evals/framework eval:smoke",
     "eval:force": "pnpm --filter @supabase-evals/framework eval:force",
+    "eval:local-mcp": "SUPABASE_MCP_SERVER_PATH=submodules/mcp/packages/mcp-server-supabase pnpm eval",
+    "mcp:build": "git submodule update --init submodules/mcp && pnpm --dir submodules/mcp install && pnpm --dir submodules/mcp build",
     "test:framework": "pnpm --filter @supabase-evals/framework test:framework",
     "export-results": "pnpm --filter @supabase-evals/framework export-results",
     "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck",

packages/core/src/index.ts

@@ -1,11 +1,17 @@
 import vm from 'node:vm';
 import { createRequire } from 'node:module';
 import { createHash, createHmac } from 'node:crypto';
-import { execFile } from 'node:child_process';
+import { execFile, execFileSync } from 'node:child_process';
 import { createServer } from 'node:net';
 import { promisify } from 'node:util';
-import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
-import { basename, dirname, join } from 'node:path';
+import {
+  existsSync,
+  mkdtempSync,
+  readFileSync,
+  realpathSync,
+  rmSync,
+} from 'node:fs';
+import { basename, dirname, join, resolve } from 'node:path';
 import { tmpdir } from 'node:os';
 import { fileURLToPath } from 'node:url';
 import type { ToolName } from './transcript/types.js';
@@ -402,6 +408,20 @@ export type AgentHarness = {
  */
 export type SkillSource = { name: string; dir: string };
 
+/**
+ * A host directory bind-mounted into the agent sandbox. Read-only by default;
+ * mounted at the identical container path unless `containerPath` overrides it
+ * (identical paths let one command config work on both host and container).
+ */
+export type SandboxMount = {
+  /** Host directory to mount. */
+  hostPath: string;
+  /** Mount point inside the container; defaults to `hostPath`. */
+  containerPath?: string;
+  /** Mount read-only (default true). */
+  readonly?: boolean;
+};
+
 export type LocalStackSessionArgs = {
   /** Supabase CLI version this scenario requires, overriding the runtime default. */
   cliVersion?: string;
@@ -437,6 +457,12 @@ export type LocalStackSessionArgs = {
    * instead, so they ignore this.
    */
   skills?: readonly SkillSource[];
+  /**
+   * Extra host directories to bind-mount into the sandbox (read-only by
+   * default) — e.g. a local MCP server build the in-container agent must be
+   * able to launch. See `supabaseMcpServerMounts`.
+   */
+  mounts?: readonly SandboxMount[];
 };
 
 /** A mocked hosted project (platform-lite) the sandbox CLI is linked to. */
@@ -895,8 +921,9 @@ export function supabaseMcpServer(
   return {
     name: 'supabase-mcp',
     async createConfig({ apiUrl, accessToken } = {}) {
-      const args = [
-        `@supabase/mcp-server-supabase@${version}`,
+      // Server flags are identical whether we launch the published package via
+      // npx or a local build directly with node.
+      const serverArgs = [
         // The server refuses to boot without a token; with only platform-
         // independent features (docs) it never authenticates against the
         // management API, so a well-formed throwaway is enough.
@@ -908,12 +935,111 @@ export function supabaseMcpServer(
       // Only point the server at a platform when one is given. `docs` is
       // platform-independent (it queries the public docs GraphQL API), so a
       // docs-only server runs standalone with no `--api-url`.
-      if (apiUrl) args.push('--api-url', apiUrl);
-      return { config: { command: 'npx', args } };
+      if (apiUrl) serverArgs.push('--api-url', apiUrl);
+
+      const local = resolveLocalMcpServer();
+      if (local) {
+        // `node`, not process.execPath: CLI agents run this command INSIDE the
+        // sandbox container, where the host's node binary path does not exist.
+        // Both container and host resolve `node` via PATH.
+        return {
+          config: { command: 'node', args: [local.entry, ...serverArgs] },
+        };
+      }
+
+      return {
+        config: {
+          command: 'npx',
+          args: [`@supabase/mcp-server-supabase@${version}`, ...serverArgs],
+        },
+      };
     },
   };
 }
 
+/**
+ * SUPABASE_MCP_SERVER_PATH swaps the published npx package for a local build
+ * (a repo/package dir or a direct .js/.mjs/.cjs entrypoint), so a workspace
+ * can test an unpublished server change without publishing to npm. Relative
+ * paths resolve against the evals checkout root (not the process CWD), so
+ * `submodules/mcp/packages/mcp-server-supabase` works from any directory.
+ *
+ * Memoized per env value: createConfig and the sandbox mounts both resolve,
+ * and each resolution spawns git (anchor + mount root) — cache so repeat
+ * calls within a run cost nothing. Keyed on the raw env string because tests
+ * (and in principle callers) change it between calls; the not-found error
+ * path is deliberately uncached so a fixed build is picked up on retry.
+ */
+type LocalMcpServer = { entry: string; baseDir: string; mountRoot: string };
+let localMcpServerCache: { key: string; value: LocalMcpServer } | null = null;
+
+function resolveLocalMcpServer(): LocalMcpServer | null {
+  const localServerPath = process.env.SUPABASE_MCP_SERVER_PATH;
+  if (!localServerPath) return null;
+  if (localMcpServerCache?.key === localServerPath)
+    return localMcpServerCache.value;
+
+  const anchor =
+    gitToplevel(dirname(fileURLToPath(import.meta.url))) ?? process.cwd();
+  const isEntryFile = /\.[cm]?js$/.test(localServerPath);
+  const base = resolve(anchor, localServerPath);
+  const probe = isEntryFile
+    ? base
+    : join(base, 'dist', 'transports', 'stdio.js');
+  if (!existsSync(probe)) {
+    throw new Error(
+      `SUPABASE_MCP_SERVER_PATH resolved to ${probe}, which does not exist — ` +
+        `build the server first (pnpm install && pnpm build in the mcp checkout); ` +
+        `see README "Running against an exact MCP server revision".`
+    );
+  }
+  // One filesystem view for command AND mount: the sandbox bind-mounts the
+  // realpath (Docker resolves sources against the daemon's view), so the
+  // command must reference the same view — an override under a symlinked dir
+  // (macOS /tmp -> /private/tmp) would otherwise exec a path that does not
+  // exist in-container. Canonicalize the BASE once and derive the entry from
+  // it (never realpath the entry separately: a symlinked dist/ target could
+  // resolve outside the mounted baseDir).
+  const realBase = realpathSync(base);
+  const baseDir = isEntryFile ? dirname(realBase) : realBase;
+  const value: LocalMcpServer = {
+    entry: isEntryFile
+      ? realBase
+      : join(realBase, 'dist', 'transports', 'stdio.js'),
+    baseDir,
+    // The whole git toplevel (not just dist/) because the build is unbundled:
+    // it requires its node_modules at runtime.
+    mountRoot: gitToplevel(baseDir) ?? baseDir,
+  };
+  localMcpServerCache = { key: localServerPath, value };
+  return value;
+}
+
+function gitToplevel(dir: string): string | null {
+  try {
+    return execFileSync('git', ['rev-parse', '--show-toplevel'], {
+      cwd: dir,
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    }).trim();
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Sandbox mounts required to launch the SUPABASE_MCP_SERVER_PATH build inside
+ * a containerized agent sandbox. A CLI agent's MCP command runs INSIDE the
+ * container, where the host build is invisible — so the build's checkout is
+ * bind-mounted read-only at its identical (real) path, letting the same
+ * config work on both sides, with host rebuilds visible immediately (no
+ * re-copy). Empty when unset.
+ */
+export function supabaseMcpServerMounts(): SandboxMount[] {
+  const local = resolveLocalMcpServer();
+  return local ? [{ hostPath: local.mountRoot, readonly: true }] : [];
+}
+
 export function executorMcpServer(): McpServerDefinition {
   return {
     name: 'executor-mcp',