From 9235091f77edf6e0c0eb7506ab4df6f16a7f4425 Mon Sep 17 00:00:00 2001 From: eval-workspace Date: Thu, 23 Jul 2026 13:39:47 +0200 Subject: [PATCH 1/6] evals-mcp-local-build-override --- packages/core/src/index.ts | 42 ++++++++++++++-- packages/core/src/mcp-server.test.ts | 72 ++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 packages/core/src/mcp-server.test.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index a4a9dab5..a358d4a6 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -5,7 +5,7 @@ import { execFile } from "node:child_process"; import { createServer } from "node:net"; import { promisify } from "node:util"; import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; -import { basename, dirname, join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; import type { ToolName } from "./transcript/types.js"; @@ -892,8 +892,11 @@ export function supabaseMcpServer( return { name: "supabase-mcp", async createConfig({ apiUrl, accessToken } = {}) { - const args = [ - `@supabase/mcp-server-supabase@${version}`, + const localServerPath = process.env.SUPABASE_MCP_SERVER_PATH; + + // Server flags are identical whether we launch the published package via + // npx or a local build directly with node. + const serverArgs = [ // The server refuses to boot without a token; with only platform- // independent features (docs) it never authenticates against the // management API, so a well-formed throwaway is enough. @@ -905,8 +908,37 @@ export function supabaseMcpServer( // Only point the server at a platform when one is given. `docs` is // platform-independent (it queries the public docs GraphQL API), so a // docs-only server runs standalone with no `--api-url`. - if (apiUrl) args.push("--api-url", apiUrl); - return { config: { command: "npx", args } }; + if (apiUrl) serverArgs.push("--api-url", apiUrl); + + // SUPABASE_MCP_SERVER_PATH swaps the published npx package for a local + // build (a repo/package dir or a direct .js/.mjs/.cjs entrypoint), so a + // workspace can test an unpublished server change without publishing to + // npm. Relative paths resolve against the eval process CWD — prefer + // absolute paths. + if (localServerPath) { + const entry = resolve( + /\.[cm]?js$/.test(localServerPath) + ? localServerPath + : join(localServerPath, "dist", "transports", "stdio.js"), + ); + if (!existsSync(entry)) { + throw new Error( + `SUPABASE_MCP_SERVER_PATH resolved to ${entry}, which does not exist — ` + + `build the server first (pnpm install && pnpm build in the mcp checkout); ` + + `see README "Running against an exact MCP server revision".`, + ); + } + return { + config: { command: process.execPath, args: [entry, ...serverArgs] }, + }; + } + + return { + config: { + command: "npx", + args: [`@supabase/mcp-server-supabase@${version}`, ...serverArgs], + }, + }; }, }; } diff --git a/packages/core/src/mcp-server.test.ts b/packages/core/src/mcp-server.test.ts new file mode 100644 index 00000000..67e67838 --- /dev/null +++ b/packages/core/src/mcp-server.test.ts @@ -0,0 +1,72 @@ +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { MCP_SERVER_VERSION, supabaseMcpServer } from "./index.js"; + +// Stub (not mutate) env so pre-existing SUPABASE_* values are restored per test. +function clearEnv() { + vi.stubEnv("SUPABASE_MCP_SERVER_PATH", undefined); +} + +// A real on-disk build layout: the override path is existence-checked, so the +// fixtures must actually exist for the happy paths (and not for the error one). +let fixtureDir: string; +let fixtureEntry: string; +beforeAll(() => { + fixtureDir = mkdtempSync(join(tmpdir(), "mcp-override-")); + fixtureEntry = join(fixtureDir, "dist", "transports", "stdio.js"); + mkdirSync(join(fixtureDir, "dist", "transports"), { recursive: true }); + writeFileSync(fixtureEntry, ""); +}); +afterAll(() => rmSync(fixtureDir, { recursive: true, force: true })); + +describe("supabaseMcpServer().createConfig", () => { + afterEach(() => vi.unstubAllEnvs()); + + it("defaults to the published package via npx", async () => { + clearEnv(); + const { config } = await supabaseMcpServer().createConfig({ + apiUrl: "http://api.test", + }); + expect(config.command).toBe("npx"); + expect(config.args[0]).toBe( + `@supabase/mcp-server-supabase@${MCP_SERVER_VERSION}`, + ); + expect(config.args).toContain("--api-url"); + }); + + it("launches a local build dir with node when SUPABASE_MCP_SERVER_PATH is set", async () => { + clearEnv(); + vi.stubEnv("SUPABASE_MCP_SERVER_PATH", fixtureDir); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.command).toBe(process.execPath); + expect(config.args[0]).toBe(fixtureEntry); + }); + + it("uses a direct .js override path as-is", async () => { + clearEnv(); + vi.stubEnv("SUPABASE_MCP_SERVER_PATH", fixtureEntry); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.args[0]).toBe(fixtureEntry); + }); + + it("preserves --api-url on the local override path", async () => { + clearEnv(); + vi.stubEnv("SUPABASE_MCP_SERVER_PATH", fixtureDir); + const { config } = await supabaseMcpServer().createConfig({ + apiUrl: "http://api.test", + }); + const i = config.args.indexOf("--api-url"); + expect(i).toBeGreaterThan(-1); + expect(config.args[i + 1]).toBe("http://api.test"); + }); + + it("fails fast with an actionable error when the override path does not exist", async () => { + clearEnv(); + vi.stubEnv("SUPABASE_MCP_SERVER_PATH", join(fixtureDir, "not-built")); + await expect(supabaseMcpServer().createConfig({})).rejects.toThrow( + /does not exist.*build the server first/s, + ); + }); +}); From 90aeb81c8e7d40847866672d5b2cb9187b2363cc Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Thu, 23 Jul 2026 11:34:04 +0200 Subject: [PATCH 2/6] add supabase/mcp as a pinned submodule at submodules/mcp Pinned at mcp-server-supabase-v0.8.1 (5a8d965), the release tag for the npm package version the harness runs (MCP_SERVER_VERSION = 0.8.1). Note: this identifies the SOURCE of package 0.8.1 via the release-please tag; npm publishes no gitHead for cryptographic proof, and the hosted (prod) deployment commit is separate, unverified provenance. Sits next to submodules/agent-skills; build with pnpm install && pnpm build inside the submodule and point the harness at it to eval an exact mcp revision instead of the npx download. --- .gitmodules | 3 +++ submodules/mcp | 1 + 2 files changed, 4 insertions(+) create mode 160000 submodules/mcp diff --git a/.gitmodules b/.gitmodules index cc410ee6..094026de 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "submodules/agent-skills"] path = submodules/agent-skills url = git@github.com:supabase/agent-skills.git +[submodule "submodules/mcp"] + path = submodules/mcp + url = git@github.com:supabase/mcp.git diff --git a/submodules/mcp b/submodules/mcp new file mode 160000 index 00000000..5a8d9652 --- /dev/null +++ b/submodules/mcp @@ -0,0 +1 @@ +Subproject commit 5a8d9652ab975b9f99e622feb296f1d946994dee From 7d2e7f61435983565b2d270cf9bb3310228a3d3f Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Thu, 23 Jul 2026 11:44:40 +0200 Subject: [PATCH 3/6] README: document evaling an exact MCP revision via the pinned submodule --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index 7729428e..17e573b7 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,34 @@ pnpm eval -- \ `--suite`, `--experiment-suite`, `--experiment`, and `--eval` accept multiple inputs via repeated flags as well as comma-separated values. +### Running against an exact MCP server revision + +Evals launch `@supabase/mcp-server-supabase` via npx, pinned to the version in +`MCP_SERVER_VERSION` (`packages/core/src/index.ts`). To eval an exact source +revision instead — an unpublished branch, a PR, or the pinned submodule — +build it and point the harness at the build: + +```bash +# submodules/mcp is pinned at the release tag for the npx version above +git submodule update --init submodules/mcp +(cd submodules/mcp && pnpm install && pnpm build) + +SUPABASE_MCP_SERVER_PATH="$PWD/submodules/mcp/packages/mcp-server-supabase" \ + pnpm eval -- --eval --experiment +``` + +`SUPABASE_MCP_SERVER_PATH` accepts a package dir (launches +`dist/transports/stdio.js`) or a direct `.js`/`.mjs`/`.cjs` entrypoint path; +prefer absolute paths (relative ones resolve against the eval process CWD). +The path is existence-checked at config time, so an unbuilt checkout fails +fast with a pointer back here. To test a different revision, check out any +commit inside `submodules/mcp` and rebuild — the gitlink pin only moves when +a bump is committed here. + +Note the pin tracks the published npm package's source (via its release tag), +not the hosted production deployment — the prod deploy commit is separate +provenance that this repo does not record. + Run all benchmark and no-skills experiments across all benchmark evals: ```bash From 3fd0aa7770e59bb675dae811a0ccad1c4a4c0798 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 08:19:55 +0200 Subject: [PATCH 4/6] fix: expose the local MCP server build to containerized agents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CLI agent's MCP command runs inside the sandbox container, where the SUPABASE_MCP_SERVER_PATH build (and the host's node binary path) don't exist — the agent silently got no MCP tools. Now: - supabaseMcpServerMounts() (core) resolves the override to its git checkout root and the sandbox bind-mounts it read-only at the identical container path, so one config works on both sides and host rebuilds are picked up with no re-copy. Whole checkout, not dist/: the build is unbundled and needs its node_modules at runtime. - The override config launches with `node` (PATH) instead of process.execPath, which pointed at a host-only binary in-container. - Relative override paths resolve against the evals checkout root instead of the process CWD, and `pnpm mcp:build` + `pnpm eval:local-mcp` prefill the submodule path (review DX feedback). Verified: core 56/56 + sandbox 31/31 + framework typecheck; docker smoke boots a sandbox with the mount and runs the built stdio.js --version in-container (0.8.1), write probe rejected (ro). --- README.md | 29 +++--- apps/framework/harness/run-eval.ts | 9 +- package.json | 2 + packages/core/src/index.ts | 107 +++++++++++++++----- packages/core/src/mcp-server.test.ts | 58 ++++++++++- packages/sandbox/src/agent-environment.ts | 9 +- packages/sandbox/src/bare-sandbox.ts | 9 +- packages/sandbox/src/docker-sandbox.ts | 18 ++++ packages/sandbox/src/local-stack-runtime.ts | 2 + 9 files changed, 198 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 17e573b7..fb1e021c 100644 --- a/README.md +++ b/README.md @@ -63,24 +63,25 @@ pnpm eval -- \ Evals launch `@supabase/mcp-server-supabase` via npx, pinned to the version in `MCP_SERVER_VERSION` (`packages/core/src/index.ts`). To eval an exact source revision instead — an unpublished branch, a PR, or the pinned submodule — -build it and point the harness at the build: +build it and run against the build: ```bash -# submodules/mcp is pinned at the release tag for the npx version above -git submodule update --init submodules/mcp -(cd submodules/mcp && pnpm install && pnpm build) - -SUPABASE_MCP_SERVER_PATH="$PWD/submodules/mcp/packages/mcp-server-supabase" \ - pnpm eval -- --eval --experiment +pnpm mcp:build # init submodules/mcp + pnpm install + build (rerun after source edits) +pnpm eval:local-mcp -- --eval --experiment ``` -`SUPABASE_MCP_SERVER_PATH` accepts a package dir (launches -`dist/transports/stdio.js`) or a direct `.js`/`.mjs`/`.cjs` entrypoint path; -prefer absolute paths (relative ones resolve against the eval process CWD). -The path is existence-checked at config time, so an unbuilt checkout fails -fast with a pointer back here. To test a different revision, check out any -commit inside `submodules/mcp` and rebuild — the gitlink pin only moves when -a bump is committed here. +`eval:local-mcp` is `pnpm eval` with `SUPABASE_MCP_SERVER_PATH` prefilled to +the submodule's server package. To test a different revision, check out any +commit inside `submodules/mcp` and `pnpm mcp:build` again — the gitlink pin +only moves when a bump is committed here. Containerized agents (Claude Code, +Codex) see the build through a read-only bind mount of the checkout, so a +host-side rebuild is picked up by the next run with no extra copying. + +For a build outside the submodule, set `SUPABASE_MCP_SERVER_PATH` yourself: it +accepts a package dir (launches `dist/transports/stdio.js`) or a direct +`.js`/`.mjs`/`.cjs` entrypoint path; relative paths resolve against this +repo's root. The path is existence-checked at config time, so an unbuilt +checkout fails fast with a pointer back here. Note the pin tracks the published npm package's source (via its release tag), not the hosted production deployment — the prod deploy commit is separate diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index eab5b516..f7b0e063 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -32,6 +32,7 @@ import { buildSkillResult, rehydrateTruncatedDocsResults, getExperimentDisplayMetadata, + supabaseMcpServerMounts, } from "@supabase-evals/core"; import type { ExperimentConfig, @@ -439,6 +440,7 @@ async function runOne( // (the session folds the discovery listing into its promptAddendum), // so no skill text is injected into the prompt here. skills: skillSources, + mounts: supabaseMcpServerMounts(), }), ); @@ -507,7 +509,12 @@ async function runOne( // platform-lite via host.docker.internal (so platform-lite binds 0.0.0.0). // An in-process agent runs host-side with no sandbox. await using cliSandbox = agentRunsInSandbox - ? disposable(await createBareSandbox({ skills: skillSources })) + ? disposable( + await createBareSandbox({ + skills: skillSources, + mounts: supabaseMcpServerMounts(), + }), + ) : undefined; await using session = disposable( await exp.runtime.startSession({ diff --git a/package.json b/package.json index cf70715a..c8f53133 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,8 @@ "eval:dry": "pnpm --filter @supabase-evals/framework eval:dry", "eval:smoke": "pnpm --filter @supabase-evals/framework eval:smoke", "eval:force": "pnpm --filter @supabase-evals/framework eval:force", + "eval:local-mcp": "SUPABASE_MCP_SERVER_PATH=submodules/mcp/packages/mcp-server-supabase pnpm eval", + "mcp:build": "git submodule update --init submodules/mcp && pnpm --dir submodules/mcp install && pnpm --dir submodules/mcp build", "test:framework": "pnpm --filter @supabase-evals/framework test:framework", "export-results": "pnpm --filter @supabase-evals/framework export-results", "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck", diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index a358d4a6..a174e96c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,10 +1,10 @@ import vm from "node:vm"; import { createRequire } from "node:module"; import { createHash, createHmac } from "node:crypto"; -import { execFile } from "node:child_process"; +import { execFile, execFileSync } from "node:child_process"; import { createServer } from "node:net"; import { promisify } from "node:util"; -import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, realpathSync, rmSync } from "node:fs"; import { basename, dirname, join, resolve } from "node:path"; import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; @@ -399,6 +399,20 @@ export type AgentHarness = { */ export type SkillSource = { name: string; dir: string }; +/** + * A host directory bind-mounted into the agent sandbox. Read-only by default; + * mounted at the identical container path unless `containerPath` overrides it + * (identical paths let one command config work on both host and container). + */ +export type SandboxMount = { + /** Host directory to mount. */ + hostPath: string; + /** Mount point inside the container; defaults to `hostPath`. */ + containerPath?: string; + /** Mount read-only (default true). */ + readonly?: boolean; +}; + export type LocalStackSessionArgs = { /** Supabase CLI version this scenario requires, overriding the runtime default. */ cliVersion?: string; @@ -434,6 +448,12 @@ export type LocalStackSessionArgs = { * instead, so they ignore this. */ skills?: readonly SkillSource[]; + /** + * Extra host directories to bind-mount into the sandbox (read-only by + * default) — e.g. a local MCP server build the in-container agent must be + * able to launch. See `supabaseMcpServerMounts`. + */ + mounts?: readonly SandboxMount[]; }; /** A mocked hosted project (platform-lite) the sandbox CLI is linked to. */ @@ -892,8 +912,6 @@ export function supabaseMcpServer( return { name: "supabase-mcp", async createConfig({ apiUrl, accessToken } = {}) { - const localServerPath = process.env.SUPABASE_MCP_SERVER_PATH; - // Server flags are identical whether we launch the published package via // npx or a local build directly with node. const serverArgs = [ @@ -910,26 +928,13 @@ export function supabaseMcpServer( // docs-only server runs standalone with no `--api-url`. if (apiUrl) serverArgs.push("--api-url", apiUrl); - // SUPABASE_MCP_SERVER_PATH swaps the published npx package for a local - // build (a repo/package dir or a direct .js/.mjs/.cjs entrypoint), so a - // workspace can test an unpublished server change without publishing to - // npm. Relative paths resolve against the eval process CWD — prefer - // absolute paths. - if (localServerPath) { - const entry = resolve( - /\.[cm]?js$/.test(localServerPath) - ? localServerPath - : join(localServerPath, "dist", "transports", "stdio.js"), - ); - if (!existsSync(entry)) { - throw new Error( - `SUPABASE_MCP_SERVER_PATH resolved to ${entry}, which does not exist — ` + - `build the server first (pnpm install && pnpm build in the mcp checkout); ` + - `see README "Running against an exact MCP server revision".`, - ); - } + const local = resolveLocalMcpServer(); + if (local) { + // `node`, not process.execPath: CLI agents run this command INSIDE the + // sandbox container, where the host's node binary path does not exist. + // Both container and host resolve `node` via PATH. return { - config: { command: process.execPath, args: [entry, ...serverArgs] }, + config: { command: "node", args: [local.entry, ...serverArgs] }, }; } @@ -943,6 +948,62 @@ export function supabaseMcpServer( }; } +/** + * SUPABASE_MCP_SERVER_PATH swaps the published npx package for a local build + * (a repo/package dir or a direct .js/.mjs/.cjs entrypoint), so a workspace + * can test an unpublished server change without publishing to npm. Relative + * paths resolve against the evals checkout root (not the process CWD), so + * `submodules/mcp/packages/mcp-server-supabase` works from any directory. + */ +function resolveLocalMcpServer(): { entry: string; baseDir: string } | null { + const localServerPath = process.env.SUPABASE_MCP_SERVER_PATH; + if (!localServerPath) return null; + + const anchor = + gitToplevel(dirname(fileURLToPath(import.meta.url))) ?? process.cwd(); + const isEntryFile = /\.[cm]?js$/.test(localServerPath); + const base = resolve(anchor, localServerPath); + const entry = isEntryFile ? base : join(base, "dist", "transports", "stdio.js"); + if (!existsSync(entry)) { + throw new Error( + `SUPABASE_MCP_SERVER_PATH resolved to ${entry}, which does not exist — ` + + `build the server first (pnpm install && pnpm build in the mcp checkout); ` + + `see README "Running against an exact MCP server revision".`, + ); + } + return { entry, baseDir: isEntryFile ? dirname(base) : base }; +} + +function gitToplevel(dir: string): string | null { + try { + return execFileSync("git", ["rev-parse", "--show-toplevel"], { + cwd: dir, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }).trim(); + } catch { + return null; + } +} + +/** + * Sandbox mounts required to launch the SUPABASE_MCP_SERVER_PATH build inside + * a containerized agent sandbox. A CLI agent's MCP command runs INSIDE the + * container, where the host build is invisible — so the build's checkout is + * bind-mounted read-only at its identical path, letting the same config work + * on both sides, with host rebuilds visible immediately (no re-copy). The + * whole git toplevel is mounted (not just dist/) because the build is + * unbundled: it requires its node_modules at runtime. Empty when unset. + */ +export function supabaseMcpServerMounts(): SandboxMount[] { + const local = resolveLocalMcpServer(); + if (!local) return []; + // Real path (git already reports one): Docker resolves bind-mount sources + // against the daemon's filesystem view, where e.g. macOS /var symlinks miss. + const root = gitToplevel(local.baseDir) ?? realpathSync(local.baseDir); + return [{ hostPath: root, readonly: true }]; +} + export function executorMcpServer(): McpServerDefinition { return { name: "executor-mcp", diff --git a/packages/core/src/mcp-server.test.ts b/packages/core/src/mcp-server.test.ts index 67e67838..64c84780 100644 --- a/packages/core/src/mcp-server.test.ts +++ b/packages/core/src/mcp-server.test.ts @@ -1,8 +1,13 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { join, relative } from "node:path"; import { tmpdir } from "node:os"; -import { MCP_SERVER_VERSION, supabaseMcpServer } from "./index.js"; +import { + MCP_SERVER_VERSION, + supabaseMcpServer, + supabaseMcpServerMounts, +} from "./index.js"; // Stub (not mutate) env so pre-existing SUPABASE_* values are restored per test. function clearEnv() { @@ -40,7 +45,7 @@ describe("supabaseMcpServer().createConfig", () => { clearEnv(); vi.stubEnv("SUPABASE_MCP_SERVER_PATH", fixtureDir); const { config } = await supabaseMcpServer().createConfig({}); - expect(config.command).toBe(process.execPath); + expect(config.command).toBe("node"); expect(config.args[0]).toBe(fixtureEntry); }); @@ -69,4 +74,49 @@ describe("supabaseMcpServer().createConfig", () => { /does not exist.*build the server first/s, ); }); + it("resolves a relative override path against the evals checkout root", async () => { + clearEnv(); + const repoRoot = execFileSync("git", ["rev-parse", "--show-toplevel"], { + cwd: process.cwd(), + encoding: "utf8", + }).trim(); + vi.stubEnv("SUPABASE_MCP_SERVER_PATH", relative(repoRoot, fixtureEntry)); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.args[0]).toBe(fixtureEntry); + }); +}); + +describe("supabaseMcpServerMounts", () => { + afterEach(() => vi.unstubAllEnvs()); + + it("is empty when no override is set", () => { + clearEnv(); + expect(supabaseMcpServerMounts()).toEqual([]); + }); + it("mounts the override checkout root read-only (a CLI agent's MCP command runs in-container)", () => { + clearEnv(); + // A git checkout wrapping the package dir: the mount must cover the whole + // checkout (the unbundled build needs its node_modules), not just dist/. + const checkout = realpathSync(mkdtempSync(join(tmpdir(), "mcp-mount-"))); + try { + execFileSync("git", ["init", "-q"], { cwd: checkout }); + const pkgDir = join(checkout, "packages", "server"); + mkdirSync(join(pkgDir, "dist", "transports"), { recursive: true }); + writeFileSync(join(pkgDir, "dist", "transports", "stdio.js"), ""); + vi.stubEnv("SUPABASE_MCP_SERVER_PATH", pkgDir); + expect(supabaseMcpServerMounts()).toEqual([ + { hostPath: checkout, readonly: true }, + ]); + } finally { + rmSync(checkout, { recursive: true, force: true }); + } + }); + + it("falls back to the package dir when the override is not inside a git checkout", () => { + clearEnv(); + vi.stubEnv("SUPABASE_MCP_SERVER_PATH", fixtureDir); + expect(supabaseMcpServerMounts()).toEqual([ + { hostPath: realpathSync(fixtureDir), readonly: true }, + ]); + }); }); diff --git a/packages/sandbox/src/agent-environment.ts b/packages/sandbox/src/agent-environment.ts index c5a3ef13..cdbac625 100644 --- a/packages/sandbox/src/agent-environment.ts +++ b/packages/sandbox/src/agent-environment.ts @@ -13,7 +13,7 @@ * this builder, so adding/removing an environment component happens in one place. */ -import type { SkillSource } from "@supabase-evals/core"; +import type { SandboxMount, SkillSource } from "@supabase-evals/core"; import { DockerSandbox } from "./docker-sandbox.js"; import { ensureSupabaseSandboxImage, setupSupabaseSandbox } from "./supabase.js"; import { installSkills, type SkillEntry } from "./skills.js"; @@ -40,6 +40,12 @@ export interface AgentEnvironmentOptions { * mode. This is the only difference between the two environments. */ localStack?: LocalStackSetup; + /** + * Extra host directories bind-mounted into the sandbox (read-only by + * default) — e.g. a local MCP server build the in-container agent must be + * able to launch. + */ + mounts?: readonly SandboxMount[]; } export interface AgentEnvironment { @@ -66,6 +72,7 @@ export async function createAgentEnvironment( // stack and instead reaches host-side platform-lite over the default bridge // via host.docker.internal — so bridge there. network: options.localStack ? "host" : undefined, + mounts: options.mounts, }); try { if (options.localStack) { diff --git a/packages/sandbox/src/bare-sandbox.ts b/packages/sandbox/src/bare-sandbox.ts index 64edd94b..2812d6d7 100644 --- a/packages/sandbox/src/bare-sandbox.ts +++ b/packages/sandbox/src/bare-sandbox.ts @@ -1,4 +1,4 @@ -import type { AgentSandbox, SkillSource } from "@supabase-evals/core"; +import type { AgentSandbox, SandboxMount, SkillSource } from "@supabase-evals/core"; import { createAgentEnvironment } from "./agent-environment.js"; import { toAgentSandbox } from "./local-stack-runtime.js"; import { buildSkillsPrompt } from "./skills.js"; @@ -20,11 +20,16 @@ export interface BareSandboxHandle { * platform-lite via `host.docker.internal` on the default bridge). */ export async function createBareSandbox( - options: { cliVersion?: string; skills?: readonly SkillSource[] } = {}, + options: { + cliVersion?: string; + skills?: readonly SkillSource[]; + mounts?: readonly SandboxMount[]; + } = {}, ): Promise { const env = await createAgentEnvironment({ cliVersion: options.cliVersion, skills: options.skills, + mounts: options.mounts, }); return { sandbox: toAgentSandbox(env.sandbox), diff --git a/packages/sandbox/src/docker-sandbox.ts b/packages/sandbox/src/docker-sandbox.ts index a1471875..57b669ae 100644 --- a/packages/sandbox/src/docker-sandbox.ts +++ b/packages/sandbox/src/docker-sandbox.ts @@ -16,6 +16,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { promisify } from "node:util"; +import type { SandboxMount } from "@supabase-evals/core"; import type { SandboxCommandResult } from "./types.js"; const execFileAsync = promisify(execFile); @@ -78,6 +79,13 @@ export interface DockerSandboxOptions { * Omitted means Docker's default bridge. */ network?: string; + /** + * Extra host directories bind-mounted into the container (read-only unless + * a mount sets `readonly: false`), at the identical path unless + * `containerPath` overrides it. Used to expose host artifacts the agent's + * tools must execute — e.g. a local MCP server build. + */ + mounts?: readonly SandboxMount[]; } export interface RunCommandOptions { @@ -90,6 +98,7 @@ export class DockerSandbox { private defaultTimeoutMs: number; private network: string | undefined; private image: string; + private mounts: readonly SandboxMount[]; readonly workdir: string; /** * Env vars injected into every `runShell` (non-root) command — both the @@ -102,6 +111,7 @@ export class DockerSandbox { this.defaultTimeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; this.network = options.network; this.image = options.image ?? DEFAULT_IMAGE; + this.mounts = options.mounts ?? []; this.workdir = `${WORKSPACE_BASE}-${randomUUID().slice(0, 8)}`; } @@ -134,6 +144,14 @@ export class DockerSandbox { "/var/run/docker.sock:/var/run/docker.sock", "--volume", `${this.workdir}:${this.workdir}`, + // Caller-requested host mounts (e.g. a local MCP server build the + // in-container agent launches). Read-only unless the mount opts out. + ...this.mounts.flatMap((mount) => [ + "--volume", + `${mount.hostPath}:${mount.containerPath ?? mount.hostPath}${ + mount.readonly === false ? "" : ":ro" + }`, + ]), "--workdir", this.workdir, // Reach host-side servers (e.g. the linked platform-lite) at diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index df7b63ca..58ef0a1a 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -79,6 +79,7 @@ export function localStackRuntime( projectRunning, hosted, skills, + mounts, }) { // Local-stack mode = the shared agent environment with the Supabase local // stack started. Everything else (image, tooling, skills) is identical to @@ -87,6 +88,7 @@ export function localStackRuntime( cliVersion: cliVersion ?? options.cliVersion, localDir, skills, + mounts, localStack: { includeServices, projectRunning, From 2ad5b5bf28546a0befeab449a641d083d877256e Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 14:05:04 +0200 Subject: [PATCH 5/6] fix: realpath the local MCP override once, deriving command and mount from one view Pedro's review nit: entry (config command) was not realpath'd while the mount root was - an override under a symlinked dir (macOS /tmp -> /private/tmp) would mount at the resolved path but exec the symlinked one -> ENOENT in-container. Canonicalize the base once and derive the entry from it (never realpath the entry separately: a symlinked dist/ target could resolve outside the mounted baseDir). Symlinked-override regression test added; mount fallback drops its now-redundant realpath. --- packages/core/src/index.ts | 26 +++++++++++++++++++------- packages/core/src/mcp-server.test.ts | 24 ++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index a174e96c..f576b246 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -963,15 +963,26 @@ function resolveLocalMcpServer(): { entry: string; baseDir: string } | null { gitToplevel(dirname(fileURLToPath(import.meta.url))) ?? process.cwd(); const isEntryFile = /\.[cm]?js$/.test(localServerPath); const base = resolve(anchor, localServerPath); - const entry = isEntryFile ? base : join(base, "dist", "transports", "stdio.js"); - if (!existsSync(entry)) { + const probe = isEntryFile ? base : join(base, "dist", "transports", "stdio.js"); + if (!existsSync(probe)) { throw new Error( - `SUPABASE_MCP_SERVER_PATH resolved to ${entry}, which does not exist — ` + + `SUPABASE_MCP_SERVER_PATH resolved to ${probe}, which does not exist — ` + `build the server first (pnpm install && pnpm build in the mcp checkout); ` + `see README "Running against an exact MCP server revision".`, ); } - return { entry, baseDir: isEntryFile ? dirname(base) : base }; + // One filesystem view for command AND mount: the sandbox bind-mounts the + // realpath (Docker resolves sources against the daemon's view), so the + // command must reference the same view — an override under a symlinked dir + // (macOS /tmp -> /private/tmp) would otherwise exec a path that does not + // exist in-container. Canonicalize the BASE once and derive the entry from + // it (never realpath the entry separately: a symlinked dist/ target could + // resolve outside the mounted baseDir). + const realBase = realpathSync(base); + return { + entry: isEntryFile ? realBase : join(realBase, "dist", "transports", "stdio.js"), + baseDir: isEntryFile ? dirname(realBase) : realBase, + }; } function gitToplevel(dir: string): string | null { @@ -998,9 +1009,10 @@ function gitToplevel(dir: string): string | null { export function supabaseMcpServerMounts(): SandboxMount[] { const local = resolveLocalMcpServer(); if (!local) return []; - // Real path (git already reports one): Docker resolves bind-mount sources - // against the daemon's filesystem view, where e.g. macOS /var symlinks miss. - const root = gitToplevel(local.baseDir) ?? realpathSync(local.baseDir); + // baseDir is already realpath'd (resolveLocalMcpServer): Docker resolves + // bind-mount sources against the daemon's filesystem view, where e.g. + // macOS /var symlinks miss. + const root = gitToplevel(local.baseDir) ?? local.baseDir; return [{ hostPath: root, readonly: true }]; } diff --git a/packages/core/src/mcp-server.test.ts b/packages/core/src/mcp-server.test.ts index 64c84780..b85b75c8 100644 --- a/packages/core/src/mcp-server.test.ts +++ b/packages/core/src/mcp-server.test.ts @@ -1,6 +1,6 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { execFileSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { join, relative } from "node:path"; import { tmpdir } from "node:os"; import { @@ -19,7 +19,10 @@ function clearEnv() { let fixtureDir: string; let fixtureEntry: string; beforeAll(() => { - fixtureDir = mkdtempSync(join(tmpdir(), "mcp-override-")); + // realpath'd: the resolver realpaths the override (command must match the + // container mount view), so unresolved tmpdir paths (macOS /var symlink) + // would fail every exact-path assertion below. + fixtureDir = realpathSync(mkdtempSync(join(tmpdir(), "mcp-override-"))); fixtureEntry = join(fixtureDir, "dist", "transports", "stdio.js"); mkdirSync(join(fixtureDir, "dist", "transports"), { recursive: true }); writeFileSync(fixtureEntry, ""); @@ -84,6 +87,23 @@ describe("supabaseMcpServer().createConfig", () => { const { config } = await supabaseMcpServer().createConfig({}); expect(config.args[0]).toBe(fixtureEntry); }); + + it("realpaths a symlinked override so the command matches the container mount", async () => { + clearEnv(); + const linkDir = mkdtempSync(join(tmpdir(), "mcp-link-")); + const link = join(linkDir, "pkg"); + symlinkSync(fixtureDir, link); + try { + vi.stubEnv("SUPABASE_MCP_SERVER_PATH", link); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.args[0]).toBe(fixtureEntry); // the real path, not the symlink + expect(supabaseMcpServerMounts()).toEqual([ + { hostPath: realpathSync(fixtureDir), readonly: true }, + ]); + } finally { + rmSync(linkDir, { recursive: true, force: true }); + } + }); }); describe("supabaseMcpServerMounts", () => { From 380975547df240832c9b0f77e6cdc325b00379c5 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 14:09:08 +0200 Subject: [PATCH 6/6] perf: memoize the local MCP override resolution (Pedro's nit) createConfig and the sandbox mounts each resolved the override, spawning git per call (anchor + mount root) - including in local-stack mode where the mount goes unused. The resolution (entry, baseDir, mount root) is now computed once and cached keyed on the raw env value, so tests and callers that change SUPABASE_MCP_SERVER_PATH still see fresh state; the not-found error path stays uncached so a fixed build is picked up on retry. --- packages/core/src/index.ts | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f576b246..2222d211 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -954,10 +954,20 @@ export function supabaseMcpServer( * can test an unpublished server change without publishing to npm. Relative * paths resolve against the evals checkout root (not the process CWD), so * `submodules/mcp/packages/mcp-server-supabase` works from any directory. + * + * Memoized per env value: createConfig and the sandbox mounts both resolve, + * and each resolution spawns git (anchor + mount root) — cache so repeat + * calls within a run cost nothing. Keyed on the raw env string because tests + * (and in principle callers) change it between calls; the not-found error + * path is deliberately uncached so a fixed build is picked up on retry. */ -function resolveLocalMcpServer(): { entry: string; baseDir: string } | null { +type LocalMcpServer = { entry: string; baseDir: string; mountRoot: string }; +let localMcpServerCache: { key: string; value: LocalMcpServer } | null = null; + +function resolveLocalMcpServer(): LocalMcpServer | null { const localServerPath = process.env.SUPABASE_MCP_SERVER_PATH; if (!localServerPath) return null; + if (localMcpServerCache?.key === localServerPath) return localMcpServerCache.value; const anchor = gitToplevel(dirname(fileURLToPath(import.meta.url))) ?? process.cwd(); @@ -979,10 +989,16 @@ function resolveLocalMcpServer(): { entry: string; baseDir: string } | null { // it (never realpath the entry separately: a symlinked dist/ target could // resolve outside the mounted baseDir). const realBase = realpathSync(base); - return { + const baseDir = isEntryFile ? dirname(realBase) : realBase; + const value: LocalMcpServer = { entry: isEntryFile ? realBase : join(realBase, "dist", "transports", "stdio.js"), - baseDir: isEntryFile ? dirname(realBase) : realBase, + baseDir, + // The whole git toplevel (not just dist/) because the build is unbundled: + // it requires its node_modules at runtime. + mountRoot: gitToplevel(baseDir) ?? baseDir, }; + localMcpServerCache = { key: localServerPath, value }; + return value; } function gitToplevel(dir: string): string | null { @@ -1001,19 +1017,13 @@ function gitToplevel(dir: string): string | null { * Sandbox mounts required to launch the SUPABASE_MCP_SERVER_PATH build inside * a containerized agent sandbox. A CLI agent's MCP command runs INSIDE the * container, where the host build is invisible — so the build's checkout is - * bind-mounted read-only at its identical path, letting the same config work - * on both sides, with host rebuilds visible immediately (no re-copy). The - * whole git toplevel is mounted (not just dist/) because the build is - * unbundled: it requires its node_modules at runtime. Empty when unset. + * bind-mounted read-only at its identical (real) path, letting the same + * config work on both sides, with host rebuilds visible immediately (no + * re-copy). Empty when unset. */ export function supabaseMcpServerMounts(): SandboxMount[] { const local = resolveLocalMcpServer(); - if (!local) return []; - // baseDir is already realpath'd (resolveLocalMcpServer): Docker resolves - // bind-mount sources against the daemon's filesystem view, where e.g. - // macOS /var symlinks miss. - const root = gitToplevel(local.baseDir) ?? local.baseDir; - return [{ hostPath: root, readonly: true }]; + return local ? [{ hostPath: local.mountRoot, readonly: true }] : []; } export function executorMcpServer(): McpServerDefinition {