diff --git a/.gitmodules b/.gitmodules index cc410ee6..094026de 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "submodules/agent-skills"] path = submodules/agent-skills url = git@github.com:supabase/agent-skills.git +[submodule "submodules/mcp"] + path = submodules/mcp + url = git@github.com:supabase/mcp.git diff --git a/README.md b/README.md index 7662b01a..ebf6991d 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,35 @@ pnpm eval -- \ `--suite`, `--experiment-suite`, `--experiment`, and `--eval` accept multiple inputs via repeated flags as well as comma-separated values. +### Running against an exact MCP server revision + +Evals launch `@supabase/mcp-server-supabase` via npx, pinned to the version in +`MCP_SERVER_VERSION` (`packages/core/src/index.ts`). To eval an exact source +revision instead — an unpublished branch, a PR, or the pinned submodule — +build it and run against the build: + +```bash +pnpm mcp:build # init submodules/mcp + pnpm install + build (rerun after source edits) +pnpm eval:local-mcp -- --eval --experiment +``` + +`eval:local-mcp` is `pnpm eval` with `SUPABASE_MCP_SERVER_PATH` prefilled to +the submodule's server package. To test a different revision, check out any +commit inside `submodules/mcp` and `pnpm mcp:build` again — the gitlink pin +only moves when a bump is committed here. Containerized agents (Claude Code, +Codex) see the build through a read-only bind mount of the checkout, so a +host-side rebuild is picked up by the next run with no extra copying. + +For a build outside the submodule, set `SUPABASE_MCP_SERVER_PATH` yourself: it +accepts a package dir (launches `dist/transports/stdio.js`) or a direct +`.js`/`.mjs`/`.cjs` entrypoint path; relative paths resolve against this +repo's root. The path is existence-checked at config time, so an unbuilt +checkout fails fast with a pointer back here. + +Note the pin tracks the published npm package's source (via its release tag), +not the hosted production deployment — the prod deploy commit is separate +provenance that this repo does not record. + Run all benchmark and no-skills experiments across all benchmark evals: ```bash diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 0f758e53..19cd9da1 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -32,6 +32,7 @@ import { buildSkillResult, rehydrateTruncatedDocsResults, getExperimentDisplayMetadata, + supabaseMcpServerMounts, } from '@supabase-evals/core'; import type { ExperimentConfig, @@ -439,6 +440,7 @@ async function runOne( // (the session folds the discovery listing into its promptAddendum), // so no skill text is injected into the prompt here. skills: skillSources, + mounts: supabaseMcpServerMounts(), }) ); @@ -507,7 +509,12 @@ async function runOne( // platform-lite via host.docker.internal (so platform-lite binds 0.0.0.0). // An in-process agent runs host-side with no sandbox. await using cliSandbox = agentRunsInSandbox - ? disposable(await createBareSandbox({ skills: skillSources })) + ? disposable( + await createBareSandbox({ + skills: skillSources, + mounts: supabaseMcpServerMounts(), + }) + ) : undefined; await using session = disposable( await exp.runtime.startSession({ diff --git a/package.json b/package.json index 08a0ef27..6d6c48af 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,8 @@ "eval:dry": "pnpm --filter @supabase-evals/framework eval:dry", "eval:smoke": "pnpm --filter @supabase-evals/framework eval:smoke", "eval:force": "pnpm --filter @supabase-evals/framework eval:force", + "eval:local-mcp": "SUPABASE_MCP_SERVER_PATH=submodules/mcp/packages/mcp-server-supabase pnpm eval", + "mcp:build": "git submodule update --init submodules/mcp && pnpm --dir submodules/mcp install && pnpm --dir submodules/mcp build", "test:framework": "pnpm --filter @supabase-evals/framework test:framework", "export-results": "pnpm --filter @supabase-evals/framework export-results", "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck", diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c3ccfc0e..e25fc37b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,11 +1,17 @@ import vm from 'node:vm'; import { createRequire } from 'node:module'; import { createHash, createHmac } from 'node:crypto'; -import { execFile } from 'node:child_process'; +import { execFile, execFileSync } from 'node:child_process'; import { createServer } from 'node:net'; import { promisify } from 'node:util'; -import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; -import { basename, dirname, join } from 'node:path'; +import { + existsSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, +} from 'node:fs'; +import { basename, dirname, join, resolve } from 'node:path'; import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; import type { ToolName } from './transcript/types.js'; @@ -402,6 +408,20 @@ export type AgentHarness = { */ export type SkillSource = { name: string; dir: string }; +/** + * A host directory bind-mounted into the agent sandbox. Read-only by default; + * mounted at the identical container path unless `containerPath` overrides it + * (identical paths let one command config work on both host and container). + */ +export type SandboxMount = { + /** Host directory to mount. */ + hostPath: string; + /** Mount point inside the container; defaults to `hostPath`. */ + containerPath?: string; + /** Mount read-only (default true). */ + readonly?: boolean; +}; + export type LocalStackSessionArgs = { /** Supabase CLI version this scenario requires, overriding the runtime default. */ cliVersion?: string; @@ -437,6 +457,12 @@ export type LocalStackSessionArgs = { * instead, so they ignore this. */ skills?: readonly SkillSource[]; + /** + * Extra host directories to bind-mount into the sandbox (read-only by + * default) — e.g. a local MCP server build the in-container agent must be + * able to launch. See `supabaseMcpServerMounts`. + */ + mounts?: readonly SandboxMount[]; }; /** A mocked hosted project (platform-lite) the sandbox CLI is linked to. */ @@ -895,8 +921,9 @@ export function supabaseMcpServer( return { name: 'supabase-mcp', async createConfig({ apiUrl, accessToken } = {}) { - const args = [ - `@supabase/mcp-server-supabase@${version}`, + // Server flags are identical whether we launch the published package via + // npx or a local build directly with node. + const serverArgs = [ // The server refuses to boot without a token; with only platform- // independent features (docs) it never authenticates against the // management API, so a well-formed throwaway is enough. @@ -908,12 +935,111 @@ export function supabaseMcpServer( // Only point the server at a platform when one is given. `docs` is // platform-independent (it queries the public docs GraphQL API), so a // docs-only server runs standalone with no `--api-url`. - if (apiUrl) args.push('--api-url', apiUrl); - return { config: { command: 'npx', args } }; + if (apiUrl) serverArgs.push('--api-url', apiUrl); + + const local = resolveLocalMcpServer(); + if (local) { + // `node`, not process.execPath: CLI agents run this command INSIDE the + // sandbox container, where the host's node binary path does not exist. + // Both container and host resolve `node` via PATH. + return { + config: { command: 'node', args: [local.entry, ...serverArgs] }, + }; + } + + return { + config: { + command: 'npx', + args: [`@supabase/mcp-server-supabase@${version}`, ...serverArgs], + }, + }; }, }; } +/** + * SUPABASE_MCP_SERVER_PATH swaps the published npx package for a local build + * (a repo/package dir or a direct .js/.mjs/.cjs entrypoint), so a workspace + * can test an unpublished server change without publishing to npm. Relative + * paths resolve against the evals checkout root (not the process CWD), so + * `submodules/mcp/packages/mcp-server-supabase` works from any directory. + * + * Memoized per env value: createConfig and the sandbox mounts both resolve, + * and each resolution spawns git (anchor + mount root) — cache so repeat + * calls within a run cost nothing. Keyed on the raw env string because tests + * (and in principle callers) change it between calls; the not-found error + * path is deliberately uncached so a fixed build is picked up on retry. + */ +type LocalMcpServer = { entry: string; baseDir: string; mountRoot: string }; +let localMcpServerCache: { key: string; value: LocalMcpServer } | null = null; + +function resolveLocalMcpServer(): LocalMcpServer | null { + const localServerPath = process.env.SUPABASE_MCP_SERVER_PATH; + if (!localServerPath) return null; + if (localMcpServerCache?.key === localServerPath) + return localMcpServerCache.value; + + const anchor = + gitToplevel(dirname(fileURLToPath(import.meta.url))) ?? process.cwd(); + const isEntryFile = /\.[cm]?js$/.test(localServerPath); + const base = resolve(anchor, localServerPath); + const probe = isEntryFile + ? base + : join(base, 'dist', 'transports', 'stdio.js'); + if (!existsSync(probe)) { + throw new Error( + `SUPABASE_MCP_SERVER_PATH resolved to ${probe}, which does not exist — ` + + `build the server first (pnpm install && pnpm build in the mcp checkout); ` + + `see README "Running against an exact MCP server revision".` + ); + } + // One filesystem view for command AND mount: the sandbox bind-mounts the + // realpath (Docker resolves sources against the daemon's view), so the + // command must reference the same view — an override under a symlinked dir + // (macOS /tmp -> /private/tmp) would otherwise exec a path that does not + // exist in-container. Canonicalize the BASE once and derive the entry from + // it (never realpath the entry separately: a symlinked dist/ target could + // resolve outside the mounted baseDir). + const realBase = realpathSync(base); + const baseDir = isEntryFile ? dirname(realBase) : realBase; + const value: LocalMcpServer = { + entry: isEntryFile + ? realBase + : join(realBase, 'dist', 'transports', 'stdio.js'), + baseDir, + // The whole git toplevel (not just dist/) because the build is unbundled: + // it requires its node_modules at runtime. + mountRoot: gitToplevel(baseDir) ?? baseDir, + }; + localMcpServerCache = { key: localServerPath, value }; + return value; +} + +function gitToplevel(dir: string): string | null { + try { + return execFileSync('git', ['rev-parse', '--show-toplevel'], { + cwd: dir, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + } catch { + return null; + } +} + +/** + * Sandbox mounts required to launch the SUPABASE_MCP_SERVER_PATH build inside + * a containerized agent sandbox. A CLI agent's MCP command runs INSIDE the + * container, where the host build is invisible — so the build's checkout is + * bind-mounted read-only at its identical (real) path, letting the same + * config work on both sides, with host rebuilds visible immediately (no + * re-copy). Empty when unset. + */ +export function supabaseMcpServerMounts(): SandboxMount[] { + const local = resolveLocalMcpServer(); + return local ? [{ hostPath: local.mountRoot, readonly: true }] : []; +} + export function executorMcpServer(): McpServerDefinition { return { name: 'executor-mcp', diff --git a/packages/core/src/mcp-server.test.ts b/packages/core/src/mcp-server.test.ts new file mode 100644 index 00000000..bd3f81c3 --- /dev/null +++ b/packages/core/src/mcp-server.test.ts @@ -0,0 +1,157 @@ +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + it, + vi, +} from 'vitest'; +import { execFileSync } from 'node:child_process'; +import { + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { join, relative } from 'node:path'; +import { tmpdir } from 'node:os'; +import { + MCP_SERVER_VERSION, + supabaseMcpServer, + supabaseMcpServerMounts, +} from './index.js'; + +// Stub (not mutate) env so pre-existing SUPABASE_* values are restored per test. +function clearEnv() { + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', undefined); +} + +// A real on-disk build layout: the override path is existence-checked, so the +// fixtures must actually exist for the happy paths (and not for the error one). +let fixtureDir: string; +let fixtureEntry: string; +beforeAll(() => { + // realpath'd: the resolver realpaths the override (command must match the + // container mount view), so unresolved tmpdir paths (macOS /var symlink) + // would fail every exact-path assertion below. + fixtureDir = realpathSync(mkdtempSync(join(tmpdir(), 'mcp-override-'))); + fixtureEntry = join(fixtureDir, 'dist', 'transports', 'stdio.js'); + mkdirSync(join(fixtureDir, 'dist', 'transports'), { recursive: true }); + writeFileSync(fixtureEntry, ''); +}); +afterAll(() => rmSync(fixtureDir, { recursive: true, force: true })); + +describe('supabaseMcpServer().createConfig', () => { + afterEach(() => vi.unstubAllEnvs()); + + it('defaults to the published package via npx', async () => { + clearEnv(); + const { config } = await supabaseMcpServer().createConfig({ + apiUrl: 'http://api.test', + }); + expect(config.command).toBe('npx'); + expect(config.args[0]).toBe( + `@supabase/mcp-server-supabase@${MCP_SERVER_VERSION}` + ); + expect(config.args).toContain('--api-url'); + }); + + it('launches a local build dir with node when SUPABASE_MCP_SERVER_PATH is set', async () => { + clearEnv(); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureDir); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.command).toBe('node'); + expect(config.args[0]).toBe(fixtureEntry); + }); + + it('uses a direct .js override path as-is', async () => { + clearEnv(); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureEntry); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.args[0]).toBe(fixtureEntry); + }); + + it('preserves --api-url on the local override path', async () => { + clearEnv(); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureDir); + const { config } = await supabaseMcpServer().createConfig({ + apiUrl: 'http://api.test', + }); + const i = config.args.indexOf('--api-url'); + expect(i).toBeGreaterThan(-1); + expect(config.args[i + 1]).toBe('http://api.test'); + }); + + it('fails fast with an actionable error when the override path does not exist', async () => { + clearEnv(); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', join(fixtureDir, 'not-built')); + await expect(supabaseMcpServer().createConfig({})).rejects.toThrow( + /does not exist.*build the server first/s + ); + }); + it('resolves a relative override path against the evals checkout root', async () => { + clearEnv(); + const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], { + cwd: process.cwd(), + encoding: 'utf8', + }).trim(); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', relative(repoRoot, fixtureEntry)); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.args[0]).toBe(fixtureEntry); + }); + + it('realpaths a symlinked override so the command matches the container mount', async () => { + clearEnv(); + const linkDir = mkdtempSync(join(tmpdir(), 'mcp-link-')); + const link = join(linkDir, 'pkg'); + symlinkSync(fixtureDir, link); + try { + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', link); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.args[0]).toBe(fixtureEntry); // the real path, not the symlink + expect(supabaseMcpServerMounts()).toEqual([ + { hostPath: realpathSync(fixtureDir), readonly: true }, + ]); + } finally { + rmSync(linkDir, { recursive: true, force: true }); + } + }); +}); + +describe('supabaseMcpServerMounts', () => { + afterEach(() => vi.unstubAllEnvs()); + + it('is empty when no override is set', () => { + clearEnv(); + expect(supabaseMcpServerMounts()).toEqual([]); + }); + it("mounts the override checkout root read-only (a CLI agent's MCP command runs in-container)", () => { + clearEnv(); + // A git checkout wrapping the package dir: the mount must cover the whole + // checkout (the unbundled build needs its node_modules), not just dist/. + const checkout = realpathSync(mkdtempSync(join(tmpdir(), 'mcp-mount-'))); + try { + execFileSync('git', ['init', '-q'], { cwd: checkout }); + const pkgDir = join(checkout, 'packages', 'server'); + mkdirSync(join(pkgDir, 'dist', 'transports'), { recursive: true }); + writeFileSync(join(pkgDir, 'dist', 'transports', 'stdio.js'), ''); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', pkgDir); + expect(supabaseMcpServerMounts()).toEqual([ + { hostPath: checkout, readonly: true }, + ]); + } finally { + rmSync(checkout, { recursive: true, force: true }); + } + }); + + it('falls back to the package dir when the override is not inside a git checkout', () => { + clearEnv(); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureDir); + expect(supabaseMcpServerMounts()).toEqual([ + { hostPath: realpathSync(fixtureDir), readonly: true }, + ]); + }); +}); diff --git a/packages/sandbox/src/agent-environment.ts b/packages/sandbox/src/agent-environment.ts index 4d046395..b17918f5 100644 --- a/packages/sandbox/src/agent-environment.ts +++ b/packages/sandbox/src/agent-environment.ts @@ -13,7 +13,7 @@ * this builder, so adding/removing an environment component happens in one place. */ -import type { SkillSource } from '@supabase-evals/core'; +import type { SandboxMount, SkillSource } from '@supabase-evals/core'; import { DockerSandbox } from './docker-sandbox.js'; import { ensureSupabaseSandboxImage, @@ -43,6 +43,12 @@ export interface AgentEnvironmentOptions { * mode. This is the only difference between the two environments. */ localStack?: LocalStackSetup; + /** + * Extra host directories bind-mounted into the sandbox (read-only by + * default) — e.g. a local MCP server build the in-container agent must be + * able to launch. + */ + mounts?: readonly SandboxMount[]; } export interface AgentEnvironment { @@ -69,6 +75,7 @@ export async function createAgentEnvironment( // stack and instead reaches host-side platform-lite over the default bridge // via host.docker.internal — so bridge there. network: options.localStack ? 'host' : undefined, + mounts: options.mounts, }); try { if (options.localStack) { diff --git a/packages/sandbox/src/bare-sandbox.ts b/packages/sandbox/src/bare-sandbox.ts index 0ea025ef..4c51a14d 100644 --- a/packages/sandbox/src/bare-sandbox.ts +++ b/packages/sandbox/src/bare-sandbox.ts @@ -1,4 +1,8 @@ -import type { AgentSandbox, SkillSource } from '@supabase-evals/core'; +import type { + AgentSandbox, + SandboxMount, + SkillSource, +} from '@supabase-evals/core'; import { createAgentEnvironment } from './agent-environment.js'; import { toAgentSandbox } from './local-stack-runtime.js'; import { buildSkillsPrompt } from './skills.js'; @@ -20,11 +24,16 @@ export interface BareSandboxHandle { * platform-lite via `host.docker.internal` on the default bridge). */ export async function createBareSandbox( - options: { cliVersion?: string; skills?: readonly SkillSource[] } = {} + options: { + cliVersion?: string; + skills?: readonly SkillSource[]; + mounts?: readonly SandboxMount[]; + } = {} ): Promise { const env = await createAgentEnvironment({ cliVersion: options.cliVersion, skills: options.skills, + mounts: options.mounts, }); return { sandbox: toAgentSandbox(env.sandbox), diff --git a/packages/sandbox/src/docker-sandbox.ts b/packages/sandbox/src/docker-sandbox.ts index 42f98052..586956ca 100644 --- a/packages/sandbox/src/docker-sandbox.ts +++ b/packages/sandbox/src/docker-sandbox.ts @@ -16,6 +16,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { promisify } from 'node:util'; +import type { SandboxMount } from '@supabase-evals/core'; import type { SandboxCommandResult } from './types.js'; const execFileAsync = promisify(execFile); @@ -78,6 +79,13 @@ export interface DockerSandboxOptions { * Omitted means Docker's default bridge. */ network?: string; + /** + * Extra host directories bind-mounted into the container (read-only unless + * a mount sets `readonly: false`), at the identical path unless + * `containerPath` overrides it. Used to expose host artifacts the agent's + * tools must execute — e.g. a local MCP server build. + */ + mounts?: readonly SandboxMount[]; } export interface RunCommandOptions { @@ -90,6 +98,7 @@ export class DockerSandbox { private defaultTimeoutMs: number; private network: string | undefined; private image: string; + private mounts: readonly SandboxMount[]; readonly workdir: string; /** * Env vars injected into every `runShell` (non-root) command — both the @@ -102,6 +111,7 @@ export class DockerSandbox { this.defaultTimeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; this.network = options.network; this.image = options.image ?? DEFAULT_IMAGE; + this.mounts = options.mounts ?? []; this.workdir = `${WORKSPACE_BASE}-${randomUUID().slice(0, 8)}`; } @@ -134,6 +144,14 @@ export class DockerSandbox { '/var/run/docker.sock:/var/run/docker.sock', '--volume', `${this.workdir}:${this.workdir}`, + // Caller-requested host mounts (e.g. a local MCP server build the + // in-container agent launches). Read-only unless the mount opts out. + ...this.mounts.flatMap((mount) => [ + '--volume', + `${mount.hostPath}:${mount.containerPath ?? mount.hostPath}${ + mount.readonly === false ? '' : ':ro' + }`, + ]), '--workdir', this.workdir, // Reach host-side servers (e.g. the linked platform-lite) at diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 5fe61875..6ee95e72 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -79,6 +79,7 @@ export function localStackRuntime( projectRunning, hosted, skills, + mounts, }) { // Local-stack mode = the shared agent environment with the Supabase local // stack started. Everything else (image, tooling, skills) is identical to @@ -87,6 +88,7 @@ export function localStackRuntime( cliVersion: cliVersion ?? options.cliVersion, localDir, skills, + mounts, localStack: { includeServices, projectRunning, diff --git a/submodules/mcp b/submodules/mcp new file mode 160000 index 00000000..5a8d9652 --- /dev/null +++ b/submodules/mcp @@ -0,0 +1 @@ +Subproject commit 5a8d9652ab975b9f99e622feb296f1d946994dee