feat: add Cursor CLI agent harness and experiments

.env.example

@@ -8,3 +8,6 @@ OPENAI_API_KEY=
 
 # Vercel AI Gateway. The opencode harness routes through this.
 AI_GATEWAY_API_KEY=
+
+# Cursor CLI harness (composer-2.5 / cursor-grok-4.5-high experiments).
+CURSOR_API_KEY=

README.md

@@ -23,7 +23,7 @@ pnpm install
 cp .env.example .env
 ```
 
-Agent-backed runs require the relevant provider key in `.env` (e.g. `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`)
+Agent-backed runs require the relevant provider key in `.env` (e.g. `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `CURSOR_API_KEY`). Rubric/judge checks also need `OPENAI_API_KEY` regardless of harness.
 
 ## Concepts
 

apps/web/src/lib/format.ts

@@ -6,6 +6,7 @@ export const AGENT_LABELS = {
   "ai-sdk": "AI SDK",
   "claude-code": "Claude Code",
   codex: "Codex",
+  cursor: "Cursor",
   opencode: "OpenCode",
 } satisfies Record<ExperimentDisplay["agent"], string>
 

experiments/cursor-composer-2.5-no-skills.ts

@@ -0,0 +1,22 @@
+import {
+  cursorAgent,
+  defineExperiment,
+  platformLiteRuntime,
+  supabaseMcpServer,
+} from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+
+// Same as cursor-composer-2.5 but with no skills, to measure skills' impact.
+export default defineExperiment({
+  suite: ['no-skills', 'regression'],
+  agent: cursorAgent({
+    model: 'composer-2.5',
+  }),
+  runtime: platformLiteRuntime({
+    mcpServers: [supabaseMcpServer()],
+  }),
+  localStack: localStackRuntime(),
+  skills: [],
+  // Evals that override `skills: []` already run under the baseline experiment. Skip them from running again here.
+  skipEval: (ev) => ev.metadata.skills?.length === 0,
+});

experiments/cursor-composer-2.5.ts

@@ -0,0 +1,19 @@
+import {
+  cursorAgent,
+  defineExperiment,
+  platformLiteRuntime,
+  supabaseMcpServer,
+} from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+
+export default defineExperiment({
+  suite: ['benchmark', 'regression'],
+  agent: cursorAgent({
+    model: 'composer-2.5',
+  }),
+  runtime: platformLiteRuntime({
+    mcpServers: [supabaseMcpServer()],
+  }),
+  localStack: localStackRuntime(),
+  skills: ['supabase', 'supabase-postgres-best-practices'],
+});

experiments/cursor-grok-4.5-high-no-skills.ts

@@ -0,0 +1,22 @@
+import {
+  cursorAgent,
+  defineExperiment,
+  platformLiteRuntime,
+  supabaseMcpServer,
+} from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+
+// Same as cursor-grok-4.5-high but with no skills, to measure skills' impact.
+export default defineExperiment({
+  suite: ['no-skills', 'regression'],
+  agent: cursorAgent({
+    model: 'cursor-grok-4.5-high',
+  }),
+  runtime: platformLiteRuntime({
+    mcpServers: [supabaseMcpServer()],
+  }),
+  localStack: localStackRuntime(),
+  skills: [],
+  // Evals that override `skills: []` already run under the baseline experiment. Skip them from running again here.
+  skipEval: (ev) => ev.metadata.skills?.length === 0,
+});

experiments/cursor-grok-4.5-high.ts

@@ -0,0 +1,19 @@
+import {
+  cursorAgent,
+  defineExperiment,
+  platformLiteRuntime,
+  supabaseMcpServer,
+} from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+
+export default defineExperiment({
+  suite: ['benchmark', 'regression'],
+  agent: cursorAgent({
+    model: 'cursor-grok-4.5-high',
+  }),
+  runtime: platformLiteRuntime({
+    mcpServers: [supabaseMcpServer()],
+  }),
+  localStack: localStackRuntime(),
+  skills: ['supabase', 'supabase-postgres-best-practices'],
+});

packages/core/src/agents/cursor/fixtures/stream-json-docs-sample.jsonl

@@ -0,0 +1,10 @@
+{"type":"system","subtype":"init","apiKeySource":"login","cwd":"/Users/user/project","session_id":"c6b62c6f-7ead-4fd6-9922-e952131177ff","model":"Claude 4 Sonnet","permissionMode":"default"}
+{"type":"user","message":{"role":"user","content":[{"type":"text","text":"Read README.md and create a summary"}]},"session_id":"c6b62c6f-7ead-4fd6-9922-e952131177ff"}
+{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"I'll read the README.md file"}]},"session_id":"c6b62c6f-7ead-4fd6-9922-e952131177ff"}
+{"type":"tool_call","subtype":"started","call_id":"toolu_vrtx_01NnjaR886UcE8whekg2MGJd","tool_call":{"readToolCall":{"args":{"path":"README.md"}}},"session_id":"c6b62c6f-7ead-4fd6-9922-e952131177ff"}
+{"type":"tool_call","subtype":"completed","call_id":"toolu_vrtx_01NnjaR886UcE8whekg2MGJd","tool_call":{"readToolCall":{"args":{"path":"README.md"},"result":{"success":{"content":"# Project\n\nThis is a sample project...","isEmpty":false,"exceededLimit":false,"totalLines":54,"totalChars":1254}}}},"session_id":"c6b62c6f-7ead-4fd6-9922-e952131177ff"}
+{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Based on the README, I'll create a summary"}]},"session_id":"c6b62c6f-7ead-4fd6-9922-e952131177ff"}
+{"type":"tool_call","subtype":"started","call_id":"toolu_vrtx_01Q3VHVnWFSKygaRPT7WDxrv","tool_call":{"writeToolCall":{"args":{"path":"summary.txt","fileText":"# README Summary\n\nThis project contains...","toolCallId":"toolu_vrtx_01Q3VHVnWFSKygaRPT7WDxrv"}}},"session_id":"c6b62c6f-7ead-4fd6-9922-e952131177ff"}
+{"type":"tool_call","subtype":"completed","call_id":"toolu_vrtx_01Q3VHVnWFSKygaRPT7WDxrv","tool_call":{"writeToolCall":{"args":{"path":"summary.txt","fileText":"# README Summary\n\nThis project contains...","toolCallId":"toolu_vrtx_01Q3VHVnWFSKygaRPT7WDxrv"},"result":{"success":{"path":"/Users/user/project/summary.txt","linesCreated":19,"fileSize":942}}}},"session_id":"c6b62c6f-7ead-4fd6-9922-e952131177ff"}
+{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Done! I've created the summary in summary.txt"}]},"session_id":"c6b62c6f-7ead-4fd6-9922-e952131177ff"}
+{"type":"result","subtype":"success","duration_ms":5234,"duration_api_ms":5234,"is_error":false,"result":"I'll read the README.md fileBased on the README, I'll create a summaryDone! I've created the summary in summary.txt","session_id":"c6b62c6f-7ead-4fd6-9922-e952131177ff","request_id":"10e11780-df2f-45dc-a1ff-4540af32e9c0"}

packages/core/src/agents/cursor/fixtures/stream-json-harbor-tool-shapes.jsonl

@@ -0,0 +1,5 @@
+{"type":"system","subtype":"init","apiKeySource":"env","cwd":"/workspace","session_id":"session-1","model":"composer-2.5","permissionMode":"default"}
+{"type":"assistant","session_id":"session-1","model_call_id":"model-call-1","timestamp_ms":1700000000000,"message":{"role":"assistant","content":[{"type":"text","text":"Using tools now."}]}}
+{"type":"tool_call","subtype":"completed","call_id":"call-1","session_id":"session-1","model_call_id":"model-call-1","timestamp_ms":1700000000000,"tool_call":{"shellToolCall":{"args":{"command":"ls"},"result":{"success":{"stdout":"README.md"}}}}}
+{"type":"tool_call","subtype":"completed","call_id":"call-2","session_id":"session-1","model_call_id":"model-call-1","timestamp_ms":1700000001000,"tool_call":{"write_file":{"args":{"path":"/tmp/out.txt"},"result":"file created successfully"}}}
+{"type":"result","subtype":"success","duration_ms":1000,"duration_api_ms":1000,"is_error":false,"result":"Using tools now.","session_id":"session-1"}

packages/core/src/agents/cursor/fixtures/stream-json-live-composer-2.5.jsonl

@@ -0,0 +1,8 @@
+{"type":"system","subtype":"init","apiKeySource":"env","cwd":"/home/vivek/Desktop/projects/xulux/evals/evalprojects/docs/modules/m0-discovery/tmp-workspace","session_id":"cd97ac09-d300-408d-9f66-385fdb57f261","model":"Composer 2.5","permissionMode":"default"}
+{"type":"user","message":{"role":"user","content":[{"type":"text","text":"Reply with exactly: PONG. Do not use tools."}]},"session_id":"cd97ac09-d300-408d-9f66-385fdb57f261"}
+{"type":"thinking","subtype":"delta","text":"The response will be","session_id":"cd97ac09-d300-408d-9f66-385fdb57f261","timestamp_ms":1785848255137}
+{"type":"thinking","subtype":"delta","text":" exactly \"PONG\" without","session_id":"cd97ac09-d300-408d-9f66-385fdb57f261","timestamp_ms":1785848255142}
+{"type":"thinking","subtype":"delta","text":" using tools.","session_id":"cd97ac09-d300-408d-9f66-385fdb57f261","timestamp_ms":1785848255143}
+{"type":"thinking","subtype":"completed","session_id":"cd97ac09-d300-408d-9f66-385fdb57f261","timestamp_ms":1785848255144}
+{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"PONG"}]},"session_id":"cd97ac09-d300-408d-9f66-385fdb57f261"}
+{"type":"result","subtype":"success","duration_ms":7708,"duration_api_ms":7708,"is_error":false,"result":"PONG","session_id":"cd97ac09-d300-408d-9f66-385fdb57f261","request_id":"23d47179-318d-4c01-9dd6-8d1fbe54e8b0","usage":{"inputTokens":5544,"outputTokens":28,"cacheReadTokens":7002,"cacheWriteTokens":0}}

packages/core/src/agents/cursor/fixtures/stream-json-live-mcp-composer-2.5.jsonl

@@ -0,0 +1,21 @@
+{"type":"system","subtype":"init","apiKeySource":"env","cwd":"/home/vivek/Desktop/projects/xulux/evals/evalprojects/docs/modules/m0-discovery/tmp-workspace","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","model":"Composer 2.5","permissionMode":"default"}
+{"type":"user","message":{"role":"user","content":[{"type":"text","text":"You have an MCP server m0-ping with tool m0_ping. Call m0_ping once, then reply with the tool result text only."}]},"session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58"}
+{"type":"thinking","subtype":"delta","text":"I will call the m0_ping","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848345075}
+{"type":"thinking","subtype":"delta","text":" tool once and return","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848345080}
+{"type":"thinking","subtype":"delta","text":" only its result text.","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848345084}
+{"type":"thinking","subtype":"delta","text":"\n\nFirst I will fetch","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848345086}
+{"type":"thinking","subtype":"delta","text":" the MCP tool schema.","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848345090}
+{"type":"thinking","subtype":"completed","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848345093}
+{"type":"tool_call","subtype":"started","call_id":"tool_835acca9-b527-4742-b102-f346c24fc89","tool_call":{"getMcpToolsToolCall":{"args":{"server":"m0-ping","toolName":"m0_ping","toolCallId":"tool_835acca9-b527-4742-b102-f346c24fc89"}},"hookAdditionalContexts":[],"toolCallId":"tool_835acca9-b527-4742-b102-f346c24fc89","startedAtMs":"1785848344974"},"model_call_id":"55387b0b-56af-40ec-b2b1-c277b120d0cd-0-iosm","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848345103}
+{"type":"tool_call","subtype":"completed","call_id":"tool_835acca9-b527-4742-b102-f346c24fc89","tool_call":{"getMcpToolsToolCall":{"args":{"server":"m0-ping","toolName":"m0_ping","toolCallId":"tool_835acca9-b527-4742-b102-f346c24fc89"},"result":{"success":{"content":"{\n  \"mode\": \"single_tool\",\n  \"server\": \"m0-ping\",\n  \"serverStatus\": \"ready\",\n  \"tool\": {\n    \"tool\": \"m0_ping\",\n    \"description\": \"Return pong for Module 0 MCP probe\",\n    \"inputSchema\": {\n      \"type\": \"object\",\n      \"properties\": {\n        \"note\": {\n          \"type\": \"string\"\n        }\n      },\n      \"additionalProperties\": false\n    }\n  }\n}"}}},"hookAdditionalContexts":[],"toolCallId":"tool_835acca9-b527-4742-b102-f346c24fc89","startedAtMs":"1785848344974","completedAtMs":"1785848345191"},"model_call_id":"55387b0b-56af-40ec-b2b1-c277b120d0cd-0-iosm","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848345229}
+{"type":"thinking","subtype":"delta","text":"Calling m0_ping now.","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848348100}
+{"type":"thinking","subtype":"completed","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848348103}
+{"type":"tool_call","subtype":"started","call_id":"tool_35ba9454-cd3d-4cd9-89fd-3ee3afc9263","tool_call":{"mcpToolCall":{"args":{"name":"m0-ping-m0_ping","args":{},"toolCallId":"tool_35ba9454-cd3d-4cd9-89fd-3ee3afc9263","providerIdentifier":"m0-ping","toolName":"m0_ping","smartModeApprovalOnly":false,"skipApproval":false,"serverIdentifier":"m0-ping"},"description":"Call m0_ping probe tool once"},"hookAdditionalContexts":[],"toolCallId":"tool_35ba9454-cd3d-4cd9-89fd-3ee3afc9263","startedAtMs":"1785848348059"},"model_call_id":"55387b0b-56af-40ec-b2b1-c277b120d0cd-1-8ucj","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848348105}
+{"type":"tool_call","subtype":"completed","call_id":"tool_35ba9454-cd3d-4cd9-89fd-3ee3afc9263","tool_call":{"mcpToolCall":{"args":{"name":"m0-ping-m0_ping","args":{},"toolCallId":"tool_35ba9454-cd3d-4cd9-89fd-3ee3afc9263","providerIdentifier":"m0-ping","toolName":"m0_ping","smartModeApprovalOnly":false,"skipApproval":false,"serverIdentifier":"m0-ping"},"result":{"success":{"content":[{"text":{"text":"PONG_FROM_MCP"}}],"isError":false}},"description":"Call m0_ping probe tool once"},"hookAdditionalContexts":[],"toolCallId":"tool_35ba9454-cd3d-4cd9-89fd-3ee3afc9263","startedAtMs":"1785848348059","completedAtMs":"1785848348236"},"model_call_id":"55387b0b-56af-40ec-b2b1-c277b120d0cd-1-8ucj","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848348279}
+{"type":"thinking","subtype":"delta","text":"I will call the m0_ping","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848351924}
+{"type":"thinking","subtype":"delta","text":" tool once and return","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848351928}
+{"type":"thinking","subtype":"delta","text":" only the result text.","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848351929}
+{"type":"thinking","subtype":"delta","text":"\n\nThe result is \"PONG_FROM_MCP\".","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848351931}
+{"type":"thinking","subtype":"completed","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","timestamp_ms":1785848351932}
+{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"PONG_FROM_MCP"}]},"session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58"}
+{"type":"result","subtype":"success","duration_ms":13623,"duration_api_ms":13623,"is_error":false,"result":"PONG_FROM_MCP","session_id":"8c78e182-9de5-49a1-b412-5e9689b0be58","request_id":"55387b0b-56af-40ec-b2b1-c277b120d0cd","usage":{"inputTokens":4998,"outputTokens":177,"cacheReadTokens":36298,"cacheWriteTokens":0}}

packages/core/src/agents/cursor/index.test.ts

@@ -0,0 +1,33 @@
+import { describe, expect, it } from 'vitest';
+import { createParser, supportedParsers } from '../registry.js';
+import { cursorAgent, cursorDefinition } from './index.js';
+import { cursorParser } from './parser.js';
+
+describe('cursorAgent wiring', () => {
+  it('registers the cursor parser in the agent registry', () => {
+    expect(supportedParsers()).toContain('cursor');
+    expect(createParser('cursor')).toBe(cursorParser);
+    expect(cursorDefinition.parser).toBe(cursorParser);
+  });
+
+  it('builds a harness with cursor metadata defaults', () => {
+    const harness = cursorAgent();
+    expect(harness.id).toBe('cursor');
+    expect(harness.modelId).toBe('composer-2.5');
+    expect(harness.metadata).toEqual({
+      agent: 'cursor',
+      modelProvider: 'cursor',
+      modelId: 'composer-2.5',
+    });
+    expect(harness.runsInSandbox).toBe(true);
+  });
+
+  it('accepts model and reasoningEffort overrides', () => {
+    const harness = cursorAgent({
+      model: 'cursor-grok-4.5-high',
+      reasoningEffort: 'high',
+    });
+    expect(harness.modelId).toBe('cursor-grok-4.5-high');
+    expect(harness.metadata.reasoningEffort).toBe('high');
+  });
+});

packages/core/src/agents/cursor/index.ts

@@ -0,0 +1,40 @@
+/**
+ * Cursor agent. Owns everything Cursor-specific: it wires its own runner +
+ * parser into the public `cursorAgent` factory (via the generic `createCliAgent`
+ * engine) and exports the registry definition the harness uses to parse Cursor
+ * transcripts.
+ */
+
+import type { AgentHarness } from '../../index.js';
+import type { ReasoningEffortLevel } from '../../eval-metadata.js';
+import { createCliAgent } from '../engine.js';
+import type { AgentDefinition } from '../types.js';
+import { cursorParser } from './parser.js';
+import { cursorRunner, type CursorModel } from './runner.js';
+
+/** Cursor CLI agent as an `AgentHarness`. */
+export function cursorAgent(
+  options: {
+    /** Cursor model id from `cursor-agent --list-models` (e.g. composer-2.5). */
+    model?: CursorModel;
+    /**
+     * Reasoning effort applied as Cursor's `model[effort=…]` syntax.
+     * Omit to use Cursor's default for the model.
+     */
+    reasoningEffort?: ReasoningEffortLevel;
+    /** Override the pinned CLI tarball version. */
+    cliVersion?: string;
+  } = {}
+): AgentHarness {
+  return createCliAgent(cursorRunner, cursorParser, {
+    model: options.model ?? cursorRunner.defaultModel,
+    reasoningEffort: options.reasoningEffort,
+    cliVersion: options.cliVersion,
+  });
+}
+
+/** Runner + parser pairing for the agent registry (id comes from `runner.id`). */
+export const cursorDefinition: AgentDefinition = {
+  runner: cursorRunner,
+  parser: cursorParser,
+};