feat(agents): add Mistral Vibe as a supported agent harness

.changeset/add-mistral-vibe-agent.md

@@ -0,0 +1,5 @@
+---
+"@vercel/agent-eval": minor
+---
+
+Add Mistral Vibe as a supported agent harness, enabling native evaluation of Mistral's coding agent with Devstral models

README.md

@@ -226,10 +226,11 @@ agent: 'vercel-ai-gateway/codex'        // OpenAI Codex via AI Gateway
 agent: 'vercel-ai-gateway/opencode'     // OpenCode via AI Gateway
 
 // Direct API (uses provider keys directly)
-agent: 'claude-code'  // requires ANTHROPIC_API_KEY
-agent: 'codex'        // requires OPENAI_API_KEY
-agent: 'gemini'       // requires GEMINI_API_KEY
-agent: 'cursor'       // requires CURSOR_API_KEY
+agent: 'claude-code'    // requires ANTHROPIC_API_KEY
+agent: 'codex'          // requires OPENAI_API_KEY
+agent: 'gemini'         // requires GEMINI_API_KEY
+agent: 'cursor'         // requires CURSOR_API_KEY
+agent: 'mistral-vibe'   // requires MISTRAL_API_KEY
 ```
 
 ### Multi-model experiments
@@ -448,6 +449,7 @@ Every run requires an API key for the agent and a token for the sandbox. Classif
 | `OPENAI_API_KEY`     | `agent: 'codex'`                       | Direct OpenAI API key                                                                        |
 | `GEMINI_API_KEY`     | `agent: 'gemini'`                      | Direct Google Gemini API key                                                                 |
 | `CURSOR_API_KEY`     | `agent: 'cursor'`                      | Direct Cursor API key                                                                        |
+| `MISTRAL_API_KEY`    | `agent: 'mistral-vibe'`                | Direct Mistral API key                                                                       |
 | `VERCEL_TOKEN`       | Always (pick one)                      | Vercel personal access token -- for local dev                                                |
 | `VERCEL_OIDC_TOKEN`  | Always (pick one) OR for classifier    | Vercel OIDC token -- for CI/CD pipelines, or enables classifier without `AI_GATEWAY_API_KEY` |
 

packages/agent-eval/src/integration.test.ts

@@ -44,6 +44,7 @@ const hasAnthropicCredentials = !!process.env.ANTHROPIC_API_KEY && hasSandbox;
 const hasOpenAiCredentials = !!process.env.OPENAI_API_KEY && hasSandbox;
 const hasGeminiCredentials = !!process.env.GEMINI_API_KEY && hasSandbox;
 const hasCursorCredentials = !!process.env.CURSOR_API_KEY && hasSandbox;
+const hasMistralCredentials = !!process.env.MISTRAL_API_KEY && hasSandbox;
 // OpenCode credentials (only supports AI Gateway)
 const hasOpenCodeCredentials = hasAiGatewayCredentials;
 
@@ -902,6 +903,151 @@ test('contains greeting', () => {
     }, 300000); // 5 minute timeout
   });
 
+  describe.skipIf(!hasMistralCredentials)('Mistral Vibe (Direct API) sandbox execution', () => {
+    it('can run a simple eval with Mistral Vibe', async () => {
+      const fixtureDir = join(TEST_DIR, 'simple-eval-vibe');
+      mkdirSync(join(fixtureDir, 'src'), { recursive: true });
+
+      writeFileSync(
+        join(fixtureDir, 'PROMPT.md'),
+        'Add a function called greet that returns "Hello from Vibe!"'
+      );
+      writeFileSync(
+        join(fixtureDir, 'EVAL.ts'),
+        `
+import { test, expect } from 'vitest';
+import { readFileSync } from 'fs';
+
+test('greet exists', () => {
+  const content = readFileSync('src/index.ts', 'utf-8');
+  expect(content).toContain('greet');
+});
+`
+      );
+      writeFileSync(
+        join(fixtureDir, 'package.json'),
+        JSON.stringify({
+          name: 'simple-eval-vibe',
+          type: 'module',
+          scripts: { build: 'tsc' },
+          devDependencies: { typescript: '^5.0.0', vitest: '^2.1.0' },
+        })
+      );
+      writeFileSync(
+        join(fixtureDir, 'tsconfig.json'),
+        JSON.stringify({
+          compilerOptions: {
+            target: 'ES2020',
+            module: 'ESNext',
+            moduleResolution: 'bundler',
+            outDir: 'dist',
+          },
+          include: ['src'],
+        })
+      );
+      writeFileSync(join(fixtureDir, 'src/index.ts'), '// TODO: implement');
+
+      const fixture = loadFixture(TEST_DIR, 'simple-eval-vibe');
+
+      const result = await runSingleEval(fixture, {
+        agent: 'mistral-vibe',
+        model: 'devstral-2',
+        timeout: 180,
+        apiKey: process.env.MISTRAL_API_KEY!,
+        scripts: ['build'],
+      });
+
+      expect(result.result.duration).toBeGreaterThan(0);
+      expect(result.result.status).toBeDefined();
+      if (result.result.status === 'failed') {
+        console.error('Agent failed with error:', result.result.error);
+      }
+      expect(result.result.status).toBe('passed');
+
+      if (result.outputContent) {
+        expect(typeof result.outputContent).toBe('object');
+      }
+    }, 300000);
+
+    it('verifies Mistral Vibe result output structure matches expected format', async () => {
+      const fixtureDir = join(TEST_DIR, 'result-structure-vibe');
+      mkdirSync(join(fixtureDir, 'src'), { recursive: true });
+
+      writeFileSync(
+        join(fixtureDir, 'PROMPT.md'),
+        'Create a simple hello.ts file that exports a greeting constant.'
+      );
+      writeFileSync(
+        join(fixtureDir, 'EVAL.ts'),
+        `
+import { test, expect } from 'vitest';
+import { readFileSync, existsSync } from 'fs';
+
+test('hello.ts exists', () => {
+  expect(existsSync('src/hello.ts')).toBe(true);
+});
+
+test('contains greeting', () => {
+  const content = readFileSync('src/hello.ts', 'utf-8');
+  expect(content).toContain('greeting');
+});
+`
+      );
+      writeFileSync(
+        join(fixtureDir, 'package.json'),
+        JSON.stringify({
+          name: 'result-structure-vibe',
+          type: 'module',
+          scripts: { build: 'tsc' },
+          devDependencies: { typescript: '^5.0.0', vitest: '^2.1.0' },
+        })
+      );
+      writeFileSync(
+        join(fixtureDir, 'tsconfig.json'),
+        JSON.stringify({
+          compilerOptions: {
+            target: 'ES2020',
+            module: 'ESNext',
+            moduleResolution: 'bundler',
+            outDir: 'dist',
+          },
+          include: ['src'],
+        })
+      );
+
+      const fixture = loadFixture(TEST_DIR, 'result-structure-vibe');
+
+      const result = await runSingleEval(fixture, {
+        agent: 'mistral-vibe',
+        model: 'devstral-2',
+        timeout: 180,
+        apiKey: process.env.MISTRAL_API_KEY!,
+        scripts: ['build'],
+      });
+
+      expect(result).toHaveProperty('result');
+      expect(result.result).toHaveProperty('status');
+      expect(result.result).toHaveProperty('duration');
+
+      if (result.result.error) {
+        expect(typeof result.result.error).toBe('string');
+      }
+
+      if (result.transcript) {
+        expect(typeof result.transcript).toBe('string');
+      }
+
+      if (result.outputContent) {
+        if (result.outputContent.eval) {
+          expect(typeof result.outputContent.eval).toBe('string');
+        }
+        if (result.outputContent.scripts?.build) {
+          expect(typeof result.outputContent.scripts.build).toBe('string');
+        }
+      }
+    }, 300000);
+  });
+
   describe.skipIf(!hasOpenCodeCredentials)('OpenCode (Vercel AI Gateway) sandbox execution', () => {
     it('can run a simple eval with OpenCode', async () => {
       // Create a simple test fixture

packages/agent-eval/src/lib/agents/index.ts

@@ -8,6 +8,7 @@ import { createCodexAgent } from './codex.js';
 import { createOpenCodeAgent } from './opencode.js';
 import { createGeminiAgent } from './gemini.js';
 import { createCursorAgent } from './cursor.js';
+import { createMistralVibeAgent } from './mistral-vibe.js';
 
 // Register all agent variants (Vercel AI Gateway + Direct API)
 registerAgent(createClaudeCodeAgent({ useVercelAiGateway: true }));   // vercel-ai-gateway/claude-code
@@ -17,6 +18,7 @@ registerAgent(createCodexAgent({ useVercelAiGateway: false }));       // codex
 registerAgent(createOpenCodeAgent());                                 // vercel-ai-gateway/opencode
 registerAgent(createGeminiAgent());                                   // gemini
 registerAgent(createCursorAgent());                                   // cursor
+registerAgent(createMistralVibeAgent());                              // mistral-vibe
 
 // Re-export registry functions
 export { registerAgent, getAgent, listAgents, hasAgent };