From f90433a4f84d212c5cf4c0361de6cf71951526c7 Mon Sep 17 00:00:00 2001 From: Benjamin Canac Date: Wed, 1 Apr 2026 15:14:51 +0200 Subject: [PATCH] feat(agents): add Mistral Vibe as a supported agent harness --- .changeset/add-mistral-vibe-agent.md | 5 + README.md | 10 +- package-lock.json | 2 +- packages/agent-eval/src/integration.test.ts | 146 +++++++++ packages/agent-eval/src/lib/agents/index.ts | 2 + .../agent-eval/src/lib/agents/mistral-vibe.ts | 231 ++++++++++++++ packages/agent-eval/src/lib/agents/shared.ts | 7 + packages/agent-eval/src/lib/config.ts | 1 + packages/agent-eval/src/lib/o11y/index.ts | 1 + packages/agent-eval/src/lib/o11y/o11y.test.ts | 204 +++++++++++- .../agent-eval/src/lib/o11y/parsers/index.ts | 5 +- .../src/lib/o11y/parsers/mistral-vibe.ts | 292 ++++++++++++++++++ packages/agent-eval/src/lib/types.ts | 3 +- 13 files changed, 901 insertions(+), 8 deletions(-) create mode 100644 .changeset/add-mistral-vibe-agent.md create mode 100644 packages/agent-eval/src/lib/agents/mistral-vibe.ts create mode 100644 packages/agent-eval/src/lib/o11y/parsers/mistral-vibe.ts diff --git a/.changeset/add-mistral-vibe-agent.md b/.changeset/add-mistral-vibe-agent.md new file mode 100644 index 0000000..ef185bb --- /dev/null +++ b/.changeset/add-mistral-vibe-agent.md @@ -0,0 +1,5 @@ +--- +"@vercel/agent-eval": minor +--- + +Add Mistral Vibe as a supported agent harness, enabling native evaluation of Mistral's coding agent with Devstral models diff --git a/README.md b/README.md index f1346ba..d0442d8 100644 --- a/README.md +++ b/README.md @@ -226,10 +226,11 @@ agent: 'vercel-ai-gateway/codex' // OpenAI Codex via AI Gateway agent: 'vercel-ai-gateway/opencode' // OpenCode via AI Gateway // Direct API (uses provider keys directly) -agent: 'claude-code' // requires ANTHROPIC_API_KEY -agent: 'codex' // requires OPENAI_API_KEY -agent: 'gemini' // requires GEMINI_API_KEY -agent: 'cursor' // requires CURSOR_API_KEY +agent: 'claude-code' // requires ANTHROPIC_API_KEY +agent: 'codex' // requires OPENAI_API_KEY +agent: 'gemini' // requires GEMINI_API_KEY +agent: 'cursor' // requires CURSOR_API_KEY +agent: 'mistral-vibe' // requires MISTRAL_API_KEY ``` ### Multi-model experiments @@ -448,6 +449,7 @@ Every run requires an API key for the agent and a token for the sandbox. Classif | `OPENAI_API_KEY` | `agent: 'codex'` | Direct OpenAI API key | | `GEMINI_API_KEY` | `agent: 'gemini'` | Direct Google Gemini API key | | `CURSOR_API_KEY` | `agent: 'cursor'` | Direct Cursor API key | +| `MISTRAL_API_KEY` | `agent: 'mistral-vibe'` | Direct Mistral API key | | `VERCEL_TOKEN` | Always (pick one) | Vercel personal access token -- for local dev | | `VERCEL_OIDC_TOKEN` | Always (pick one) OR for classifier | Vercel OIDC token -- for CI/CD pipelines, or enables classifier without `AI_GATEWAY_API_KEY` | diff --git a/package-lock.json b/package-lock.json index b57d16f..5d737a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9070,7 +9070,7 @@ }, "packages/agent-eval": { "name": "@vercel/agent-eval", - "version": "0.9.3", + "version": "0.9.5", "license": "MIT", "dependencies": { "@ai-sdk/anthropic": "^1.2.12", diff --git a/packages/agent-eval/src/integration.test.ts b/packages/agent-eval/src/integration.test.ts index 9dd8237..639b95e 100644 --- a/packages/agent-eval/src/integration.test.ts +++ b/packages/agent-eval/src/integration.test.ts @@ -44,6 +44,7 @@ const hasAnthropicCredentials = !!process.env.ANTHROPIC_API_KEY && hasSandbox; const hasOpenAiCredentials = !!process.env.OPENAI_API_KEY && hasSandbox; const hasGeminiCredentials = !!process.env.GEMINI_API_KEY && hasSandbox; const hasCursorCredentials = !!process.env.CURSOR_API_KEY && hasSandbox; +const hasMistralCredentials = !!process.env.MISTRAL_API_KEY && hasSandbox; // OpenCode credentials (only supports AI Gateway) const hasOpenCodeCredentials = hasAiGatewayCredentials; @@ -902,6 +903,151 @@ test('contains greeting', () => { }, 300000); // 5 minute timeout }); + describe.skipIf(!hasMistralCredentials)('Mistral Vibe (Direct API) sandbox execution', () => { + it('can run a simple eval with Mistral Vibe', async () => { + const fixtureDir = join(TEST_DIR, 'simple-eval-vibe'); + mkdirSync(join(fixtureDir, 'src'), { recursive: true }); + + writeFileSync( + join(fixtureDir, 'PROMPT.md'), + 'Add a function called greet that returns "Hello from Vibe!"' + ); + writeFileSync( + join(fixtureDir, 'EVAL.ts'), + ` +import { test, expect } from 'vitest'; +import { readFileSync } from 'fs'; + +test('greet exists', () => { + const content = readFileSync('src/index.ts', 'utf-8'); + expect(content).toContain('greet'); +}); +` + ); + writeFileSync( + join(fixtureDir, 'package.json'), + JSON.stringify({ + name: 'simple-eval-vibe', + type: 'module', + scripts: { build: 'tsc' }, + devDependencies: { typescript: '^5.0.0', vitest: '^2.1.0' }, + }) + ); + writeFileSync( + join(fixtureDir, 'tsconfig.json'), + JSON.stringify({ + compilerOptions: { + target: 'ES2020', + module: 'ESNext', + moduleResolution: 'bundler', + outDir: 'dist', + }, + include: ['src'], + }) + ); + writeFileSync(join(fixtureDir, 'src/index.ts'), '// TODO: implement'); + + const fixture = loadFixture(TEST_DIR, 'simple-eval-vibe'); + + const result = await runSingleEval(fixture, { + agent: 'mistral-vibe', + model: 'devstral-2', + timeout: 180, + apiKey: process.env.MISTRAL_API_KEY!, + scripts: ['build'], + }); + + expect(result.result.duration).toBeGreaterThan(0); + expect(result.result.status).toBeDefined(); + if (result.result.status === 'failed') { + console.error('Agent failed with error:', result.result.error); + } + expect(result.result.status).toBe('passed'); + + if (result.outputContent) { + expect(typeof result.outputContent).toBe('object'); + } + }, 300000); + + it('verifies Mistral Vibe result output structure matches expected format', async () => { + const fixtureDir = join(TEST_DIR, 'result-structure-vibe'); + mkdirSync(join(fixtureDir, 'src'), { recursive: true }); + + writeFileSync( + join(fixtureDir, 'PROMPT.md'), + 'Create a simple hello.ts file that exports a greeting constant.' + ); + writeFileSync( + join(fixtureDir, 'EVAL.ts'), + ` +import { test, expect } from 'vitest'; +import { readFileSync, existsSync } from 'fs'; + +test('hello.ts exists', () => { + expect(existsSync('src/hello.ts')).toBe(true); +}); + +test('contains greeting', () => { + const content = readFileSync('src/hello.ts', 'utf-8'); + expect(content).toContain('greeting'); +}); +` + ); + writeFileSync( + join(fixtureDir, 'package.json'), + JSON.stringify({ + name: 'result-structure-vibe', + type: 'module', + scripts: { build: 'tsc' }, + devDependencies: { typescript: '^5.0.0', vitest: '^2.1.0' }, + }) + ); + writeFileSync( + join(fixtureDir, 'tsconfig.json'), + JSON.stringify({ + compilerOptions: { + target: 'ES2020', + module: 'ESNext', + moduleResolution: 'bundler', + outDir: 'dist', + }, + include: ['src'], + }) + ); + + const fixture = loadFixture(TEST_DIR, 'result-structure-vibe'); + + const result = await runSingleEval(fixture, { + agent: 'mistral-vibe', + model: 'devstral-2', + timeout: 180, + apiKey: process.env.MISTRAL_API_KEY!, + scripts: ['build'], + }); + + expect(result).toHaveProperty('result'); + expect(result.result).toHaveProperty('status'); + expect(result.result).toHaveProperty('duration'); + + if (result.result.error) { + expect(typeof result.result.error).toBe('string'); + } + + if (result.transcript) { + expect(typeof result.transcript).toBe('string'); + } + + if (result.outputContent) { + if (result.outputContent.eval) { + expect(typeof result.outputContent.eval).toBe('string'); + } + if (result.outputContent.scripts?.build) { + expect(typeof result.outputContent.scripts.build).toBe('string'); + } + } + }, 300000); + }); + describe.skipIf(!hasOpenCodeCredentials)('OpenCode (Vercel AI Gateway) sandbox execution', () => { it('can run a simple eval with OpenCode', async () => { // Create a simple test fixture diff --git a/packages/agent-eval/src/lib/agents/index.ts b/packages/agent-eval/src/lib/agents/index.ts index 7312064..eb089b3 100644 --- a/packages/agent-eval/src/lib/agents/index.ts +++ b/packages/agent-eval/src/lib/agents/index.ts @@ -8,6 +8,7 @@ import { createCodexAgent } from './codex.js'; import { createOpenCodeAgent } from './opencode.js'; import { createGeminiAgent } from './gemini.js'; import { createCursorAgent } from './cursor.js'; +import { createMistralVibeAgent } from './mistral-vibe.js'; // Register all agent variants (Vercel AI Gateway + Direct API) registerAgent(createClaudeCodeAgent({ useVercelAiGateway: true })); // vercel-ai-gateway/claude-code @@ -17,6 +18,7 @@ registerAgent(createCodexAgent({ useVercelAiGateway: false })); // codex registerAgent(createOpenCodeAgent()); // vercel-ai-gateway/opencode registerAgent(createGeminiAgent()); // gemini registerAgent(createCursorAgent()); // cursor +registerAgent(createMistralVibeAgent()); // mistral-vibe // Re-export registry functions export { registerAgent, getAgent, listAgents, hasAgent }; diff --git a/packages/agent-eval/src/lib/agents/mistral-vibe.ts b/packages/agent-eval/src/lib/agents/mistral-vibe.ts new file mode 100644 index 0000000..169d0df --- /dev/null +++ b/packages/agent-eval/src/lib/agents/mistral-vibe.ts @@ -0,0 +1,231 @@ +/** + * Mistral Vibe CLI agent implementation. + * Uses direct Mistral API access. + */ + +import type { Agent, AgentRunOptions, AgentRunResult } from './types.js'; +import type { ModelTier } from '../types.js'; +import { + createSandbox, + collectLocalFiles, + splitTestFiles, + verifyNoTestFiles, + type SandboxManager, +} from '../sandbox.js'; +import type { DockerSandboxManager } from '../docker-sandbox.js'; +import { + runValidation, + captureGeneratedFiles, + createVitestConfig, + MISTRAL_DIRECT, + initGitAndCommit, + injectTranscriptContext, +} from './shared.js'; + +/** Union type for sandbox implementations */ +type AnySandbox = SandboxManager | DockerSandboxManager; + +/** + * Extract transcript from Vibe streaming output. + * When run with --output streaming, Vibe outputs newline-delimited JSON per message. + */ +function extractTranscriptFromOutput(output: string): string | undefined { + if (!output || !output.trim()) { + return undefined; + } + + const lines = output.split('\n').filter(line => { + const trimmed = line.trim(); + return trimmed.startsWith('{') && trimmed.endsWith('}'); + }); + + if (lines.length === 0) { + return undefined; + } + + return lines.join('\n'); +} + +/** + * Create Mistral Vibe agent with direct API authentication. + */ +export function createMistralVibeAgent(): Agent { + return { + name: 'mistral-vibe', + displayName: 'Mistral Vibe', + + getApiKeyEnvVar(): string { + return MISTRAL_DIRECT.apiKeyEnvVar; + }, + + getDefaultModel(): ModelTier { + return 'devstral-2'; + }, + + async run(fixturePath: string, options: AgentRunOptions): Promise { + const startTime = Date.now(); + let sandbox: AnySandbox | null = null; + let agentOutput = ''; + let transcript: string | undefined; + let aborted = false; + let sandboxStopped = false; + + const abortHandler = () => { + aborted = true; + if (sandbox && !sandboxStopped) { + sandboxStopped = true; + sandbox.stop().catch(() => {}); + } + }; + + if (options.signal) { + if (options.signal.aborted) { + return { + success: false, + output: '', + error: 'Aborted before start', + duration: 0, + }; + } + options.signal.addEventListener('abort', abortHandler); + } + + try { + const allFiles = await collectLocalFiles(fixturePath); + const { workspaceFiles, testFiles } = splitTestFiles(allFiles); + + if (aborted) { + return { + success: false, + output: '', + error: 'Aborted', + duration: Date.now() - startTime, + }; + } + + sandbox = await createSandbox({ + timeout: options.timeout, + runtime: 'node24', + backend: options.sandbox, + }); + + if (aborted) { + return { + success: false, + output: '', + error: 'Aborted', + duration: Date.now() - startTime, + sandboxId: sandbox.sandboxId, + }; + } + + await sandbox.uploadFiles(workspaceFiles); + + await initGitAndCommit(sandbox); + + if (options.setup) { + await options.setup(sandbox); + } + + // Install dependencies + let installResult = await sandbox.runCommand('npm', ['install']); + if (installResult.exitCode !== 0) { + installResult = await sandbox.runCommand('npm', ['install']); + } + if (installResult.exitCode !== 0) { + const output = (installResult.stdout + installResult.stderr).trim().split('\n').slice(-10).join('\n'); + throw new Error(`npm install failed (exit code ${installResult.exitCode}):\n${output}`); + } + + // Install Vibe CLI via the official install script + const cliInstall = await sandbox.runShell( + 'curl -LsSf https://mistral.ai/vibe/install.sh | bash' + ); + if (cliInstall.exitCode !== 0) { + throw new Error(`Vibe CLI install failed: ${cliInstall.stderr}`); + } + + // Ensure vibe is on PATH (the install script puts it in ~/.local/bin) + const pathSetup = await sandbox.runShell('export PATH="$HOME/.local/bin:$PATH" && which vibe'); + if (pathSetup.exitCode !== 0) { + throw new Error(`Vibe CLI not found on PATH after install: ${pathSetup.stderr}`); + } + + await verifyNoTestFiles(sandbox); + + // Run Vibe in programmatic mode with streaming output for transcript + const escapedPrompt = options.prompt.replace(/'/g, "'\\''"); + const vibeResult = await sandbox.runShell( + `export PATH="$HOME/.local/bin:$PATH" && vibe --prompt '${escapedPrompt}' --model ${options.model} --output streaming`, + { + [MISTRAL_DIRECT.apiKeyEnvVar]: options.apiKey, + } + ); + + agentOutput = vibeResult.stdout + vibeResult.stderr; + transcript = extractTranscriptFromOutput(agentOutput); + + if (vibeResult.exitCode !== 0) { + const errorLines = agentOutput.trim().split('\n').slice(-5).join('\n'); + return { + success: false, + output: agentOutput, + transcript, + error: errorLines || `Vibe CLI exited with code ${vibeResult.exitCode}`, + duration: Date.now() - startTime, + sandboxId: sandbox.sandboxId, + }; + } + + await sandbox.uploadFiles(testFiles); + + await createVitestConfig(sandbox); + + await injectTranscriptContext(sandbox, transcript, 'mistral-vibe', options.model); + + const validationResults = await runValidation(sandbox, options.scripts ?? []); + + const { generatedFiles, deletedFiles } = await captureGeneratedFiles(sandbox); + + return { + success: validationResults.allPassed, + output: agentOutput, + transcript, + duration: Date.now() - startTime, + testResult: validationResults.test, + scriptsResults: validationResults.scripts, + sandboxId: sandbox.sandboxId, + generatedFiles, + deletedFiles, + }; + } catch (error) { + if (aborted) { + return { + success: false, + output: agentOutput, + transcript, + error: 'Aborted', + duration: Date.now() - startTime, + sandboxId: sandbox?.sandboxId, + }; + } + return { + success: false, + output: agentOutput, + transcript, + error: error instanceof Error ? error.message : String(error), + duration: Date.now() - startTime, + sandboxId: sandbox?.sandboxId, + }; + } finally { + if (options.signal) { + options.signal.removeEventListener('abort', abortHandler); + } + if (sandbox && !sandboxStopped) { + sandboxStopped = true; + await sandbox.stop(); + } + } + }, + }; +} diff --git a/packages/agent-eval/src/lib/agents/shared.ts b/packages/agent-eval/src/lib/agents/shared.ts index 414004a..58db80a 100644 --- a/packages/agent-eval/src/lib/agents/shared.ts +++ b/packages/agent-eval/src/lib/agents/shared.ts @@ -243,3 +243,10 @@ export const GEMINI_DIRECT = { export const CURSOR_DIRECT = { apiKeyEnvVar: 'CURSOR_API_KEY', } as const; + +/** + * Direct API configuration for Mistral. + */ +export const MISTRAL_DIRECT = { + apiKeyEnvVar: 'MISTRAL_API_KEY', +} as const; diff --git a/packages/agent-eval/src/lib/config.ts b/packages/agent-eval/src/lib/config.ts index a6869e9..e4a6a60 100644 --- a/packages/agent-eval/src/lib/config.ts +++ b/packages/agent-eval/src/lib/config.ts @@ -37,6 +37,7 @@ const experimentConfigSchema = z.object({ 'vercel-ai-gateway/opencode', 'gemini', 'cursor', + 'mistral-vibe', ]), model: z.union([z.string(), z.array(z.string())]).optional(), evals: z diff --git a/packages/agent-eval/src/lib/o11y/index.ts b/packages/agent-eval/src/lib/o11y/index.ts index f8f65f4..001ed68 100644 --- a/packages/agent-eval/src/lib/o11y/index.ts +++ b/packages/agent-eval/src/lib/o11y/index.ts @@ -24,3 +24,4 @@ export { parseCodexTranscript } from './parsers/codex.js'; export { parseOpenCodeTranscript } from './parsers/opencode.js'; export { parseGeminiTranscript } from './parsers/gemini.js'; export { parseCursorTranscript } from './parsers/cursor.js'; +export { parseMistralVibeTranscript } from './parsers/mistral-vibe.js'; diff --git a/packages/agent-eval/src/lib/o11y/o11y.test.ts b/packages/agent-eval/src/lib/o11y/o11y.test.ts index af9128f..98af12e 100644 --- a/packages/agent-eval/src/lib/o11y/o11y.test.ts +++ b/packages/agent-eval/src/lib/o11y/o11y.test.ts @@ -10,6 +10,7 @@ import { parseCodexTranscript } from './parsers/codex.js'; import { parseOpenCodeTranscript } from './parsers/opencode.js'; import { parseGeminiTranscript } from './parsers/gemini.js'; import { parseCursorTranscript } from './parsers/cursor.js'; +import { parseMistralVibeTranscript } from './parsers/mistral-vibe.js'; describe('o11y', () => { describe('parseTranscript', () => { @@ -38,6 +39,9 @@ describe('o11y', () => { const cursorResult = parseTranscript(claudeTranscript, 'cursor'); expect(cursorResult.agent).toBe('cursor'); + + const vibeResult = parseTranscript(claudeTranscript, 'mistral-vibe'); + expect(vibeResult.agent).toBe('mistral-vibe'); }); it('returns parseSuccess: false for unsupported agents', () => { @@ -47,7 +51,7 @@ describe('o11y', () => { expect(result.parseSuccess).toBe(false); expect(result.parseErrors).toContain( - 'No parser available for agent: unsupported-agent. Supported agents: claude-code, codex, opencode, gemini, cursor' + 'No parser available for agent: unsupported-agent. Supported agents: claude-code, codex, opencode, gemini, cursor, mistral-vibe' ); expect(result.events).toEqual([]); expect(result.summary.totalToolCalls).toBe(0); @@ -737,6 +741,204 @@ describe('o11y', () => { }); }); + describe('Mistral Vibe parser', () => { + it('parses user_message events', () => { + const transcript = JSON.stringify({ + type: 'user_message', + content: 'Fix the bug in main.ts', + message_id: 'msg_123', + }); + const { events } = parseMistralVibeTranscript(transcript); + + expect(events).toHaveLength(1); + expect(events[0].type).toBe('message'); + expect(events[0].role).toBe('user'); + expect(events[0].content).toBe('Fix the bug in main.ts'); + }); + + it('parses assistant events', () => { + const transcript = JSON.stringify({ + type: 'assistant', + content: 'I will fix the bug now.', + message_id: 'msg_456', + }); + const { events } = parseMistralVibeTranscript(transcript); + + expect(events).toHaveLength(1); + expect(events[0].type).toBe('message'); + expect(events[0].role).toBe('assistant'); + expect(events[0].content).toBe('I will fix the bug now.'); + }); + + it('parses reasoning events as thinking', () => { + const transcript = JSON.stringify({ + type: 'reasoning', + content: 'Let me analyze the code structure...', + message_id: 'msg_789', + }); + const { events } = parseMistralVibeTranscript(transcript); + + expect(events).toHaveLength(1); + expect(events[0].type).toBe('thinking'); + expect(events[0].content).toBe('Let me analyze the code structure...'); + }); + + it('parses tool_call events', () => { + const transcript = JSON.stringify({ + type: 'tool_call', + tool_name: 'bash', + args: { command: 'npm test' }, + tool_call_id: 'tc_001', + }); + const { events } = parseMistralVibeTranscript(transcript); + + expect(events).toHaveLength(1); + expect(events[0].type).toBe('tool_call'); + expect(events[0].tool?.name).toBe('shell'); + expect(events[0].tool?.originalName).toBe('bash'); + expect(events[0].tool?.args?._extractedCommand).toBe('npm test'); + }); + + it('parses tool_result events', () => { + const transcript = JSON.stringify({ + type: 'tool_result', + tool_name: 'bash', + result: { output: 'All tests passed' }, + error: null, + skipped: false, + duration: 2.5, + tool_call_id: 'tc_001', + }); + const { events } = parseMistralVibeTranscript(transcript); + + expect(events).toHaveLength(1); + expect(events[0].type).toBe('tool_result'); + expect(events[0].tool?.name).toBe('shell'); + expect(events[0].tool?.success).toBe(true); + }); + + it('parses failed tool_result events', () => { + const transcript = JSON.stringify({ + type: 'tool_result', + tool_name: 'bash', + result: null, + error: 'Command not found', + tool_call_id: 'tc_002', + }); + const { events } = parseMistralVibeTranscript(transcript); + + expect(events).toHaveLength(1); + expect(events[0].tool?.success).toBe(false); + }); + + it('normalizes Vibe tool names', () => { + const tools = [ + { name: 'read_file', expected: 'file_read' }, + { name: 'write_file', expected: 'file_write' }, + { name: 'search_replace', expected: 'file_edit' }, + { name: 'bash', expected: 'shell' }, + { name: 'grep', expected: 'grep' }, + { name: 'task', expected: 'agent_task' }, + { name: 'todo', expected: 'agent_task' }, + { name: 'unknown_tool', expected: 'unknown' }, + ]; + + for (const { name, expected } of tools) { + const transcript = JSON.stringify({ + type: 'tool_call', + tool_name: name, + args: {}, + tool_call_id: 'tc_test', + }); + const { events } = parseMistralVibeTranscript(transcript); + expect(events[0].tool?.name).toBe(expected); + } + }); + + it('extracts file paths from tool args', () => { + const transcript = JSON.stringify({ + type: 'tool_call', + tool_name: 'read_file', + args: { file_path: 'src/index.ts' }, + tool_call_id: 'tc_003', + }); + const { events } = parseMistralVibeTranscript(transcript); + + expect(events[0].tool?.args?._extractedPath).toBe('src/index.ts'); + }); + + it('extracts commands from shell tool args', () => { + const transcript = JSON.stringify({ + type: 'tool_call', + tool_name: 'bash', + args: { command: 'npm install' }, + tool_call_id: 'tc_004', + }); + const { events } = parseMistralVibeTranscript(transcript); + + expect(events[0].tool?.args?._extractedCommand).toBe('npm install'); + }); + + it('skips tool_stream events', () => { + const transcript = JSON.stringify({ + type: 'tool_stream', + tool_name: 'bash', + message: 'Installing packages...', + tool_call_id: 'tc_005', + }); + const { events } = parseMistralVibeTranscript(transcript); + + expect(events).toHaveLength(0); + }); + + it('handles a full streaming transcript end-to-end', () => { + const transcript = [ + JSON.stringify({ type: 'user_message', content: 'Create a hello world app', message_id: 'msg_1' }), + JSON.stringify({ type: 'assistant', content: 'I will create the app.', message_id: 'msg_2' }), + JSON.stringify({ type: 'tool_call', tool_name: 'read_file', args: { file_path: 'package.json' }, tool_call_id: 'tc_1' }), + JSON.stringify({ type: 'tool_result', tool_name: 'read_file', result: '{}', error: null, tool_call_id: 'tc_1' }), + JSON.stringify({ type: 'tool_call', tool_name: 'bash', args: { command: 'mkdir src' }, tool_call_id: 'tc_2' }), + JSON.stringify({ type: 'tool_result', tool_name: 'bash', result: '', error: null, tool_call_id: 'tc_2' }), + JSON.stringify({ type: 'tool_call', tool_name: 'write_file', args: { file_path: 'src/index.ts', content: 'console.log("hello")' }, tool_call_id: 'tc_3' }), + JSON.stringify({ type: 'tool_result', tool_name: 'write_file', result: 'ok', error: null, tool_call_id: 'tc_3' }), + JSON.stringify({ type: 'assistant', content: 'Done! The app is ready.', message_id: 'msg_3' }), + ].join('\n'); + + const result = parseTranscript(transcript, 'mistral-vibe'); + + expect(result.parseSuccess).toBe(true); + expect(result.summary.totalTurns).toBe(2); + expect(result.summary.toolCalls.file_read).toBe(1); + expect(result.summary.toolCalls.shell).toBe(1); + expect(result.summary.toolCalls.file_write).toBe(1); + expect(result.summary.totalToolCalls).toBe(3); + expect(result.summary.filesRead).toContain('package.json'); + expect(result.summary.filesModified).toContain('src/index.ts'); + expect(result.summary.shellCommands).toHaveLength(1); + expect(result.summary.shellCommands[0].command).toBe('mkdir src'); + }); + + it('handles LLMMessage-style events from --output json format', () => { + const transcript = JSON.stringify({ + role: 'assistant', + content: 'Let me help.', + tool_calls: [ + { + function: { name: 'read_file', arguments: '{"file_path":"src/app.ts"}' }, + id: 'tc_json_1', + }, + ], + }); + const { events } = parseMistralVibeTranscript(transcript); + + expect(events).toHaveLength(2); + expect(events[0].type).toBe('message'); + expect(events[0].role).toBe('assistant'); + expect(events[1].type).toBe('tool_call'); + expect(events[1].tool?.name).toBe('file_read'); + }); + }); + describe('Summary generation', () => { it('counts tool calls correctly', () => { const transcript = [ diff --git a/packages/agent-eval/src/lib/o11y/parsers/index.ts b/packages/agent-eval/src/lib/o11y/parsers/index.ts index a44daab..23ee65a 100644 --- a/packages/agent-eval/src/lib/o11y/parsers/index.ts +++ b/packages/agent-eval/src/lib/o11y/parsers/index.ts @@ -16,6 +16,7 @@ import { parseCodexTranscript } from './codex.js'; import { parseOpenCodeTranscript } from './opencode.js'; import { parseGeminiTranscript } from './gemini.js'; import { parseCursorTranscript } from './cursor.js'; +import { parseMistralVibeTranscript } from './mistral-vibe.js'; /** * Supported agent types for parsing. @@ -27,7 +28,8 @@ export type ParseableAgent = | 'codex' | 'vercel-ai-gateway/opencode' | 'gemini' - | 'cursor'; + | 'cursor' + | 'mistral-vibe'; /** * Parser registry mapping agent key patterns to their parsers. @@ -38,6 +40,7 @@ const AGENT_PARSERS = { 'opencode': parseOpenCodeTranscript, 'gemini': parseGeminiTranscript, 'cursor': parseCursorTranscript, + 'mistral-vibe': parseMistralVibeTranscript, } as const; /** diff --git a/packages/agent-eval/src/lib/o11y/parsers/mistral-vibe.ts b/packages/agent-eval/src/lib/o11y/parsers/mistral-vibe.ts new file mode 100644 index 0000000..1de76e0 --- /dev/null +++ b/packages/agent-eval/src/lib/o11y/parsers/mistral-vibe.ts @@ -0,0 +1,292 @@ +/** + * Parser for Mistral Vibe CLI transcript format. + * When run with --output streaming, Vibe outputs newline-delimited JSON per event. + * + * Event types (from Vibe's Pydantic event system): + * - UserMessageEvent: { type: "user_message", content, message_id } + * - AssistantEvent: { type: "assistant", content, message_id } + * - ReasoningEvent: { type: "reasoning", content, message_id } + * - ToolCallEvent: { type: "tool_call", tool_name, args, tool_call_id } + * - ToolResultEvent: { type: "tool_result", tool_name, result, error, duration, tool_call_id } + * - ToolStreamEvent: { type: "tool_stream", tool_name, message, tool_call_id } + */ + +import type { TranscriptEvent, ToolName } from '../types.js'; + +/** + * Map Vibe tool names to canonical names. + */ +function normalizeToolName(name: string): ToolName { + const toolMap: Record = { + // Vibe built-in tools + read_file: 'file_read', + write_file: 'file_write', + search_replace: 'file_edit', + bash: 'shell', + grep: 'grep', + todo: 'agent_task', + task: 'agent_task', + + // Common aliases + read: 'file_read', + write: 'file_write', + edit: 'file_edit', + shell: 'shell', + exec: 'shell', + glob: 'glob', + find: 'glob', + ls: 'list_dir', + list_dir: 'list_dir', + web_fetch: 'web_fetch', + fetch: 'web_fetch', + web_search: 'web_search', + search: 'web_search', + }; + + return toolMap[name.toLowerCase()] || 'unknown'; +} + +/** + * Extract file path from tool arguments. + */ +function extractFilePath(args: Record): string | undefined { + return (args.path || args.file_path || args.filePath || args.file || args.filename) as + | string + | undefined; +} + +/** + * Extract command from tool arguments. + */ +function extractCommand(args: Record): string | undefined { + if (typeof args.command === 'string') return args.command; + if (typeof args.cmd === 'string') return args.cmd; + return undefined; +} + +/** + * Extract URL from tool arguments. + */ +function extractUrl(args: Record): string | undefined { + return (args.url || args.uri || args.href) as string | undefined; +} + +/** + * Parse a single JSONL line from a Vibe streaming transcript. + */ +function parseVibeLine(line: string): TranscriptEvent[] { + const events: TranscriptEvent[] = []; + + try { + const data = JSON.parse(line); + const eventType = data.type; + + switch (eventType) { + case 'user_message': { + if (data.content) { + events.push({ + timestamp: data.timestamp, + type: 'message', + role: 'user', + content: data.content, + raw: data, + }); + } + break; + } + + case 'assistant': { + if (data.content) { + events.push({ + timestamp: data.timestamp, + type: 'message', + role: 'assistant', + content: data.content, + raw: data, + }); + } + break; + } + + case 'reasoning': { + if (data.content) { + events.push({ + timestamp: data.timestamp, + type: 'thinking', + content: data.content, + raw: data, + }); + } + break; + } + + case 'tool_call': { + const name = data.tool_name; + const args = (typeof data.args === 'object' && data.args !== null) + ? data.args + : {}; + + if (name) { + events.push({ + timestamp: data.timestamp, + type: 'tool_call', + tool: { + name: normalizeToolName(name), + originalName: name, + args, + }, + raw: data, + }); + } + break; + } + + case 'tool_result': { + const name = data.tool_name || 'unknown'; + const hasError = !!data.error; + + events.push({ + timestamp: data.timestamp, + type: 'tool_result', + tool: { + name: normalizeToolName(name), + originalName: name, + result: data.result ?? data.output, + success: !hasError && !data.skipped, + }, + raw: data, + }); + break; + } + + case 'tool_stream': { + // Streaming tool output — skip for summary purposes + break; + } + + case 'error': { + events.push({ + timestamp: data.timestamp, + type: 'error', + content: data.error?.message || data.message || data.content, + raw: data, + }); + break; + } + + default: { + // Handle LLMMessage-style events (from --output json format) + if (data.role === 'assistant' && data.content) { + events.push({ + timestamp: data.timestamp, + type: 'message', + role: 'assistant', + content: data.content, + raw: data, + }); + + // Extract tool calls embedded in the message + if (Array.isArray(data.tool_calls)) { + for (const call of data.tool_calls) { + const fnName = call.function?.name || call.name; + const fnArgs = call.function?.arguments + ? typeof call.function.arguments === 'string' + ? JSON.parse(call.function.arguments) + : call.function.arguments + : call.arguments || call.args || {}; + + if (fnName) { + events.push({ + timestamp: data.timestamp, + type: 'tool_call', + tool: { + name: normalizeToolName(fnName), + originalName: fnName, + args: fnArgs, + }, + raw: call, + }); + } + } + } + } else if (data.role === 'user' && data.content) { + events.push({ + timestamp: data.timestamp, + type: 'message', + role: 'user', + content: data.content, + raw: data, + }); + } else if (data.role === 'tool') { + events.push({ + timestamp: data.timestamp, + type: 'tool_result', + tool: { + name: data.name ? normalizeToolName(data.name) : 'unknown', + originalName: data.name || 'unknown', + result: data.content, + success: !data.error, + }, + raw: data, + }); + } + } + } + } catch { + // Skip unparseable lines + } + + return events; +} + +/** + * Parse Mistral Vibe JSONL transcript into normalized events. + */ +export function parseMistralVibeTranscript(raw: string): { + events: TranscriptEvent[]; + errors: string[]; +} { + const events: TranscriptEvent[] = []; + const errors: string[] = []; + + const lines = raw.split('\n').filter((line) => line.trim()); + + for (const line of lines) { + try { + const lineEvents = parseVibeLine(line); + events.push(...lineEvents); + } catch (e) { + errors.push(`Failed to parse line: ${e instanceof Error ? e.message : String(e)}`); + } + } + + // Post-process: extract metadata into tool args + for (const event of events) { + if (event.type === 'tool_call' && event.tool) { + const args = event.tool.args || {}; + + if (['file_read', 'file_write', 'file_edit'].includes(event.tool.name)) { + const path = extractFilePath(args); + if (path) { + event.tool.args = { ...args, _extractedPath: path }; + } + } + + if (event.tool.name === 'web_fetch') { + const url = extractUrl(args); + if (url) { + event.tool.args = { ...args, _extractedUrl: url }; + } + } + + if (event.tool.name === 'shell') { + const command = extractCommand(args); + if (command) { + event.tool.args = { ...args, _extractedCommand: command }; + } + } + } + } + + return { events, errors }; +} diff --git a/packages/agent-eval/src/lib/types.ts b/packages/agent-eval/src/lib/types.ts index 5c982d6..b55035c 100644 --- a/packages/agent-eval/src/lib/types.ts +++ b/packages/agent-eval/src/lib/types.ts @@ -12,7 +12,8 @@ export type AgentType = | 'codex' | 'vercel-ai-gateway/opencode' | 'gemini' - | 'cursor'; + | 'cursor' + | 'mistral-vibe'; /** * Model identifier - any string accepted.