From 5f80a153236be6406a5862f4cc15ce925f9b15da Mon Sep 17 00:00:00 2001 From: CorrectRoadH <29306285+CorrectRoadH@users.noreply.github.com> Date: Sat, 7 Mar 2026 14:09:00 +0800 Subject: [PATCH 1/2] feat: add bub agent support --- packages/agent-eval/src/integration.test.ts | 149 ++++++++++ packages/agent-eval/src/lib/agents/bub.ts | 269 ++++++++++++++++++ packages/agent-eval/src/lib/agents/index.ts | 2 + packages/agent-eval/src/lib/agents/shared.ts | 7 + packages/agent-eval/src/lib/config.ts | 1 + packages/agent-eval/src/lib/o11y/index.ts | 1 + packages/agent-eval/src/lib/o11y/o11y.test.ts | 184 +++++++++++- .../agent-eval/src/lib/o11y/parsers/bub.ts | 253 ++++++++++++++++ .../agent-eval/src/lib/o11y/parsers/index.ts | 5 +- packages/agent-eval/src/lib/types.ts | 3 +- 10 files changed, 871 insertions(+), 3 deletions(-) create mode 100644 packages/agent-eval/src/lib/agents/bub.ts create mode 100644 packages/agent-eval/src/lib/o11y/parsers/bub.ts diff --git a/packages/agent-eval/src/integration.test.ts b/packages/agent-eval/src/integration.test.ts index b9cf607..deb94e7 100644 --- a/packages/agent-eval/src/integration.test.ts +++ b/packages/agent-eval/src/integration.test.ts @@ -44,6 +44,7 @@ const hasAnthropicCredentials = !!process.env.ANTHROPIC_API_KEY && hasSandbox; const hasOpenAiCredentials = !!process.env.OPENAI_API_KEY && hasSandbox; const hasGeminiCredentials = !!process.env.GEMINI_API_KEY && hasSandbox; const hasCursorCredentials = !!process.env.CURSOR_API_KEY && hasSandbox; +const hasBubCredentials = !!process.env.BUB_API_KEY && hasSandbox; // OpenCode credentials (only supports AI Gateway) const hasOpenCodeCredentials = hasAiGatewayCredentials; @@ -902,6 +903,154 @@ test('contains greeting', () => { }, 300000); // 5 minute timeout }); + describe.skipIf(!hasBubCredentials)('Bub (Direct API) sandbox execution', () => { + it('can run a simple eval with Bub', async () => { + // Create a simple test fixture + const fixtureDir = join(TEST_DIR, 'simple-eval-bub'); + mkdirSync(join(fixtureDir, 'src'), { recursive: true }); + + writeFileSync( + join(fixtureDir, 'PROMPT.md'), + 'Add a function called greet that returns "Hello from Bub!"' + ); + writeFileSync( + join(fixtureDir, 'EVAL.ts'), + ` +import { test, expect } from 'vitest'; +import { readFileSync } from 'fs'; + +test('greet exists', () => { + const content = readFileSync('src/index.ts', 'utf-8'); + expect(content).toContain('greet'); +}); +` + ); + writeFileSync( + join(fixtureDir, 'package.json'), + JSON.stringify({ + name: 'simple-eval-bub', + type: 'module', + scripts: { build: 'tsc' }, + devDependencies: { typescript: '^5.0.0', vitest: '^2.1.0' }, + }) + ); + writeFileSync( + join(fixtureDir, 'tsconfig.json'), + JSON.stringify({ + compilerOptions: { + target: 'ES2020', + module: 'ESNext', + moduleResolution: 'bundler', + outDir: 'dist', + }, + include: ['src'], + }) + ); + + const fixture = loadFixture(TEST_DIR, 'simple-eval-bub'); + + const result = await runSingleEval(fixture, { + agent: 'bub', + model: 'openrouter:qwen/qwen3-coder-next', + timeout: 180, + apiKey: process.env.BUB_API_KEY!, + scripts: ['build'], + }); + + // Verify result structure + expect(result.result.duration).toBeGreaterThan(0); + expect(result.result.status).toBeDefined(); + // Agent must actually succeed - not just return a result + if (result.result.status === 'failed') { + console.error('Agent failed with error:', result.result.error); + } + expect(result.result.status).toBe('passed'); + + // Verify output content exists (if available) + if (result.outputContent) { + expect(typeof result.outputContent).toBe('object'); + } + }, 300000); // 5 minute timeout + + it('verifies Bub result output structure matches expected format', async () => { + // Create a simple test fixture + const fixtureDir = join(TEST_DIR, 'result-structure-bub'); + mkdirSync(join(fixtureDir, 'src'), { recursive: true }); + + writeFileSync( + join(fixtureDir, 'PROMPT.md'), + 'Create a simple hello.ts file that exports a greeting constant.' + ); + writeFileSync( + join(fixtureDir, 'EVAL.ts'), + ` +import { test, expect } from 'vitest'; +import { readFileSync, existsSync } from 'fs'; + +test('hello.ts exists', () => { + expect(existsSync('src/hello.ts')).toBe(true); +}); + +test('contains greeting', () => { + const content = readFileSync('src/hello.ts', 'utf-8'); + expect(content).toContain('greeting'); +}); +` + ); + writeFileSync( + join(fixtureDir, 'package.json'), + JSON.stringify({ + name: 'result-structure-bub', + type: 'module', + scripts: { build: 'tsc' }, + devDependencies: { typescript: '^5.0.0', vitest: '^2.1.0' }, + }) + ); + writeFileSync( + join(fixtureDir, 'tsconfig.json'), + JSON.stringify({ + compilerOptions: { + target: 'ES2020', + module: 'ESNext', + moduleResolution: 'bundler', + outDir: 'dist', + }, + include: ['src'], + }) + ); + + const fixture = loadFixture(TEST_DIR, 'result-structure-bub'); + + const result = await runSingleEval(fixture, { + agent: 'bub', + model: 'openrouter:qwen/qwen3-coder-next', + timeout: 180, + apiKey: process.env.BUB_API_KEY!, + scripts: ['build'], + }); + + // Verify EvalRunData structure + expect(result).toHaveProperty('result'); + expect(result.result).toHaveProperty('status'); + expect(result.result).toHaveProperty('duration'); + + // Verify optional properties have correct types when present + if (result.result.error) { + expect(typeof result.result.error).toBe('string'); + } + + // Verify output content structure if present + if (result.outputContent) { + if (result.outputContent.eval) { + expect(typeof result.outputContent.eval).toBe('string'); + } + if (result.outputContent.scripts?.build) { + expect(typeof result.outputContent.scripts.build).toBe('string'); + } + } + }, 300000); // 5 minute timeout + }); + describe.skipIf(!hasOpenCodeCredentials)('OpenCode (Vercel AI Gateway) sandbox execution', () => { it('can run a simple eval with OpenCode', async () => { // Create a simple test fixture diff --git a/packages/agent-eval/src/lib/agents/bub.ts b/packages/agent-eval/src/lib/agents/bub.ts new file mode 100644 index 0000000..487d7ba --- /dev/null +++ b/packages/agent-eval/src/lib/agents/bub.ts @@ -0,0 +1,269 @@ +/** + * Bub agent implementation. + * Uses direct Bub CLI access. + */ + +import type { Agent, AgentRunOptions, AgentRunResult } from './types.js'; +import type { ModelTier } from '../types.js'; +import { + createSandbox, + collectLocalFiles, + splitTestFiles, + verifyNoTestFiles, + type SandboxManager, +} from '../sandbox.js'; +import type { DockerSandboxManager } from '../docker-sandbox.js'; +import { + runValidation, + captureGeneratedFiles, + createVitestConfig, + BUB_DIRECT, + initGitAndCommit, + injectTranscriptContext, +} from './shared.js'; + +/** Union type for sandbox implementations */ +type AnySandbox = SandboxManager | DockerSandboxManager; + +/** + * Shell-escape a string for single-quoted bash usage. + */ +function shellQuote(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'`; +} + +/** + * Extract transcript from Bub tapes. + * Bub persists session tapes under $BUB_HOME/tapes/.jsonl. + */ +async function captureTranscript(sandbox: AnySandbox, bubHome: string): Promise { + try { + const findResult = await sandbox.runShell( + `ls -t ${shellQuote(`${bubHome}/tapes`)}/*.jsonl 2>/dev/null | head -1` + ); + + if (findResult.exitCode !== 0 || !findResult.stdout.trim()) { + return undefined; + } + + const transcriptPath = findResult.stdout.trim(); + const content = await sandbox.readFile(transcriptPath); + return content || undefined; + } catch { + // Transcript capture is best-effort + return undefined; + } +} + +/** + * Create Bub agent. + */ +export function createBubAgent(): Agent { + return { + name: 'bub', + displayName: 'Bub', + + getApiKeyEnvVar(): string { + return BUB_DIRECT.apiKeyEnvVar; + }, + + getDefaultModel(): ModelTier { + return 'openrouter:qwen/qwen3-coder-next'; + }, + + async run(fixturePath: string, options: AgentRunOptions): Promise { + const startTime = Date.now(); + let sandbox: AnySandbox | null = null; + let agentOutput = ''; + let transcript: string | undefined; + let aborted = false; + let sandboxStopped = false; + + // Handle abort signal + const abortHandler = () => { + aborted = true; + if (sandbox && !sandboxStopped) { + sandboxStopped = true; + sandbox.stop().catch(() => {}); + } + }; + + if (options.signal) { + if (options.signal.aborted) { + return { + success: false, + output: '', + error: 'Aborted before start', + duration: 0, + }; + } + options.signal.addEventListener('abort', abortHandler); + } + + try { + // Collect files from fixture + const allFiles = await collectLocalFiles(fixturePath); + const { workspaceFiles, testFiles } = splitTestFiles(allFiles); + + // Check for abort before expensive operations + if (aborted) { + return { + success: false, + output: '', + error: 'Aborted', + duration: Date.now() - startTime, + }; + } + + // Create sandbox + sandbox = await createSandbox({ + timeout: options.timeout, + runtime: 'node24', + backend: options.sandbox, + }); + + // Check for abort after sandbox creation + if (aborted) { + return { + success: false, + output: '', + error: 'Aborted', + duration: Date.now() - startTime, + sandboxId: sandbox.sandboxId, + }; + } + + // Upload workspace files (excluding tests) + await sandbox.uploadFiles(workspaceFiles); + + await initGitAndCommit(sandbox); + + // Run setup function if provided + if (options.setup) { + await options.setup(sandbox); + } + + // Install fixture dependencies so Bub operates in a realistic workspace. + let installResult = await sandbox.runCommand('npm', ['install']); + if (installResult.exitCode !== 0) { + installResult = await sandbox.runCommand('npm', ['install']); + } + if (installResult.exitCode !== 0) { + const output = (installResult.stdout + installResult.stderr).trim().split('\n').slice(-10).join('\n'); + throw new Error(`npm install failed (exit code ${installResult.exitCode}):\n${output}`); + } + + // Install uv and Bub CLI as a tool, similar to a standalone binary install. + const bubInstall = await sandbox.runShell( + [ + 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"', + 'curl -LsSf https://astral.sh/uv/install.sh | sh', + 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"', + 'uv tool install --from git+https://github.com/bubbuild/bub.git bub', + ].join(' && ') + ); + if (bubInstall.exitCode !== 0) { + throw new Error(`Bub install failed: ${bubInstall.stderr}`); + } + + // Verify no test files in sandbox + await verifyNoTestFiles(sandbox); + + const bubbleHome = `${sandbox.getWorkingDirectory()}/.bub`; + const env: Record = { + [BUB_DIRECT.apiKeyEnvVar]: options.apiKey, + BUB_HOME: bubbleHome, + BUB_MODEL: options.model, + BUB_RUNTIME_ENABLED: '1', + }; + + if (options.model.startsWith('openrouter:')) { + env.OPENROUTER_API_KEY = options.apiKey; + } + + // Run Bub with the prompt + const bubResult = await sandbox.runShell( + [ + 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"', + `bub --workspace ${shellQuote(sandbox.getWorkingDirectory())} run ${shellQuote(options.prompt)}`, + ].join(' && '), + env + ); + + agentOutput = bubResult.stdout + bubResult.stderr; + + // Capture transcript after Bub runs + transcript = await captureTranscript(sandbox, bubbleHome); + + if (bubResult.exitCode !== 0) { + // Extract meaningful error from output + const errorLines = agentOutput.trim().split('\n').slice(-5).join('\n'); + return { + success: false, + output: agentOutput, + transcript, + error: errorLines || `Bub exited with code ${bubResult.exitCode}`, + duration: Date.now() - startTime, + sandboxId: sandbox.sandboxId, + }; + } + + // Upload test files for validation + await sandbox.uploadFiles(testFiles); + + // Create vitest config for EVAL.ts/tsx + await createVitestConfig(sandbox); + + // Inject transcript context so EVAL.ts tests can assert on agent behavior + await injectTranscriptContext(sandbox, transcript, 'bub', options.model); + + // Run validation scripts + const validationResults = await runValidation(sandbox, options.scripts ?? []); + + // Capture generated files + const { generatedFiles, deletedFiles } = await captureGeneratedFiles(sandbox); + + return { + success: validationResults.allPassed, + output: agentOutput, + transcript, + duration: Date.now() - startTime, + testResult: validationResults.test, + scriptsResults: validationResults.scripts, + sandboxId: sandbox.sandboxId, + generatedFiles, + deletedFiles, + }; + } catch (error) { + // Check if this was an abort + if (aborted) { + return { + success: false, + output: agentOutput, + transcript, + error: 'Aborted', + duration: Date.now() - startTime, + sandboxId: sandbox?.sandboxId, + }; + } + return { + success: false, + output: agentOutput, + transcript, + error: error instanceof Error ? error.message : String(error), + duration: Date.now() - startTime, + sandboxId: sandbox?.sandboxId, + }; + } finally { + // Clean up abort listener + if (options.signal) { + options.signal.removeEventListener('abort', abortHandler); + } + if (sandbox && !sandboxStopped) { + sandboxStopped = true; + await sandbox.stop(); + } + } + }, + }; +} diff --git a/packages/agent-eval/src/lib/agents/index.ts b/packages/agent-eval/src/lib/agents/index.ts index 7312064..ac809d7 100644 --- a/packages/agent-eval/src/lib/agents/index.ts +++ b/packages/agent-eval/src/lib/agents/index.ts @@ -8,6 +8,7 @@ import { createCodexAgent } from './codex.js'; import { createOpenCodeAgent } from './opencode.js'; import { createGeminiAgent } from './gemini.js'; import { createCursorAgent } from './cursor.js'; +import { createBubAgent } from './bub.js'; // Register all agent variants (Vercel AI Gateway + Direct API) registerAgent(createClaudeCodeAgent({ useVercelAiGateway: true })); // vercel-ai-gateway/claude-code @@ -17,6 +18,7 @@ registerAgent(createCodexAgent({ useVercelAiGateway: false })); // codex registerAgent(createOpenCodeAgent()); // vercel-ai-gateway/opencode registerAgent(createGeminiAgent()); // gemini registerAgent(createCursorAgent()); // cursor +registerAgent(createBubAgent()); // bub // Re-export registry functions export { registerAgent, getAgent, listAgents, hasAgent }; diff --git a/packages/agent-eval/src/lib/agents/shared.ts b/packages/agent-eval/src/lib/agents/shared.ts index 414004a..15ee98b 100644 --- a/packages/agent-eval/src/lib/agents/shared.ts +++ b/packages/agent-eval/src/lib/agents/shared.ts @@ -243,3 +243,10 @@ export const GEMINI_DIRECT = { export const CURSOR_DIRECT = { apiKeyEnvVar: 'CURSOR_API_KEY', } as const; + +/** + * Direct API configuration for Bub. + */ +export const BUB_DIRECT = { + apiKeyEnvVar: 'BUB_API_KEY', +} as const; diff --git a/packages/agent-eval/src/lib/config.ts b/packages/agent-eval/src/lib/config.ts index a6869e9..49718e8 100644 --- a/packages/agent-eval/src/lib/config.ts +++ b/packages/agent-eval/src/lib/config.ts @@ -37,6 +37,7 @@ const experimentConfigSchema = z.object({ 'vercel-ai-gateway/opencode', 'gemini', 'cursor', + 'bub', ]), model: z.union([z.string(), z.array(z.string())]).optional(), evals: z diff --git a/packages/agent-eval/src/lib/o11y/index.ts b/packages/agent-eval/src/lib/o11y/index.ts index f8f65f4..1e8abfe 100644 --- a/packages/agent-eval/src/lib/o11y/index.ts +++ b/packages/agent-eval/src/lib/o11y/index.ts @@ -24,3 +24,4 @@ export { parseCodexTranscript } from './parsers/codex.js'; export { parseOpenCodeTranscript } from './parsers/opencode.js'; export { parseGeminiTranscript } from './parsers/gemini.js'; export { parseCursorTranscript } from './parsers/cursor.js'; +export { parseBubTranscript } from './parsers/bub.js'; diff --git a/packages/agent-eval/src/lib/o11y/o11y.test.ts b/packages/agent-eval/src/lib/o11y/o11y.test.ts index af9128f..cef6e4d 100644 --- a/packages/agent-eval/src/lib/o11y/o11y.test.ts +++ b/packages/agent-eval/src/lib/o11y/o11y.test.ts @@ -10,6 +10,7 @@ import { parseCodexTranscript } from './parsers/codex.js'; import { parseOpenCodeTranscript } from './parsers/opencode.js'; import { parseGeminiTranscript } from './parsers/gemini.js'; import { parseCursorTranscript } from './parsers/cursor.js'; +import { parseBubTranscript } from './parsers/bub.js'; describe('o11y', () => { describe('parseTranscript', () => { @@ -38,6 +39,9 @@ describe('o11y', () => { const cursorResult = parseTranscript(claudeTranscript, 'cursor'); expect(cursorResult.agent).toBe('cursor'); + + const bubResult = parseTranscript(claudeTranscript, 'bub'); + expect(bubResult.agent).toBe('bub'); }); it('returns parseSuccess: false for unsupported agents', () => { @@ -47,7 +51,7 @@ describe('o11y', () => { expect(result.parseSuccess).toBe(false); expect(result.parseErrors).toContain( - 'No parser available for agent: unsupported-agent. Supported agents: claude-code, codex, opencode, gemini, cursor' + 'No parser available for agent: unsupported-agent. Supported agents: claude-code, codex, opencode, gemini, cursor, bub' ); expect(result.events).toEqual([]); expect(result.summary.totalToolCalls).toBe(0); @@ -737,6 +741,184 @@ describe('o11y', () => { }); }); + describe('parseBubTranscript', () => { + it('parses basic message events', () => { + const transcript = JSON.stringify({ + type: 'message', + role: 'assistant', + content: 'Hello from Bub', + timestamp: '2024-01-01T00:00:00Z', + }); + + const { events } = parseBubTranscript(transcript); + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + type: 'message', + role: 'assistant', + content: 'Hello from Bub', + timestamp: '2024-01-01T00:00:00Z', + }); + }); + + it('parses tool call events', () => { + const transcript = JSON.stringify({ + type: 'tool_call', + tool_name: 'fs.read', + args: { path: 'test.ts' }, + timestamp: '2024-01-01T00:00:00Z', + }); + + const { events } = parseBubTranscript(transcript); + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + type: 'tool_call', + tool: { + name: 'file_read', + originalName: 'fs.read', + args: { + path: 'test.ts', + _extractedPath: 'test.ts', + }, + }, + timestamp: '2024-01-01T00:00:00Z', + }); + }); + + it('parses tool result events', () => { + const transcript = JSON.stringify({ + type: 'tool_result', + success: true, + result: 'File content here', + timestamp: '2024-01-01T00:00:00Z', + }); + + const { events } = parseBubTranscript(transcript); + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + type: 'tool_result', + tool: { + name: 'unknown', + originalName: 'unknown', + success: true, + result: 'File content here', + }, + timestamp: '2024-01-01T00:00:00Z', + }); + }); + + it('parses error events', () => { + const transcript = JSON.stringify({ + type: 'error', + message: 'Something went wrong', + timestamp: '2024-01-01T00:00:00Z', + }); + + const { events } = parseBubTranscript(transcript); + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + type: 'error', + content: 'Something went wrong', + timestamp: '2024-01-01T00:00:00Z', + }); + }); + + it('maps Bub tool names to standard names', () => { + const transcript = [ + JSON.stringify({ type: 'tool_call', tool_name: 'fs.write', args: { path: 'a.ts' } }), + JSON.stringify({ type: 'tool_call', tool_name: 'shell', args: { command: 'ls' } }), + JSON.stringify({ type: 'tool_call', tool_name: 'web.fetch', args: { url: 'http://example.com' } }), + JSON.stringify({ type: 'tool_call', tool_name: 'unknown_tool', args: {} }), + ].join('\n'); + + const { events } = parseBubTranscript(transcript); + + expect(events[0].tool?.name).toBe('file_write'); + expect(events[1].tool?.name).toBe('shell'); + expect(events[2].tool?.name).toBe('web_fetch'); + expect(events[3].tool?.name).toBe('unknown'); + }); + + it('handles multiline transcripts', () => { + const transcript = [ + JSON.stringify({ type: 'message', role: 'user', content: 'Do something' }), + JSON.stringify({ type: 'message', role: 'assistant', content: 'OK' }), + JSON.stringify({ type: 'tool_call', tool_name: 'fs.read', args: { path: 'a.ts' } }), + JSON.stringify({ type: 'tool_result', success: true, result: 'content' }), + ].join('\n'); + + const result = parseTranscript(transcript, 'bub'); + + expect(result.parseSuccess).toBe(true); + expect(result.summary.totalTurns).toBe(1); + expect(result.summary.toolCalls.file_read).toBe(1); + expect(result.summary.totalToolCalls).toBe(1); + expect(result.summary.filesRead).toContain('a.ts'); + }); + + it('parses Bub tape tool calls', () => { + const transcript = [ + JSON.stringify({ + id: 1, + kind: 'tool_call', + payload: { + calls: [ + { + function: { + name: 'bash', + arguments: '{"cmd":"npm test"}', + }, + }, + ], + }, + timestamp: 1772758779.1400378, + }), + JSON.stringify({ + id: 2, + kind: 'tool_result', + payload: { + results: [{ exitCode: 0, stdout: 'ok' }], + }, + timestamp: 1772758779.140388, + }), + ].join('\n'); + + const result = parseTranscript(transcript, 'bub'); + + expect(result.parseSuccess).toBe(true); + expect(result.summary.toolCalls.shell).toBe(1); + expect(result.summary.shellCommands[0]?.command).toBe('npm test'); + expect(result.summary.shellCommands[0]?.exitCode).toBe(0); + }); + + it('parses Bub tape message entries', () => { + const transcript = [ + JSON.stringify({ + id: 1, + kind: 'message', + payload: { role: 'user', content: 'Implement feature' }, + timestamp: 1772758779.1400099, + }), + JSON.stringify({ + id: 2, + kind: 'message', + payload: { role: 'assistant', content: 'Done' }, + timestamp: 1772758779.140484, + }), + ].join('\n'); + + const { events } = parseBubTranscript(transcript); + + expect(events).toHaveLength(2); + expect(events[0].type).toBe('message'); + expect(events[0].role).toBe('user'); + expect(events[1].role).toBe('assistant'); + }); + }); + describe('Summary generation', () => { it('counts tool calls correctly', () => { const transcript = [ diff --git a/packages/agent-eval/src/lib/o11y/parsers/bub.ts b/packages/agent-eval/src/lib/o11y/parsers/bub.ts new file mode 100644 index 0000000..ce88b74 --- /dev/null +++ b/packages/agent-eval/src/lib/o11y/parsers/bub.ts @@ -0,0 +1,253 @@ +/** + * Bub transcript parser. + * Parses Bub tape JSONL files from $BUB_HOME/tapes/.jsonl. + */ + +import type { TranscriptEvent, ToolName } from '../types.js'; + +function toISO(value: unknown): string | undefined { + if (typeof value === 'string') { + return value; + } + if (typeof value === 'number') { + return new Date(value * 1000).toISOString(); + } + return undefined; +} + +function normalizeToolName(name: string): ToolName { + const toolMap: Record = { + 'fs.read': 'file_read', + fs_read: 'file_read', + 'fs.write': 'file_write', + fs_write: 'file_write', + 'fs.edit': 'file_edit', + fs_edit: 'file_edit', + bash: 'shell', + shell: 'shell', + 'web.fetch': 'web_fetch', + web_fetch: 'web_fetch', + 'web.search': 'web_search', + web_search: 'web_search', + glob: 'glob', + grep: 'grep', + ls: 'list_dir', + list_dir: 'list_dir', + task: 'agent_task', + update_todos: 'agent_task', + }; + + return toolMap[name] || 'unknown'; +} + +function parseToolArgs(value: unknown): Record { + if (typeof value === 'string') { + try { + const parsed = JSON.parse(value); + return typeof parsed === 'object' && parsed !== null ? parsed as Record : {}; + } catch { + return {}; + } + } + + return typeof value === 'object' && value !== null ? value as Record : {}; +} + +function enrichArgs(toolName: ToolName, args: Record): Record { + const enriched = { ...args }; + + if (toolName === 'file_read' || toolName === 'file_write' || toolName === 'file_edit') { + const path = (args.path || args.file || args.filePath) as string | undefined; + if (path) { + enriched._extractedPath = path; + } + } + + if (toolName === 'web_fetch' || toolName === 'web_search') { + const url = (args.url || args.uri || args.query) as string | undefined; + if (url) { + enriched._extractedUrl = url; + } + } + + if (toolName === 'shell') { + const command = (args.command || args.cmd) as string | undefined; + if (command) { + enriched._extractedCommand = command; + } + } + + return enriched; +} + +function parseLegacyEntry(entry: Record): TranscriptEvent[] { + const timestamp = toISO(entry.timestamp); + + if (entry.type === 'message') { + return [{ + type: 'message', + role: (entry.role as 'user' | 'assistant' | 'system') || 'assistant', + content: (entry.content as string) || '', + timestamp, + raw: entry, + }]; + } + + if (entry.type === 'tool_call') { + const originalName = (entry.tool_name as string) || 'unknown'; + const toolName = normalizeToolName(originalName); + return [{ + type: 'tool_call', + timestamp, + tool: { + name: toolName, + originalName, + args: enrichArgs(toolName, parseToolArgs(entry.args)), + }, + raw: entry, + }]; + } + + if (entry.type === 'tool_result') { + return [{ + type: 'tool_result', + timestamp, + tool: { + name: 'unknown', + originalName: 'unknown', + success: entry.success !== false, + result: entry.result, + }, + raw: entry, + }]; + } + + if (entry.type === 'error') { + return [{ + type: 'error', + timestamp, + content: (entry.message || entry.error || 'Unknown error') as string, + raw: entry, + }]; + } + + return []; +} + +function parseTapeEntry(entry: Record): TranscriptEvent[] { + const events: TranscriptEvent[] = []; + const timestamp = toISO(entry.timestamp); + const kind = entry.kind; + const payload = (typeof entry.payload === 'object' && entry.payload !== null) + ? entry.payload as Record + : {}; + + if (kind === 'message') { + const role = payload.role; + const content = payload.content; + if (typeof role === 'string' && typeof content === 'string') { + events.push({ + type: 'message', + role: role as 'user' | 'assistant' | 'system', + content, + timestamp, + raw: entry, + }); + } + return events; + } + + if (kind === 'tool_call') { + const calls = Array.isArray(payload.calls) ? payload.calls : []; + for (const call of calls) { + const fn = typeof call === 'object' && call !== null + ? (call as { function?: { name?: unknown; arguments?: unknown } }).function + : undefined; + const originalName = typeof fn?.name === 'string' ? fn.name : 'unknown'; + const toolName = normalizeToolName(originalName); + const args = enrichArgs(toolName, parseToolArgs(fn?.arguments)); + events.push({ + type: 'tool_call', + timestamp, + tool: { + name: toolName, + originalName, + args, + }, + raw: call, + }); + } + return events; + } + + if (kind === 'tool_result') { + const results = Array.isArray(payload.results) ? payload.results : [payload.results]; + for (const result of results) { + events.push({ + type: 'tool_result', + timestamp, + tool: { + name: 'unknown', + originalName: 'unknown', + success: !(typeof result === 'object' && result !== null && 'error' in result), + result, + }, + raw: result, + }); + } + return events; + } + + if (kind === 'event') { + const eventName = payload.name; + const data = (typeof payload.data === 'object' && payload.data !== null) + ? payload.data as Record + : {}; + const status = data.status; + const error = data.error; + + if (typeof status === 'string' && status !== 'ok') { + events.push({ + type: 'error', + timestamp, + content: typeof error === 'string' ? error : `${String(eventName)}: ${status}`, + raw: entry, + }); + } + return events; + } + + return events; +} + +/** + * Parse Bub tape JSONL into normalized events. + */ +export function parseBubTranscript(raw: string): { events: TranscriptEvent[]; errors: string[] } { + const events: TranscriptEvent[] = []; + const errors: string[] = []; + + if (!raw || !raw.trim()) { + return { events, errors }; + } + + const lines = raw.trim().split('\n'); + + for (const [lineIndex, line] of lines.entries()) { + const trimmed = line.trim(); + if (!trimmed) continue; + + try { + const entry = JSON.parse(trimmed) as Record; + if (typeof entry.type === 'string') { + events.push(...parseLegacyEntry(entry)); + } else { + events.push(...parseTapeEntry(entry)); + } + } catch (parseError) { + errors.push(`Line ${lineIndex + 1}: Invalid JSON - ${parseError}`); + } + } + + return { events, errors }; +} diff --git a/packages/agent-eval/src/lib/o11y/parsers/index.ts b/packages/agent-eval/src/lib/o11y/parsers/index.ts index a44daab..50a7344 100644 --- a/packages/agent-eval/src/lib/o11y/parsers/index.ts +++ b/packages/agent-eval/src/lib/o11y/parsers/index.ts @@ -16,6 +16,7 @@ import { parseCodexTranscript } from './codex.js'; import { parseOpenCodeTranscript } from './opencode.js'; import { parseGeminiTranscript } from './gemini.js'; import { parseCursorTranscript } from './cursor.js'; +import { parseBubTranscript } from './bub.js'; /** * Supported agent types for parsing. @@ -27,7 +28,8 @@ export type ParseableAgent = | 'codex' | 'vercel-ai-gateway/opencode' | 'gemini' - | 'cursor'; + | 'cursor' + | 'bub'; /** * Parser registry mapping agent key patterns to their parsers. @@ -38,6 +40,7 @@ const AGENT_PARSERS = { 'opencode': parseOpenCodeTranscript, 'gemini': parseGeminiTranscript, 'cursor': parseCursorTranscript, + 'bub': parseBubTranscript, } as const; /** diff --git a/packages/agent-eval/src/lib/types.ts b/packages/agent-eval/src/lib/types.ts index 5c982d6..f386e04 100644 --- a/packages/agent-eval/src/lib/types.ts +++ b/packages/agent-eval/src/lib/types.ts @@ -12,7 +12,8 @@ export type AgentType = | 'codex' | 'vercel-ai-gateway/opencode' | 'gemini' - | 'cursor'; + | 'cursor' + | 'bub'; /** * Model identifier - any string accepted. From 6a9cb23f58aeaf20878c8449c110c27f05f81994 Mon Sep 17 00:00:00 2001 From: CorrectRoadH <29306285+CorrectRoadH@users.noreply.github.com> Date: Sat, 7 Mar 2026 14:32:20 +0800 Subject: [PATCH 2/2] fix: support bub installs in docker sandbox --- packages/agent-eval/src/lib/agents/bub.ts | 9 ++++++++- packages/agent-eval/src/lib/docker-sandbox.ts | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/agent-eval/src/lib/agents/bub.ts b/packages/agent-eval/src/lib/agents/bub.ts index 487d7ba..c0acc97 100644 --- a/packages/agent-eval/src/lib/agents/bub.ts +++ b/packages/agent-eval/src/lib/agents/bub.ts @@ -157,7 +157,14 @@ export function createBubAgent(): Agent { const bubInstall = await sandbox.runShell( [ 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"', - 'curl -LsSf https://astral.sh/uv/install.sh | sh', + `node -e ${shellQuote( + "fetch('https://astral.sh/uv/install.sh').then(async (res) => {" + + "if (!res.ok) throw new Error(`Failed to fetch uv installer: ${res.status}`);" + + "const fs = await import('node:fs/promises');" + + "await fs.writeFile('/tmp/uv-install.sh', await res.text());" + + "}).catch((err) => { console.error(err); process.exit(1); });" + )}`, + 'sh /tmp/uv-install.sh', 'export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"', 'uv tool install --from git+https://github.com/bubbuild/bub.git bub', ].join(' && ') diff --git a/packages/agent-eval/src/lib/docker-sandbox.ts b/packages/agent-eval/src/lib/docker-sandbox.ts index d334128..5201538 100644 --- a/packages/agent-eval/src/lib/docker-sandbox.ts +++ b/packages/agent-eval/src/lib/docker-sandbox.ts @@ -105,8 +105,8 @@ export class DockerSandboxManager implements Sandbox { // Start the container await this.container.start(); - // Install CA certificates and git (slim images may not include them) - await this.runCommandAsRoot('bash', ['-c', 'apt-get update -qq && apt-get install -y -qq ca-certificates git > /dev/null 2>&1']); + // Install common bootstrap tools (slim images may not include them) + await this.runCommandAsRoot('bash', ['-c', 'apt-get update -qq && apt-get install -y -qq ca-certificates git curl > /dev/null 2>&1']); // Create workspace directory owned by the non-root user (node:node in Node.js images) // The node user (UID 1000) already exists in node:*-slim images