diff --git a/.changeset/codex-native-default-shell-canary.md b/.changeset/codex-native-default-shell-canary.md new file mode 100644 index 0000000..a83a758 --- /dev/null +++ b/.changeset/codex-native-default-shell-canary.md @@ -0,0 +1,5 @@ +--- +'@vercel/agent-eval': patch +--- + +Verify and repair the shell tool for native-default Codex runs. Codex CLI >= 0.144.0 (published 2026-07-09) exposes no shell/exec tool to the model when config.toml uses a custom `model_provider` (e.g. the AI Gateway) and omits the `model` key — exactly what native-default runs write. The model still answers, but it cannot run commands, read files, or use installed skills, and it sometimes fabricates command output instead of reporting the missing tool. run.mjs now pre-verifies native-default runs with a fabrication-proof shell canary (a `command_execution` item must carry a random nonce), repairs by re-stating the CLI's own resolved default model as an explicit top-level `model` key in the profile config, re-verifies, and fails loudly if the tool is still unavailable — preserving the canary's captured output on the failure result for triage. The verified outcome is memoized per sandbox (`~/.codex/agent-eval-canary.json`) so judge assertions that re-invoke the runner do not pay repeat canary calls. The repair is recorded as an optional `modelRepair` field propagated through `RunnerResult` → `AgentRunResult` → `EvalRunResult`, so persisted results show which runs needed it (and repairs dropping to zero signals the upstream fix). diff --git a/packages/agent-eval/src/lib/agents/codex.test.ts b/packages/agent-eval/src/lib/agents/codex.test.ts index 1ad1676..d755b0f 100644 --- a/packages/agent-eval/src/lib/agents/codex.test.ts +++ b/packages/agent-eval/src/lib/agents/codex.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from 'vitest'; -import { extractCodexThreadId, extractObservedModelFromCodexSession } from './codex/run.mjs'; +import { + buildModelRepairToml, + buildShellCanaryPrompt, + extractCodexThreadId, + extractObservedModelFromCodexSession, + parseCanaryMarker, + shellCanaryConfirmed, +} from './codex/run.mjs'; import { generateCodexConfig } from './codex/agent.js'; describe('generateCodexConfig', () => { @@ -114,3 +121,90 @@ describe('Codex observed model extraction', () => { expect(extractObservedModelFromCodexSession(transcript)).toBe('gpt-5.5'); }); }); + +describe('codex shell-tool canary (native-default toolless repair)', () => { + const nonce = 'agent-eval-shell-canary-deadbeef01234567'; + + it('builds a canary prompt containing the nonce and a no-tool escape hatch', () => { + const prompt = buildShellCanaryPrompt(nonce); + expect(prompt).toContain(`echo ${nonce}`); + expect(prompt).toContain('NO-SHELL-TOOL'); + }); + + it('confirms only on a completed command_execution carrying the nonce', () => { + const stdout = [ + JSON.stringify({ type: 'thread.started', thread_id: 't1' }), + JSON.stringify({ type: 'turn.started' }), + JSON.stringify({ + type: 'item.completed', + item: { + id: 'item_1', + type: 'command_execution', + command: `/bin/bash -lc 'echo ${nonce}'`, + aggregated_output: `${nonce}\n`, + exit_code: 0, + status: 'completed', + }, + }), + JSON.stringify({ type: 'turn.completed' }), + ].join('\n'); + + expect(shellCanaryConfirmed(stdout, nonce)).toBe(true); + }); + + it('rejects agent_message-only nonce echoes (observed fabrication mode)', () => { + // A toolless codex 0.145.0 was observed inventing command output for + // predictable commands: the nonce appearing in an agent message must NOT + // count as proof the shell ran. + const stdout = [ + JSON.stringify({ type: 'turn.started' }), + JSON.stringify({ + type: 'item.completed', + item: { id: 'item_0', type: 'agent_message', text: `\`\`\`text\n${nonce}\n\`\`\`` }, + }), + JSON.stringify({ type: 'turn.completed' }), + ].join('\n'); + + expect(shellCanaryConfirmed(stdout, nonce)).toBe(false); + }); + + it('rejects failed command executions and empty/non-JSON output', () => { + const failed = JSON.stringify({ + type: 'item.completed', + item: { type: 'command_execution', command: `echo ${nonce}`, aggregated_output: '', exit_code: 1 }, + }); + expect(shellCanaryConfirmed(failed, nonce)).toBe(false); + expect(shellCanaryConfirmed('', nonce)).toBe(false); + expect(shellCanaryConfirmed('plain text output', nonce)).toBe(false); + }); + + it('prefixes bare session model ids in the repair TOML', () => { + const toml = buildModelRepairToml('gpt-5.6-sol'); + expect(toml).toContain('model = "openai/gpt-5.6-sol"'); + // Prepended to the existing config: `model` is a top-level key, so the + // repair block must end with a newline and never start mid-line. + expect(toml.endsWith('\n')).toBe(true); + expect(toml.startsWith('\n')).toBe(false); + }); + + it('keeps already-prefixed model ids verbatim in the repair TOML', () => { + expect(buildModelRepairToml('openai/gpt-5.6-sol')).toContain('model = "openai/gpt-5.6-sol"'); + }); + + it('parses a verified canary marker with and without a repaired model', () => { + expect(parseCanaryMarker(JSON.stringify({ verified: true, repairedModel: 'gpt-5.6-sol' }))).toEqual({ + repairedModel: 'gpt-5.6-sol', + }); + expect(parseCanaryMarker(JSON.stringify({ verified: true, repairedModel: null }))).toEqual({ + repairedModel: null, + }); + }); + + it('treats unverified, corrupt, or empty markers as absent', () => { + expect(parseCanaryMarker(JSON.stringify({ verified: false, repairedModel: 'x' }))).toBeNull(); + expect(parseCanaryMarker(JSON.stringify({ repairedModel: 'x' }))).toBeNull(); + expect(parseCanaryMarker('not json')).toBeNull(); + expect(parseCanaryMarker('')).toBeNull(); + expect(parseCanaryMarker(undefined)).toBeNull(); + }); +}); diff --git a/packages/agent-eval/src/lib/agents/codex/run.mjs b/packages/agent-eval/src/lib/agents/codex/run.mjs index 12ff2d6..98f8c98 100644 --- a/packages/agent-eval/src/lib/agents/codex/run.mjs +++ b/packages/agent-eval/src/lib/agents/codex/run.mjs @@ -22,6 +22,7 @@ */ import { spawnSync } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -215,6 +216,223 @@ export function buildCodexExecArgs(input) { return args; } +/* ────────────────── native-default shell-tool canary + repair ────────────────── + * + * Codex CLI >= 0.144.0 exposes NO shell/exec tool to the model when config.toml + * has a custom `model_provider` (e.g. the AI Gateway) and omits the `model` key — + * exactly the shape native-default runs write. The model answers, but it cannot + * run commands, read files, or use installed skills, and it sometimes FABRICATES + * command output instead of reporting the missing tool. Verified empirically: + * 0.143.0 is the last good version; adding an explicit `model = ""` fully restores the tool. + * + * Because fabrication makes post-hoc transcript checks unreliable, native-default + * runs are pre-verified with a shell canary before the real task: + * 1. canary exec: ask codex to `echo `. PROOF is a + * `command_execution` item whose output contains the nonce — an + * agent_message merely echoing the nonce is NOT accepted (fabrication). + * 2. If the canary fails: read the model the CLI natively resolved from the + * canary's own session file, prepend an explicit `model = "openai/"` line + * to ~/.codex/default.config.toml (same semantics — the CLI chose the model; + * we only re-state it), and re-run the canary. + * 3. If the canary still fails, the run errors LOUDLY instead of executing a + * toolless agent and publishing it as normal behavior. The canary's captured + * output is preserved on that error result for triage. + * + * The verified outcome is memoized in ~/.codex (CANARY_MARKER_PATH): judge + * assertions re-invoke this runner once per assertion in the same sandbox, and + * without the marker each assertion would pay its own canary exec. + */ + +/** Marker file memoizing the canary outcome for the sandbox's lifetime. */ +const CANARY_MARKER_FILENAME = 'agent-eval-canary.json'; + +/** + * Parse the canary marker file's contents. Returns `{ repairedModel }` when the + * marker records a verified shell tool (repairedModel is null when no repair was + * needed), or null when the contents are not a valid marker. + * + * @param {string|undefined} raw + * @returns {{repairedModel: string|null}|null} + */ +export function parseCanaryMarker(raw) { + if (!raw) return null; + try { + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === 'object' && parsed.verified === true) { + return { repairedModel: typeof parsed.repairedModel === 'string' ? parsed.repairedModel : null }; + } + } catch { + // Corrupt marker → treat as absent and re-verify. + } + return null; +} + +/** + * Canary prompt. The nonce is random per run so a cached/fabricated answer can + * never pass. Kept terse to minimize tokens. + * + * @param {string} nonce + * @returns {string} + */ +export function buildShellCanaryPrompt(nonce) { + return `Run the shell command \`echo ${nonce}\` and reply with its exact output. If you cannot run shell commands, reply exactly: NO-SHELL-TOOL`; +} + +/** + * True iff the codex --json stdout proves the shell ran: a completed + * `command_execution` item with exit_code 0 whose command or aggregated_output + * contains the nonce. Agent messages containing the nonce do NOT count — a + * toolless codex has been observed inventing plausible command output. + * + * @param {string} stdout + * @param {string} nonce + * @returns {boolean} + */ +export function shellCanaryConfirmed(stdout, nonce) { + if (!stdout) return false; + for (const line of stdout.split('\n')) { + if (!line.trim()) continue; + try { + const event = JSON.parse(line); + const item = event.item; + if ( + event.type === 'item.completed' && + item && + item.type === 'command_execution' && + item.exit_code === 0 && + ((typeof item.command === 'string' && item.command.includes(nonce)) || + (typeof item.aggregated_output === 'string' && item.aggregated_output.includes(nonce))) + ) { + return true; + } + } catch { + // Ignore non-JSON output lines. + } + } + return false; +} + +/** + * The config repair: an explicit top-level `model` line for + * ~/.codex/default.config.toml. Gateway model ids are prefixed + * ("openai/gpt-5.6-sol"); session files record the bare id, so prefix when + * missing. The line MUST be PREPENDED to the config: `model` is a top-level + * key, and anything appended after a `[table]` header (e.g. + * `[model_providers.vercel]`) would become a key of that table instead. + * + * @param {string} observedModel + * @returns {string} + */ +export function buildModelRepairToml(observedModel) { + const fullModel = observedModel.includes('/') ? observedModel : `openai/${observedModel}`; + return `# agent-eval repair: codex >= 0.144.0 drops the shell tool when config omits \`model\`\nmodel = "${fullModel}"\n`; +} + +/** + * Run the canary exec once and report. Uses the same profile/flags as the real + * exec (buildCodexExecArgs) so it verifies the exact configuration the task will + * run under. Bounded by its own timeout so a hang cannot eat the task budget. + * + * @param {import('../plugin/contract.js').AgentRunInput} input + * @param {string} nonce + * @returns {{confirmed: boolean, stdout: string, output: string}} + */ +function runShellCanary(input, nonce) { + const res = spawnSync('codex', buildCodexExecArgs({ prompt: buildShellCanaryPrompt(nonce), extra: input.extra }), { + cwd: input.cwd, + env: process.env, + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + // A single echo round-trip; a hung canary must not eat the sandbox's task + // budget (worst case is two canary calls before the real exec). + timeout: 60_000, + }); + const stdout = res.stdout || ''; + // stdout THEN stderr, matching the runner's real-exec output convention. + const output = stdout + (res.stderr || ''); + return { confirmed: shellCanaryConfirmed(stdout, nonce), stdout, output }; +} + +/** + * Canary + (if needed) config repair for native-default runs. `error` is null + * when the shell tool is verified (possibly after repair); otherwise the run + * must fail loudly. `canaryOutput` carries the combined canary CLI output so + * failure results stay debuggable (what did the model actually reply?). + * + * @param {import('../plugin/contract.js').AgentRunInput} input + * @returns {{error: string|null, repairedModel: string|null, canaryOutput: string}} + */ +function ensureShellToolForNativeDefault(input) { + const markerPath = join(homedir(), '.codex', CANARY_MARKER_FILENAME); + + // Memoized outcome: judge assertions re-invoke this runner in the same + // sandbox; only the first invocation pays the canary exec. The marker also + // re-reports the original repair so every result carries the same evidence. + let marker; + try { + marker = parseCanaryMarker(readFileSync(markerPath, 'utf8')); + } catch { + marker = null; + } + if (marker) return { error: null, repairedModel: marker.repairedModel, canaryOutput: '' }; + + const nonce = `agent-eval-shell-canary-${randomBytes(8).toString('hex')}`; + const first = runShellCanary(input, nonce); + + const writeMarker = (repairedModel) => { + try { + writeFileSync(markerPath, JSON.stringify({ verified: true, repairedModel })); + } catch { + // Best-effort memo: without it, later invocations just re-run the canary. + } + }; + + if (first.confirmed) { + writeMarker(null); + return { error: null, repairedModel: null, canaryOutput: first.output }; + } + + // Toolless: recover the model the CLI natively resolved from the canary's own + // session, then re-state it explicitly in the profile config. + const threadId = extractCodexThreadId(first.stdout); + const sessionTranscript = captureCodexSessionTranscript(threadId); + const observedModel = extractObservedModelFromCodexSession(sessionTranscript); + if (!observedModel) { + return { + error: + 'codex shell-tool canary failed (no command_execution reached the sandbox) and the native default model could not be observed from the session file, so the explicit-model repair is impossible. Failing loudly instead of running a toolless agent.', + repairedModel: null, + canaryOutput: first.output, + }; + } + + try { + // Prepend, never append: `model` must land in the top-level TOML section, + // before any `[table]` header. See buildModelRepairToml. + const configPath = join(homedir(), '.codex', 'default.config.toml'); + const existing = readFileSync(configPath, 'utf8'); + writeFileSync(configPath, buildModelRepairToml(observedModel) + existing); + } catch (e) { + return { + error: `codex shell-tool canary failed and the config repair could not be written: ${e && e.message ? e.message : String(e)}`, + repairedModel: null, + canaryOutput: first.output, + }; + } + + const second = runShellCanary(input, nonce); + if (!second.confirmed) { + return { + error: `codex shell-tool canary still failed after explicit-model repair (model = ${observedModel}). Failing loudly instead of running a toolless agent.`, + repairedModel: null, + canaryOutput: first.output + second.output, + }; + } + writeMarker(observedModel); + return { error: null, repairedModel: observedModel, canaryOutput: first.output + second.output }; +} + /** * Run Codex over the workspace at `input.cwd` and return a RunnerResult. * @@ -270,6 +488,30 @@ export async function runAgent(input) { }; } + // Step 1.5: native-default runs (no explicit model in config) are pre-verified + // with a shell canary and repaired when codex resolves its default model into a + // toolless session (codex >= 0.144.0 with a custom provider). Explicit-model + // runs already carry `model` in the profile config and are unaffected. + let modelRepair = null; + const cliModel = input.extra?.cliModel ?? null; + if (!cliModel) { + const ensured = ensureShellToolForNativeDefault(input); + if (ensured.error) { + // Preserve the canary CLI output/transcript on the fail-loud path, the + // same way the login and real-exec failure paths preserve theirs — this + // is the only evidence of what the toolless model actually replied. + return { + ok: false, + output: ensured.canaryOutput, + transcript: extractTranscriptFromOutput(ensured.canaryOutput) ?? null, + observedModel: null, + error: ensured.error, + agentExitCode: -1, + }; + } + modelRepair = ensured.repairedModel; + } + // Step 2: codex exec. Blocking is fine — the runner has nothing else to do while // the agent works. The sandbox-level timeout bounds it. const res = spawnSync('codex', buildCodexExecArgs(input), { @@ -307,10 +549,19 @@ export async function runAgent(input) { observedModel, error: errorLines || fallback, agentExitCode, + ...(modelRepair ? { modelRepair } : {}), }; } - return { ok: true, output, transcript, observedModel, error: null, agentExitCode }; + return { + ok: true, + output, + transcript, + observedModel, + error: null, + agentExitCode, + ...(modelRepair ? { modelRepair } : {}), + }; } /* ─────────────────────────── runnable (CLI) entry ─────────────────────────── */ @@ -348,6 +599,8 @@ if (isMain) { } // Fallback channel: a compact status line (no transcript — it can be huge). + // Must carry every field readRunnerResult's fallback reconstructs, or that + // field is silently lost whenever the result file can't be read back. process.stdout.write( '__AGENT_RESULT__ ' + JSON.stringify({ @@ -355,6 +608,8 @@ if (isMain) { observedModel: result.observedModel, error: result.error, agentExitCode: result.agentExitCode, + // undefined when no repair ran → omitted by JSON.stringify. + modelRepair: result.modelRepair, }) + '\n' ); diff --git a/packages/agent-eval/src/lib/agents/eval-helper.mjs b/packages/agent-eval/src/lib/agents/eval-helper.mjs index e4dc924..98e3bad 100644 --- a/packages/agent-eval/src/lib/agents/eval-helper.mjs +++ b/packages/agent-eval/src/lib/agents/eval-helper.mjs @@ -17,7 +17,10 @@ * It is AGENTIC and reuses the SAME harness: each assertion re-invokes the codegen * agent's runner (`__agent_eval__/run.mjs`, already shipped) IN this sandbox. The * judge explores the final state (cwd) or reads the materialized transcript file — - * no fresh sandbox, no copying evidence around, no new harness. + * no fresh sandbox, no copying evidence around, no new harness. Codex runner + * re-invocations skip the native-default shell canary via the per-sandbox marker + * (~/.codex/agent-eval-canary.json, see codex/run.mjs) — only the first + * invocation pays it. * * Zero-dependency apart from `vitest` (already present in the fixture). Runs only * in-sandbox. The path constants below mirror shared.ts (the orchestrator writes diff --git a/packages/agent-eval/src/lib/agents/plugin/contract.ts b/packages/agent-eval/src/lib/agents/plugin/contract.ts index c4923ce..a2aa918 100644 --- a/packages/agent-eval/src/lib/agents/plugin/contract.ts +++ b/packages/agent-eval/src/lib/agents/plugin/contract.ts @@ -164,4 +164,14 @@ export interface RunnerResult { error: string | null; /** The agent CLI's exit code (diagnostics; -1 if it could not be spawned). */ agentExitCode: number; + /** + * Codex only: the model the CLI natively resolved, re-stated explicitly in the + * profile config by the shell-tool repair (codex >= 0.144.0 drops the exec tool + * on custom providers when config omits `model`). Absent when no repair ran. + * Evidence, not configuration — the model choice was still the CLI's own. + * Propagated through AgentRunResult into EvalRunResult so persisted results + * record which runs needed the repair (also the workaround's removal signal: + * repairs dropping to zero means the upstream CLI is fixed). + */ + modelRepair?: string; } diff --git a/packages/agent-eval/src/lib/agents/plugin/orchestrator.ts b/packages/agent-eval/src/lib/agents/plugin/orchestrator.ts index 051f097..414554b 100644 --- a/packages/agent-eval/src/lib/agents/plugin/orchestrator.ts +++ b/packages/agent-eval/src/lib/agents/plugin/orchestrator.ts @@ -211,6 +211,7 @@ async function readRunnerResult( observedModel: status.observedModel ?? null, error: status.error ?? null, agentExitCode: status.agentExitCode ?? -1, + ...(status.modelRepair ? { modelRepair: status.modelRepair } : {}), }; } catch { // fall through to the throw @@ -236,6 +237,7 @@ export async function runWithDefinition( let agentOutput = ''; let transcript: string | undefined; let observedModel: string | undefined; + let modelRepair: string | undefined; let aborted = false; let sandboxStopped = false; @@ -264,7 +266,11 @@ export async function runWithDefinition( return { success: false, output: '', error: 'Aborted', duration: Date.now() - startTime }; } - // 2. Create the sandbox. + // 2. Create the sandbox. One sandbox serves the codegen run AND every judge + // re-invocation of the runner (eval-helper.mjs); codex's shell-canary + // memoization (~/.codex/agent-eval-canary.json, see codex/run.mjs) relies + // on that shared lifetime — a sandbox-per-invocation change would make + // every judge assertion re-pay the canary exec. sandbox = await createSandbox({ timeout: options.timeout, runtime: 'node24', @@ -334,6 +340,7 @@ export async function runWithDefinition( agentOutput = runnerResult.output; transcript = runnerResult.transcript ?? undefined; observedModel = runnerResult.observedModel ?? undefined; + modelRepair = runnerResult.modelRepair ?? undefined; if (aborted) { return { @@ -357,6 +364,7 @@ export async function runWithDefinition( duration: Date.now() - startTime, sandboxId: sandbox.sandboxId, observedModel, + modelRepair, }; } @@ -404,6 +412,7 @@ export async function runWithDefinition( generatedFiles, deletedFiles, observedModel, + modelRepair, }; } catch (error) { // Abort wins over a generic error (same as the old adapter). @@ -425,6 +434,7 @@ export async function runWithDefinition( duration: Date.now() - startTime, sandboxId: sandbox?.sandboxId, observedModel, + modelRepair, }; } finally { if (options.signal) { diff --git a/packages/agent-eval/src/lib/agents/types.ts b/packages/agent-eval/src/lib/agents/types.ts index cd68f79..6e42e85 100644 --- a/packages/agent-eval/src/lib/agents/types.ts +++ b/packages/agent-eval/src/lib/agents/types.ts @@ -86,6 +86,8 @@ export interface AgentRunResult { deletedFiles?: string[]; /** Model reported by the underlying agent CLI, when observable */ observedModel?: string; + /** Codex only: model re-stated explicitly by the shell-tool repair (see RunnerResult.modelRepair) */ + modelRepair?: string; } /** diff --git a/packages/agent-eval/src/lib/results.test.ts b/packages/agent-eval/src/lib/results.test.ts index fab9cc2..4f182e5 100644 --- a/packages/agent-eval/src/lib/results.test.ts +++ b/packages/agent-eval/src/lib/results.test.ts @@ -48,6 +48,23 @@ describe('results utilities', () => { expect(runData.transcript).toBe('{"role":"assistant","content":"Hello"}'); expect(runData.outputContent?.eval).toBe('test output'); expect(runData.outputContent?.scripts?.build).toBe('build output'); + // No repair ran → field omitted, mirroring observedModel's omit-when-absent shape. + expect(runData.result.modelRepair).toBeUndefined(); + }); + + it('copies modelRepair through like observedModel (shell-tool repair evidence)', () => { + const agentResult: AgentRunResult = { + success: true, + output: 'Agent output', + duration: 45000, + observedModel: 'openai/gpt-5.6-sol', + modelRepair: 'gpt-5.6-sol', + }; + + const runData = agentResultToEvalRunData(agentResult); + + expect(runData.result.observedModel).toBe('openai/gpt-5.6-sol'); + expect(runData.result.modelRepair).toBe('gpt-5.6-sol'); }); it('converts failed agent result', () => { @@ -243,7 +260,7 @@ describe('results utilities', () => { const evals = [ createEvalSummary('eval-1', [ - { result: { status: 'passed', duration: 10, observedModel: 'vercel/openai/gpt-5.5' } }, + { result: { status: 'passed', duration: 10, observedModel: 'vercel/openai/gpt-5.5', modelRepair: 'gpt-5.5' } }, ]), ]; @@ -266,6 +283,9 @@ describe('results utilities', () => { expect(resultJson.model).toBe('vercel/openai/gpt-5.5'); expect(resultJson.requestedModel).toBeUndefined(); expect(resultJson.observedModel).toBe('vercel/openai/gpt-5.5'); + // Repair evidence must survive into the persisted result.json — it is the + // removal signal for the codex shell-tool workaround (see codex/run.mjs). + expect(resultJson.modelRepair).toBe('gpt-5.5'); }); it('does not collide when script is named "eval"', () => { diff --git a/packages/agent-eval/src/lib/results.ts b/packages/agent-eval/src/lib/results.ts index 4c8436f..56dccf2 100644 --- a/packages/agent-eval/src/lib/results.ts +++ b/packages/agent-eval/src/lib/results.ts @@ -62,6 +62,9 @@ export function agentResultToEvalRunData( if (agentResult.observedModel) { result.observedModel = agentResult.observedModel; } + if (agentResult.modelRepair) { + result.modelRepair = agentResult.modelRepair; + } return { result, diff --git a/packages/agent-eval/src/lib/runner.test.ts b/packages/agent-eval/src/lib/runner.test.ts index 1e6913e..7eba470 100644 --- a/packages/agent-eval/src/lib/runner.test.ts +++ b/packages/agent-eval/src/lib/runner.test.ts @@ -621,6 +621,7 @@ describe('runExperiment', () => { output: 'Agent output', duration: 1000, observedModel: 'runtime-model', + modelRepair: 'runtime-model', testResult: { success: true, output: 'Test passed' }, scriptsResults: {}, }), @@ -661,6 +662,8 @@ describe('runExperiment', () => { modelPolicy: 'native-default', })); expect(results.evals[0].runs[0].result.observedModel).toBe('runtime-model'); + // Shell-tool repair evidence follows the same path as observedModel. + expect(results.evals[0].runs[0].result.modelRepair).toBe('runtime-model'); }); it('passes response-only validation mode to the agent', async () => { diff --git a/packages/agent-eval/src/lib/types.ts b/packages/agent-eval/src/lib/types.ts index 571723d..bd29ba0 100644 --- a/packages/agent-eval/src/lib/types.ts +++ b/packages/agent-eval/src/lib/types.ts @@ -264,6 +264,8 @@ export interface EvalRunResult { requestedModel?: string; /** Model reported by the underlying agent CLI, when observable */ observedModel?: string; + /** Codex only: model re-stated explicitly by the shell-tool repair (evidence the run needed it) */ + modelRepair?: string; /** Path to parsed transcript file (relative to run directory) */ transcriptPath?: string; /** Path to raw transcript file (relative to run directory) */