Verify and repair the shell tool for native-default Codex runs

.changeset/codex-native-default-shell-canary.md

@@ -0,0 +1,5 @@
+---
+'@vercel/agent-eval': patch
+---
+
+Verify and repair the shell tool for native-default Codex runs. Codex CLI >= 0.144.0 (published 2026-07-09) exposes no shell/exec tool to the model when config.toml uses a custom `model_provider` (e.g. the AI Gateway) and omits the `model` key — exactly what native-default runs write. The model still answers, but it cannot run commands, read files, or use installed skills, and it sometimes fabricates command output instead of reporting the missing tool. run.mjs now pre-verifies native-default runs with a fabrication-proof shell canary (a `command_execution` item must carry a random nonce), repairs by re-stating the CLI's own resolved default model as an explicit top-level `model` key in the profile config, re-verifies, and fails loudly if the tool is still unavailable — preserving the canary's captured output on the failure result for triage. The verified outcome is memoized per sandbox (`~/.codex/agent-eval-canary.json`) so judge assertions that re-invoke the runner do not pay repeat canary calls. The repair is recorded as an optional `modelRepair` field propagated through `RunnerResult` → `AgentRunResult` → `EvalRunResult`, so persisted results show which runs needed it (and repairs dropping to zero signals the upstream fix).

packages/agent-eval/src/lib/agents/codex.test.ts

@@ -1,5 +1,12 @@
 import { describe, expect, it } from 'vitest';
-import { extractCodexThreadId, extractObservedModelFromCodexSession } from './codex/run.mjs';
+import {
+  buildModelRepairToml,
+  buildShellCanaryPrompt,
+  extractCodexThreadId,
+  extractObservedModelFromCodexSession,
+  parseCanaryMarker,
+  shellCanaryConfirmed,
+} from './codex/run.mjs';
 import { generateCodexConfig } from './codex/agent.js';
 
 describe('generateCodexConfig', () => {
@@ -114,3 +121,90 @@ describe('Codex observed model extraction', () => {
     expect(extractObservedModelFromCodexSession(transcript)).toBe('gpt-5.5');
   });
 });
+
+describe('codex shell-tool canary (native-default toolless repair)', () => {
+  const nonce = 'agent-eval-shell-canary-deadbeef01234567';
+
+  it('builds a canary prompt containing the nonce and a no-tool escape hatch', () => {
+    const prompt = buildShellCanaryPrompt(nonce);
+    expect(prompt).toContain(`echo ${nonce}`);
+    expect(prompt).toContain('NO-SHELL-TOOL');
+  });
+
+  it('confirms only on a completed command_execution carrying the nonce', () => {
+    const stdout = [
+      JSON.stringify({ type: 'thread.started', thread_id: 't1' }),
+      JSON.stringify({ type: 'turn.started' }),
+      JSON.stringify({
+        type: 'item.completed',
+        item: {
+          id: 'item_1',
+          type: 'command_execution',
+          command: `/bin/bash -lc 'echo ${nonce}'`,
+          aggregated_output: `${nonce}\n`,
+          exit_code: 0,
+          status: 'completed',
+        },
+      }),
+      JSON.stringify({ type: 'turn.completed' }),
+    ].join('\n');
+
+    expect(shellCanaryConfirmed(stdout, nonce)).toBe(true);
+  });
+
+  it('rejects agent_message-only nonce echoes (observed fabrication mode)', () => {
+    // A toolless codex 0.145.0 was observed inventing command output for
+    // predictable commands: the nonce appearing in an agent message must NOT
+    // count as proof the shell ran.
+    const stdout = [
+      JSON.stringify({ type: 'turn.started' }),
+      JSON.stringify({
+        type: 'item.completed',
+        item: { id: 'item_0', type: 'agent_message', text: `\`\`\`text\n${nonce}\n\`\`\`` },
+      }),
+      JSON.stringify({ type: 'turn.completed' }),
+    ].join('\n');
+
+    expect(shellCanaryConfirmed(stdout, nonce)).toBe(false);
+  });
+
+  it('rejects failed command executions and empty/non-JSON output', () => {
+    const failed = JSON.stringify({
+      type: 'item.completed',
+      item: { type: 'command_execution', command: `echo ${nonce}`, aggregated_output: '', exit_code: 1 },
+    });
+    expect(shellCanaryConfirmed(failed, nonce)).toBe(false);
+    expect(shellCanaryConfirmed('', nonce)).toBe(false);
+    expect(shellCanaryConfirmed('plain text output', nonce)).toBe(false);
+  });
+
+  it('prefixes bare session model ids in the repair TOML', () => {
+    const toml = buildModelRepairToml('gpt-5.6-sol');
+    expect(toml).toContain('model = "openai/gpt-5.6-sol"');
+    // Prepended to the existing config: `model` is a top-level key, so the
+    // repair block must end with a newline and never start mid-line.
+    expect(toml.endsWith('\n')).toBe(true);
+    expect(toml.startsWith('\n')).toBe(false);
+  });
+
+  it('keeps already-prefixed model ids verbatim in the repair TOML', () => {
+    expect(buildModelRepairToml('openai/gpt-5.6-sol')).toContain('model = "openai/gpt-5.6-sol"');
+  });
+
+  it('parses a verified canary marker with and without a repaired model', () => {
+    expect(parseCanaryMarker(JSON.stringify({ verified: true, repairedModel: 'gpt-5.6-sol' }))).toEqual({
+      repairedModel: 'gpt-5.6-sol',
+    });
+    expect(parseCanaryMarker(JSON.stringify({ verified: true, repairedModel: null }))).toEqual({
+      repairedModel: null,
+    });
+  });
+
+  it('treats unverified, corrupt, or empty markers as absent', () => {
+    expect(parseCanaryMarker(JSON.stringify({ verified: false, repairedModel: 'x' }))).toBeNull();
+    expect(parseCanaryMarker(JSON.stringify({ repairedModel: 'x' }))).toBeNull();
+    expect(parseCanaryMarker('not json')).toBeNull();
+    expect(parseCanaryMarker('')).toBeNull();
+    expect(parseCanaryMarker(undefined)).toBeNull();
+  });
+});