From f661766173cd008e13729190868e8015134ebb26 Mon Sep 17 00:00:00 2001
From: molebox <rich@vercel.com>
Date: Wed, 22 Jul 2026 14:11:13 +0200
Subject: [PATCH 1/3] agent-eval: verify and repair the shell tool for
 native-default Codex runs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Codex CLI >= 0.144.0 (published 2026-07-09; 0.143.0 is the last good
version) exposes no shell/exec tool to the model when config.toml uses a
custom model_provider and omits the model key — exactly the shape
native-default gateway runs write. The model answers, but cannot run
commands, read files, or use installed skills, and it was observed
fabricating command output instead of reporting the missing tool.

run.mjs now pre-verifies native-default runs with a fabrication-proof
shell canary (a command_execution item must carry a random nonce),
repairs by re-stating the CLI's own resolved default model as an
explicit top-level model key in the profile config (prepended — a line
appended after a [table] header would join that table), re-verifies,
and fails loudly if the tool is still unavailable. The repair is
surfaced as an optional modelRepair field on RunnerResult.

Verified end to end against the AI Gateway with codex-cli 0.145.0:
canary fails -> repair writes model = "openai/gpt-5.6-sol" -> canary
passes -> real task runs with a genuine command_execution.
---
 .../codex-native-default-shell-canary.md      |   5 +
 .../agent-eval/src/lib/agents/codex.test.ts   |  78 +++++++-
 .../agent-eval/src/lib/agents/codex/run.mjs   | 187 +++++++++++++++++-
 .../src/lib/agents/plugin/contract.ts         |   7 +
 4 files changed, 275 insertions(+), 2 deletions(-)
 create mode 100644 .changeset/codex-native-default-shell-canary.md

diff --git a/.changeset/codex-native-default-shell-canary.md b/.changeset/codex-native-default-shell-canary.md
new file mode 100644
index 0000000..baad296
--- /dev/null
+++ b/.changeset/codex-native-default-shell-canary.md
@@ -0,0 +1,5 @@
+---
+'@vercel/agent-eval': patch
+---
+
+Verify and repair the shell tool for native-default Codex runs. Codex CLI >= 0.144.0 (published 2026-07-09) exposes no shell/exec tool to the model when config.toml uses a custom `model_provider` (e.g. the AI Gateway) and omits the `model` key — exactly what native-default runs write. The model still answers, but it cannot run commands, read files, or use installed skills, and it sometimes fabricates command output instead of reporting the missing tool. run.mjs now pre-verifies native-default runs with a fabrication-proof shell canary (a `command_execution` item must carry a random nonce), repairs by re-stating the CLI's own resolved default model as an explicit top-level `model` key in the profile config, re-verifies, and fails loudly if the tool is still unavailable. The repair is surfaced as an optional `modelRepair` field on `RunnerResult`.
diff --git a/packages/agent-eval/src/lib/agents/codex.test.ts b/packages/agent-eval/src/lib/agents/codex.test.ts
index 1ad1676..f0a3a11 100644
--- a/packages/agent-eval/src/lib/agents/codex.test.ts
+++ b/packages/agent-eval/src/lib/agents/codex.test.ts
@@ -1,5 +1,11 @@
 import { describe, expect, it } from 'vitest';
-import { extractCodexThreadId, extractObservedModelFromCodexSession } from './codex/run.mjs';
+import {
+  buildModelRepairToml,
+  buildShellCanaryPrompt,
+  extractCodexThreadId,
+  extractObservedModelFromCodexSession,
+  shellCanaryConfirmed,
+} from './codex/run.mjs';
 import { generateCodexConfig } from './codex/agent.js';
 
 describe('generateCodexConfig', () => {
@@ -114,3 +120,73 @@ describe('Codex observed model extraction', () => {
     expect(extractObservedModelFromCodexSession(transcript)).toBe('gpt-5.5');
   });
 });
+
+describe('codex shell-tool canary (native-default toolless repair)', () => {
+  const nonce = 'agent-eval-shell-canary-deadbeef01234567';
+
+  it('builds a canary prompt containing the nonce and a no-tool escape hatch', () => {
+    const prompt = buildShellCanaryPrompt(nonce);
+    expect(prompt).toContain(`echo ${nonce}`);
+    expect(prompt).toContain('NO-SHELL-TOOL');
+  });
+
+  it('confirms only on a completed command_execution carrying the nonce', () => {
+    const stdout = [
+      JSON.stringify({ type: 'thread.started', thread_id: 't1' }),
+      JSON.stringify({ type: 'turn.started' }),
+      JSON.stringify({
+        type: 'item.completed',
+        item: {
+          id: 'item_1',
+          type: 'command_execution',
+          command: `/bin/bash -lc 'echo ${nonce}'`,
+          aggregated_output: `${nonce}\n`,
+          exit_code: 0,
+          status: 'completed',
+        },
+      }),
+      JSON.stringify({ type: 'turn.completed' }),
+    ].join('\n');
+
+    expect(shellCanaryConfirmed(stdout, nonce)).toBe(true);
+  });
+
+  it('rejects agent_message-only nonce echoes (observed fabrication mode)', () => {
+    // A toolless codex 0.145.0 was observed inventing command output for
+    // predictable commands: the nonce appearing in an agent message must NOT
+    // count as proof the shell ran.
+    const stdout = [
+      JSON.stringify({ type: 'turn.started' }),
+      JSON.stringify({
+        type: 'item.completed',
+        item: { id: 'item_0', type: 'agent_message', text: `\`\`\`text\n${nonce}\n\`\`\`` },
+      }),
+      JSON.stringify({ type: 'turn.completed' }),
+    ].join('\n');
+
+    expect(shellCanaryConfirmed(stdout, nonce)).toBe(false);
+  });
+
+  it('rejects failed command executions and empty/non-JSON output', () => {
+    const failed = JSON.stringify({
+      type: 'item.completed',
+      item: { type: 'command_execution', command: `echo ${nonce}`, aggregated_output: '', exit_code: 1 },
+    });
+    expect(shellCanaryConfirmed(failed, nonce)).toBe(false);
+    expect(shellCanaryConfirmed('', nonce)).toBe(false);
+    expect(shellCanaryConfirmed('plain text output', nonce)).toBe(false);
+  });
+
+  it('prefixes bare session model ids in the repair TOML', () => {
+    const toml = buildModelRepairToml('gpt-5.6-sol');
+    expect(toml).toContain('model = "openai/gpt-5.6-sol"');
+    // Prepended to the existing config: `model` is a top-level key, so the
+    // repair block must end with a newline and never start mid-line.
+    expect(toml.endsWith('\n')).toBe(true);
+    expect(toml.startsWith('\n')).toBe(false);
+  });
+
+  it('keeps already-prefixed model ids verbatim in the repair TOML', () => {
+    expect(buildModelRepairToml('openai/gpt-5.6-sol')).toContain('model = "openai/gpt-5.6-sol"');
+  });
+});
diff --git a/packages/agent-eval/src/lib/agents/codex/run.mjs b/packages/agent-eval/src/lib/agents/codex/run.mjs
index 12ff2d6..ac22f36 100644
--- a/packages/agent-eval/src/lib/agents/codex/run.mjs
+++ b/packages/agent-eval/src/lib/agents/codex/run.mjs
@@ -22,6 +22,7 @@
  */
 
 import { spawnSync } from 'node:child_process';
+import { randomBytes } from 'node:crypto';
 import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { homedir } from 'node:os';
 import { dirname, join } from 'node:path';
@@ -215,6 +216,160 @@ export function buildCodexExecArgs(input) {
   return args;
 }
 
+/* ────────────────── native-default shell-tool canary + repair ──────────────────
+ *
+ * Codex CLI >= 0.144.0 exposes NO shell/exec tool to the model when config.toml
+ * has a custom `model_provider` (e.g. the AI Gateway) and omits the `model` key —
+ * exactly the shape native-default runs write. The model answers, but it cannot
+ * run commands, read files, or use installed skills, and it sometimes FABRICATES
+ * command output instead of reporting the missing tool. Verified empirically:
+ * 0.143.0 is the last good version; adding an explicit `model = "<the same model
+ * the CLI resolves natively>"` fully restores the tool.
+ *
+ * Because fabrication makes post-hoc transcript checks unreliable, native-default
+ * runs are pre-verified with a shell canary before the real task:
+ *   1. canary exec: ask codex to `echo <random nonce>`. PROOF is a
+ *      `command_execution` item whose output contains the nonce — an
+ *      agent_message merely echoing the nonce is NOT accepted (fabrication).
+ *   2. If the canary fails: read the model the CLI natively resolved from the
+ *      canary's own session file, append an explicit `model = "openai/<it>"` line
+ *      to ~/.codex/default.config.toml (same semantics — the CLI chose the model;
+ *      we only re-state it), and re-run the canary.
+ *   3. If the canary still fails, the run errors LOUDLY instead of executing a
+ *      toolless agent and publishing it as normal behavior.
+ */
+
+/**
+ * Canary prompt. The nonce is random per run so a cached/fabricated answer can
+ * never pass. Kept terse to minimize tokens.
+ *
+ * @param {string} nonce
+ * @returns {string}
+ */
+export function buildShellCanaryPrompt(nonce) {
+  return `Run the shell command \`echo ${nonce}\` and reply with its exact output. If you cannot run shell commands, reply exactly: NO-SHELL-TOOL`;
+}
+
+/**
+ * True iff the codex --json stdout proves the shell ran: a completed
+ * `command_execution` item with exit_code 0 whose command or aggregated_output
+ * contains the nonce. Agent messages containing the nonce do NOT count — a
+ * toolless codex has been observed inventing plausible command output.
+ *
+ * @param {string} stdout
+ * @param {string} nonce
+ * @returns {boolean}
+ */
+export function shellCanaryConfirmed(stdout, nonce) {
+  if (!stdout) return false;
+  for (const line of stdout.split('\n')) {
+    if (!line.trim()) continue;
+    try {
+      const event = JSON.parse(line);
+      const item = event.item;
+      if (
+        event.type === 'item.completed' &&
+        item &&
+        item.type === 'command_execution' &&
+        item.exit_code === 0 &&
+        ((typeof item.command === 'string' && item.command.includes(nonce)) ||
+          (typeof item.aggregated_output === 'string' && item.aggregated_output.includes(nonce)))
+      ) {
+        return true;
+      }
+    } catch {
+      // Ignore non-JSON output lines.
+    }
+  }
+  return false;
+}
+
+/**
+ * The config repair: an explicit top-level `model` line for
+ * ~/.codex/default.config.toml. Gateway model ids are prefixed
+ * ("openai/gpt-5.6-sol"); session files record the bare id, so prefix when
+ * missing. The line MUST be PREPENDED to the config: `model` is a top-level
+ * key, and anything appended after a `[table]` header (e.g.
+ * `[model_providers.vercel]`) would become a key of that table instead.
+ *
+ * @param {string} observedModel
+ * @returns {string}
+ */
+export function buildModelRepairToml(observedModel) {
+  const fullModel = observedModel.includes('/') ? observedModel : `openai/${observedModel}`;
+  return `# agent-eval repair: codex >= 0.144.0 drops the shell tool when config omits \`model\`\nmodel = "${fullModel}"\n`;
+}
+
+/**
+ * Run the canary exec once and report. Uses the same profile/flags as the real
+ * exec (buildCodexExecArgs) so it verifies the exact configuration the task will
+ * run under. Bounded by its own timeout so a hang cannot eat the task budget.
+ *
+ * @param {import('../plugin/contract.js').AgentRunInput} input
+ * @param {string} nonce
+ * @returns {{confirmed: boolean, stdout: string}}
+ */
+function runShellCanary(input, nonce) {
+  const res = spawnSync('codex', buildCodexExecArgs({ prompt: buildShellCanaryPrompt(nonce), extra: input.extra }), {
+    cwd: input.cwd,
+    env: process.env,
+    encoding: 'utf8',
+    maxBuffer: 64 * 1024 * 1024,
+    timeout: 180_000,
+  });
+  const stdout = res.stdout || '';
+  return { confirmed: shellCanaryConfirmed(stdout, nonce), stdout };
+}
+
+/**
+ * Canary + (if needed) config repair for native-default runs. Returns null when
+ * the shell tool is verified (possibly after repair), or an error string when the
+ * run must fail loudly.
+ *
+ * @param {import('../plugin/contract.js').AgentRunInput} input
+ * @returns {{error: string|null, repairedModel: string|null}}
+ */
+function ensureShellToolForNativeDefault(input) {
+  const nonce = `agent-eval-shell-canary-${randomBytes(8).toString('hex')}`;
+  const first = runShellCanary(input, nonce);
+  if (first.confirmed) return { error: null, repairedModel: null };
+
+  // Toolless: recover the model the CLI natively resolved from the canary's own
+  // session, then re-state it explicitly in the profile config.
+  const threadId = extractCodexThreadId(first.stdout);
+  const sessionTranscript = captureCodexSessionTranscript(threadId);
+  const observedModel = extractObservedModelFromCodexSession(sessionTranscript);
+  if (!observedModel) {
+    return {
+      error:
+        'codex shell-tool canary failed (no command_execution reached the sandbox) and the native default model could not be observed from the session file, so the explicit-model repair is impossible. Failing loudly instead of running a toolless agent.',
+      repairedModel: null,
+    };
+  }
+
+  try {
+    // Prepend, never append: `model` must land in the top-level TOML section,
+    // before any `[table]` header. See buildModelRepairToml.
+    const configPath = join(homedir(), '.codex', 'default.config.toml');
+    const existing = readFileSync(configPath, 'utf8');
+    writeFileSync(configPath, buildModelRepairToml(observedModel) + existing);
+  } catch (e) {
+    return {
+      error: `codex shell-tool canary failed and the config repair could not be written: ${e && e.message ? e.message : String(e)}`,
+      repairedModel: null,
+    };
+  }
+
+  const second = runShellCanary(input, nonce);
+  if (!second.confirmed) {
+    return {
+      error: `codex shell-tool canary still failed after explicit-model repair (model = ${observedModel}). Failing loudly instead of running a toolless agent.`,
+      repairedModel: null,
+    };
+  }
+  return { error: null, repairedModel: observedModel };
+}
+
 /**
  * Run Codex over the workspace at `input.cwd` and return a RunnerResult.
  *
@@ -270,6 +425,27 @@ export async function runAgent(input) {
     };
   }
 
+  // Step 1.5: native-default runs (no explicit model in config) are pre-verified
+  // with a shell canary and repaired when codex resolves its default model into a
+  // toolless session (codex >= 0.144.0 with a custom provider). Explicit-model
+  // runs already carry `model` in the profile config and are unaffected.
+  let modelRepair = null;
+  const cliModel = input.extra?.cliModel ?? null;
+  if (!cliModel) {
+    const ensured = ensureShellToolForNativeDefault(input);
+    if (ensured.error) {
+      return {
+        ok: false,
+        output: '',
+        transcript: null,
+        observedModel: null,
+        error: ensured.error,
+        agentExitCode: -1,
+      };
+    }
+    modelRepair = ensured.repairedModel;
+  }
+
   // Step 2: codex exec. Blocking is fine — the runner has nothing else to do while
   // the agent works. The sandbox-level timeout bounds it.
   const res = spawnSync('codex', buildCodexExecArgs(input), {
@@ -307,10 +483,19 @@ export async function runAgent(input) {
       observedModel,
       error: errorLines || fallback,
       agentExitCode,
+      ...(modelRepair ? { modelRepair } : {}),
     };
   }
 
-  return { ok: true, output, transcript, observedModel, error: null, agentExitCode };
+  return {
+    ok: true,
+    output,
+    transcript,
+    observedModel,
+    error: null,
+    agentExitCode,
+    ...(modelRepair ? { modelRepair } : {}),
+  };
 }
 
 /* ─────────────────────────── runnable (CLI) entry ─────────────────────────── */
diff --git a/packages/agent-eval/src/lib/agents/plugin/contract.ts b/packages/agent-eval/src/lib/agents/plugin/contract.ts
index c4923ce..6e02950 100644
--- a/packages/agent-eval/src/lib/agents/plugin/contract.ts
+++ b/packages/agent-eval/src/lib/agents/plugin/contract.ts
@@ -164,4 +164,11 @@ export interface RunnerResult {
   error: string | null;
   /** The agent CLI's exit code (diagnostics; -1 if it could not be spawned). */
   agentExitCode: number;
+  /**
+   * Codex only: the model the CLI natively resolved, re-stated explicitly in the
+   * profile config by the shell-tool repair (codex >= 0.144.0 drops the exec tool
+   * on custom providers when config omits `model`). Absent when no repair ran.
+   * Evidence, not configuration — the model choice was still the CLI's own.
+   */
+  modelRepair?: string;
 }

From f820e3317a5df51bd9bc21885c0a167b6be96571 Mon Sep 17 00:00:00 2001
From: molebox <rich@vercel.com>
Date: Wed, 22 Jul 2026 14:46:23 +0200
Subject: [PATCH 2/3] Address review: propagate modelRepair, preserve canary
 output, memoize canary per sandbox

- modelRepair now flows RunnerResult -> AgentRunResult -> EvalRunResult
  (mirroring observedModel) so persisted results record which runs needed
  the shell-tool repair; repairs dropping to zero is the workaround's
  removal signal.
- The canary fail-loud path now returns the captured canary CLI output
  and transcript instead of discarding them, matching the login and
  real-exec failure paths.
- The verified canary outcome is memoized in ~/.codex/agent-eval-canary.json
  for the sandbox's lifetime: judge assertions re-invoke run.mjs once per
  assertion and previously would each have paid a canary exec. The marker
  re-reports the original repairedModel so every invocation carries the
  same evidence.

Verified live against the gateway: first invocation repairs and records
modelRepair (14.6s); second invocation skips the canary via the marker
(1.7s) and still reports the repair. 279 tests green (2 new for the
marker parser).
---
 .../codex-native-default-shell-canary.md      |  2 +-
 .../agent-eval/src/lib/agents/codex.test.ts   | 18 ++++
 .../agent-eval/src/lib/agents/codex/run.mjs   | 88 ++++++++++++++++---
 .../src/lib/agents/plugin/contract.ts         |  3 +
 .../src/lib/agents/plugin/orchestrator.ts     |  5 ++
 packages/agent-eval/src/lib/agents/types.ts   |  2 +
 packages/agent-eval/src/lib/results.ts        |  3 +
 packages/agent-eval/src/lib/types.ts          |  2 +
 8 files changed, 110 insertions(+), 13 deletions(-)

diff --git a/.changeset/codex-native-default-shell-canary.md b/.changeset/codex-native-default-shell-canary.md
index baad296..a83a758 100644
--- a/.changeset/codex-native-default-shell-canary.md
+++ b/.changeset/codex-native-default-shell-canary.md
@@ -2,4 +2,4 @@
 '@vercel/agent-eval': patch
 ---
 
-Verify and repair the shell tool for native-default Codex runs. Codex CLI >= 0.144.0 (published 2026-07-09) exposes no shell/exec tool to the model when config.toml uses a custom `model_provider` (e.g. the AI Gateway) and omits the `model` key — exactly what native-default runs write. The model still answers, but it cannot run commands, read files, or use installed skills, and it sometimes fabricates command output instead of reporting the missing tool. run.mjs now pre-verifies native-default runs with a fabrication-proof shell canary (a `command_execution` item must carry a random nonce), repairs by re-stating the CLI's own resolved default model as an explicit top-level `model` key in the profile config, re-verifies, and fails loudly if the tool is still unavailable. The repair is surfaced as an optional `modelRepair` field on `RunnerResult`.
+Verify and repair the shell tool for native-default Codex runs. Codex CLI >= 0.144.0 (published 2026-07-09) exposes no shell/exec tool to the model when config.toml uses a custom `model_provider` (e.g. the AI Gateway) and omits the `model` key — exactly what native-default runs write. The model still answers, but it cannot run commands, read files, or use installed skills, and it sometimes fabricates command output instead of reporting the missing tool. run.mjs now pre-verifies native-default runs with a fabrication-proof shell canary (a `command_execution` item must carry a random nonce), repairs by re-stating the CLI's own resolved default model as an explicit top-level `model` key in the profile config, re-verifies, and fails loudly if the tool is still unavailable — preserving the canary's captured output on the failure result for triage. The verified outcome is memoized per sandbox (`~/.codex/agent-eval-canary.json`) so judge assertions that re-invoke the runner do not pay repeat canary calls. The repair is recorded as an optional `modelRepair` field propagated through `RunnerResult` → `AgentRunResult` → `EvalRunResult`, so persisted results show which runs needed it (and repairs dropping to zero signals the upstream fix).
diff --git a/packages/agent-eval/src/lib/agents/codex.test.ts b/packages/agent-eval/src/lib/agents/codex.test.ts
index f0a3a11..d755b0f 100644
--- a/packages/agent-eval/src/lib/agents/codex.test.ts
+++ b/packages/agent-eval/src/lib/agents/codex.test.ts
@@ -4,6 +4,7 @@ import {
   buildShellCanaryPrompt,
   extractCodexThreadId,
   extractObservedModelFromCodexSession,
+  parseCanaryMarker,
   shellCanaryConfirmed,
 } from './codex/run.mjs';
 import { generateCodexConfig } from './codex/agent.js';
@@ -189,4 +190,21 @@ describe('codex shell-tool canary (native-default toolless repair)', () => {
   it('keeps already-prefixed model ids verbatim in the repair TOML', () => {
     expect(buildModelRepairToml('openai/gpt-5.6-sol')).toContain('model = "openai/gpt-5.6-sol"');
   });
+
+  it('parses a verified canary marker with and without a repaired model', () => {
+    expect(parseCanaryMarker(JSON.stringify({ verified: true, repairedModel: 'gpt-5.6-sol' }))).toEqual({
+      repairedModel: 'gpt-5.6-sol',
+    });
+    expect(parseCanaryMarker(JSON.stringify({ verified: true, repairedModel: null }))).toEqual({
+      repairedModel: null,
+    });
+  });
+
+  it('treats unverified, corrupt, or empty markers as absent', () => {
+    expect(parseCanaryMarker(JSON.stringify({ verified: false, repairedModel: 'x' }))).toBeNull();
+    expect(parseCanaryMarker(JSON.stringify({ repairedModel: 'x' }))).toBeNull();
+    expect(parseCanaryMarker('not json')).toBeNull();
+    expect(parseCanaryMarker('')).toBeNull();
+    expect(parseCanaryMarker(undefined)).toBeNull();
+  });
 });
diff --git a/packages/agent-eval/src/lib/agents/codex/run.mjs b/packages/agent-eval/src/lib/agents/codex/run.mjs
index ac22f36..99442b3 100644
--- a/packages/agent-eval/src/lib/agents/codex/run.mjs
+++ b/packages/agent-eval/src/lib/agents/codex/run.mjs
@@ -232,12 +232,41 @@ export function buildCodexExecArgs(input) {
  *      `command_execution` item whose output contains the nonce — an
  *      agent_message merely echoing the nonce is NOT accepted (fabrication).
  *   2. If the canary fails: read the model the CLI natively resolved from the
- *      canary's own session file, append an explicit `model = "openai/<it>"` line
+ *      canary's own session file, prepend an explicit `model = "openai/<it>"` line
  *      to ~/.codex/default.config.toml (same semantics — the CLI chose the model;
  *      we only re-state it), and re-run the canary.
  *   3. If the canary still fails, the run errors LOUDLY instead of executing a
- *      toolless agent and publishing it as normal behavior.
+ *      toolless agent and publishing it as normal behavior. The canary's captured
+ *      output is preserved on that error result for triage.
+ *
+ * The verified outcome is memoized in ~/.codex (CANARY_MARKER_PATH): judge
+ * assertions re-invoke this runner once per assertion in the same sandbox, and
+ * without the marker each assertion would pay its own canary exec.
+ */
+
+/** Marker file memoizing the canary outcome for the sandbox's lifetime. */
+const CANARY_MARKER_FILENAME = 'agent-eval-canary.json';
+
+/**
+ * Parse the canary marker file's contents. Returns `{ repairedModel }` when the
+ * marker records a verified shell tool (repairedModel is null when no repair was
+ * needed), or null when the contents are not a valid marker.
+ *
+ * @param {string|undefined} raw
+ * @returns {{repairedModel: string|null}|null}
  */
+export function parseCanaryMarker(raw) {
+  if (!raw) return null;
+  try {
+    const parsed = JSON.parse(raw);
+    if (parsed && typeof parsed === 'object' && parsed.verified === true) {
+      return { repairedModel: typeof parsed.repairedModel === 'string' ? parsed.repairedModel : null };
+    }
+  } catch {
+    // Corrupt marker → treat as absent and re-verify.
+  }
+  return null;
+}
 
 /**
  * Canary prompt. The nonce is random per run so a cached/fabricated answer can
@@ -307,7 +336,7 @@ export function buildModelRepairToml(observedModel) {
  *
  * @param {import('../plugin/contract.js').AgentRunInput} input
  * @param {string} nonce
- * @returns {{confirmed: boolean, stdout: string}}
+ * @returns {{confirmed: boolean, stdout: string, output: string}}
  */
 function runShellCanary(input, nonce) {
   const res = spawnSync('codex', buildCodexExecArgs({ prompt: buildShellCanaryPrompt(nonce), extra: input.extra }), {
@@ -318,21 +347,49 @@ function runShellCanary(input, nonce) {
     timeout: 180_000,
   });
   const stdout = res.stdout || '';
-  return { confirmed: shellCanaryConfirmed(stdout, nonce), stdout };
+  // stdout THEN stderr, matching the runner's real-exec output convention.
+  const output = stdout + (res.stderr || '');
+  return { confirmed: shellCanaryConfirmed(stdout, nonce), stdout, output };
 }
 
 /**
- * Canary + (if needed) config repair for native-default runs. Returns null when
- * the shell tool is verified (possibly after repair), or an error string when the
- * run must fail loudly.
+ * Canary + (if needed) config repair for native-default runs. `error` is null
+ * when the shell tool is verified (possibly after repair); otherwise the run
+ * must fail loudly. `canaryOutput` carries the combined canary CLI output so
+ * failure results stay debuggable (what did the model actually reply?).
  *
  * @param {import('../plugin/contract.js').AgentRunInput} input
- * @returns {{error: string|null, repairedModel: string|null}}
+ * @returns {{error: string|null, repairedModel: string|null, canaryOutput: string}}
  */
 function ensureShellToolForNativeDefault(input) {
+  const markerPath = join(homedir(), '.codex', CANARY_MARKER_FILENAME);
+
+  // Memoized outcome: judge assertions re-invoke this runner in the same
+  // sandbox; only the first invocation pays the canary exec. The marker also
+  // re-reports the original repair so every result carries the same evidence.
+  let marker;
+  try {
+    marker = parseCanaryMarker(readFileSync(markerPath, 'utf8'));
+  } catch {
+    marker = null;
+  }
+  if (marker) return { error: null, repairedModel: marker.repairedModel, canaryOutput: '' };
+
   const nonce = `agent-eval-shell-canary-${randomBytes(8).toString('hex')}`;
   const first = runShellCanary(input, nonce);
-  if (first.confirmed) return { error: null, repairedModel: null };
+
+  const writeMarker = (repairedModel) => {
+    try {
+      writeFileSync(markerPath, JSON.stringify({ verified: true, repairedModel }));
+    } catch {
+      // Best-effort memo: without it, later invocations just re-run the canary.
+    }
+  };
+
+  if (first.confirmed) {
+    writeMarker(null);
+    return { error: null, repairedModel: null, canaryOutput: first.output };
+  }
 
   // Toolless: recover the model the CLI natively resolved from the canary's own
   // session, then re-state it explicitly in the profile config.
@@ -344,6 +401,7 @@ function ensureShellToolForNativeDefault(input) {
       error:
         'codex shell-tool canary failed (no command_execution reached the sandbox) and the native default model could not be observed from the session file, so the explicit-model repair is impossible. Failing loudly instead of running a toolless agent.',
       repairedModel: null,
+      canaryOutput: first.output,
     };
   }
 
@@ -357,6 +415,7 @@ function ensureShellToolForNativeDefault(input) {
     return {
       error: `codex shell-tool canary failed and the config repair could not be written: ${e && e.message ? e.message : String(e)}`,
       repairedModel: null,
+      canaryOutput: first.output,
     };
   }
 
@@ -365,9 +424,11 @@ function ensureShellToolForNativeDefault(input) {
     return {
       error: `codex shell-tool canary still failed after explicit-model repair (model = ${observedModel}). Failing loudly instead of running a toolless agent.`,
       repairedModel: null,
+      canaryOutput: first.output + second.output,
     };
   }
-  return { error: null, repairedModel: observedModel };
+  writeMarker(observedModel);
+  return { error: null, repairedModel: observedModel, canaryOutput: first.output + second.output };
 }
 
 /**
@@ -434,10 +495,13 @@ export async function runAgent(input) {
   if (!cliModel) {
     const ensured = ensureShellToolForNativeDefault(input);
     if (ensured.error) {
+      // Preserve the canary CLI output/transcript on the fail-loud path, the
+      // same way the login and real-exec failure paths preserve theirs — this
+      // is the only evidence of what the toolless model actually replied.
       return {
         ok: false,
-        output: '',
-        transcript: null,
+        output: ensured.canaryOutput,
+        transcript: extractTranscriptFromOutput(ensured.canaryOutput) ?? null,
         observedModel: null,
         error: ensured.error,
         agentExitCode: -1,
diff --git a/packages/agent-eval/src/lib/agents/plugin/contract.ts b/packages/agent-eval/src/lib/agents/plugin/contract.ts
index 6e02950..a2aa918 100644
--- a/packages/agent-eval/src/lib/agents/plugin/contract.ts
+++ b/packages/agent-eval/src/lib/agents/plugin/contract.ts
@@ -169,6 +169,9 @@ export interface RunnerResult {
    * profile config by the shell-tool repair (codex >= 0.144.0 drops the exec tool
    * on custom providers when config omits `model`). Absent when no repair ran.
    * Evidence, not configuration — the model choice was still the CLI's own.
+   * Propagated through AgentRunResult into EvalRunResult so persisted results
+   * record which runs needed the repair (also the workaround's removal signal:
+   * repairs dropping to zero means the upstream CLI is fixed).
    */
   modelRepair?: string;
 }
diff --git a/packages/agent-eval/src/lib/agents/plugin/orchestrator.ts b/packages/agent-eval/src/lib/agents/plugin/orchestrator.ts
index 051f097..c17ed66 100644
--- a/packages/agent-eval/src/lib/agents/plugin/orchestrator.ts
+++ b/packages/agent-eval/src/lib/agents/plugin/orchestrator.ts
@@ -236,6 +236,7 @@ export async function runWithDefinition(
   let agentOutput = '';
   let transcript: string | undefined;
   let observedModel: string | undefined;
+  let modelRepair: string | undefined;
   let aborted = false;
   let sandboxStopped = false;
 
@@ -334,6 +335,7 @@ export async function runWithDefinition(
     agentOutput = runnerResult.output;
     transcript = runnerResult.transcript ?? undefined;
     observedModel = runnerResult.observedModel ?? undefined;
+    modelRepair = runnerResult.modelRepair ?? undefined;
 
     if (aborted) {
       return {
@@ -357,6 +359,7 @@ export async function runWithDefinition(
         duration: Date.now() - startTime,
         sandboxId: sandbox.sandboxId,
         observedModel,
+        modelRepair,
       };
     }
 
@@ -404,6 +407,7 @@ export async function runWithDefinition(
       generatedFiles,
       deletedFiles,
       observedModel,
+      modelRepair,
     };
   } catch (error) {
     // Abort wins over a generic error (same as the old adapter).
@@ -425,6 +429,7 @@ export async function runWithDefinition(
       duration: Date.now() - startTime,
       sandboxId: sandbox?.sandboxId,
       observedModel,
+      modelRepair,
     };
   } finally {
     if (options.signal) {
diff --git a/packages/agent-eval/src/lib/agents/types.ts b/packages/agent-eval/src/lib/agents/types.ts
index cd68f79..6e42e85 100644
--- a/packages/agent-eval/src/lib/agents/types.ts
+++ b/packages/agent-eval/src/lib/agents/types.ts
@@ -86,6 +86,8 @@ export interface AgentRunResult {
   deletedFiles?: string[];
   /** Model reported by the underlying agent CLI, when observable */
   observedModel?: string;
+  /** Codex only: model re-stated explicitly by the shell-tool repair (see RunnerResult.modelRepair) */
+  modelRepair?: string;
 }
 
 /**
diff --git a/packages/agent-eval/src/lib/results.ts b/packages/agent-eval/src/lib/results.ts
index 4c8436f..56dccf2 100644
--- a/packages/agent-eval/src/lib/results.ts
+++ b/packages/agent-eval/src/lib/results.ts
@@ -62,6 +62,9 @@ export function agentResultToEvalRunData(
 	if (agentResult.observedModel) {
 		result.observedModel = agentResult.observedModel;
 	}
+	if (agentResult.modelRepair) {
+		result.modelRepair = agentResult.modelRepair;
+	}
 
 	return {
 		result,
diff --git a/packages/agent-eval/src/lib/types.ts b/packages/agent-eval/src/lib/types.ts
index 571723d..bd29ba0 100644
--- a/packages/agent-eval/src/lib/types.ts
+++ b/packages/agent-eval/src/lib/types.ts
@@ -264,6 +264,8 @@ export interface EvalRunResult {
   requestedModel?: string;
   /** Model reported by the underlying agent CLI, when observable */
   observedModel?: string;
+  /** Codex only: model re-stated explicitly by the shell-tool repair (evidence the run needed it) */
+  modelRepair?: string;
   /** Path to parsed transcript file (relative to run directory) */
   transcriptPath?: string;
   /** Path to raw transcript file (relative to run directory) */

From dd84f36f416177cd9b85b5242794609316c52f55 Mon Sep 17 00:00:00 2001
From: molebox <rich@vercel.com>
Date: Wed, 22 Jul 2026 15:33:01 +0200
Subject: [PATCH 3/3] Address second review pass: test modelRepair plumbing,
 carry it on the fallback channel, tighten canary timeout

- results.test.ts/runner.test.ts now pin modelRepair propagation at the
  same layers where observedModel (its precedent) is tested, so a refactor
  cannot silently drop the repair evidence that serves as the workaround's
  removal signal.
- The __AGENT_RESULT__ status line and readRunnerResult's fallback
  reconstruction now carry modelRepair, matching observedModel on that
  channel (previously the field was lost whenever the result file could
  not be read back).
- Canary spawnSync timeout 180s -> 60s: a single echo round-trip never
  needs more, and the worst-case pre-task budget drops from 360s to 120s
  of the 600s default sandbox lifetime.
- Cross-reference comments at the orchestrator's sandbox-creation step and
  eval-helper's header document the canary marker's dependence on the
  shared per-run sandbox lifetime.
---
 .../agent-eval/src/lib/agents/codex/run.mjs   |  8 ++++++-
 .../agent-eval/src/lib/agents/eval-helper.mjs |  5 ++++-
 .../src/lib/agents/plugin/orchestrator.ts     |  7 +++++-
 packages/agent-eval/src/lib/results.test.ts   | 22 ++++++++++++++++++-
 packages/agent-eval/src/lib/runner.test.ts    |  3 +++
 5 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/packages/agent-eval/src/lib/agents/codex/run.mjs b/packages/agent-eval/src/lib/agents/codex/run.mjs
index 99442b3..98f8c98 100644
--- a/packages/agent-eval/src/lib/agents/codex/run.mjs
+++ b/packages/agent-eval/src/lib/agents/codex/run.mjs
@@ -344,7 +344,9 @@ function runShellCanary(input, nonce) {
     env: process.env,
     encoding: 'utf8',
     maxBuffer: 64 * 1024 * 1024,
-    timeout: 180_000,
+    // A single echo round-trip; a hung canary must not eat the sandbox's task
+    // budget (worst case is two canary calls before the real exec).
+    timeout: 60_000,
   });
   const stdout = res.stdout || '';
   // stdout THEN stderr, matching the runner's real-exec output convention.
@@ -597,6 +599,8 @@ if (isMain) {
   }
 
   // Fallback channel: a compact status line (no transcript — it can be huge).
+  // Must carry every field readRunnerResult's fallback reconstructs, or that
+  // field is silently lost whenever the result file can't be read back.
   process.stdout.write(
     '__AGENT_RESULT__ ' +
       JSON.stringify({
@@ -604,6 +608,8 @@ if (isMain) {
         observedModel: result.observedModel,
         error: result.error,
         agentExitCode: result.agentExitCode,
+        // undefined when no repair ran → omitted by JSON.stringify.
+        modelRepair: result.modelRepair,
       }) +
       '\n'
   );
diff --git a/packages/agent-eval/src/lib/agents/eval-helper.mjs b/packages/agent-eval/src/lib/agents/eval-helper.mjs
index e4dc924..98e3bad 100644
--- a/packages/agent-eval/src/lib/agents/eval-helper.mjs
+++ b/packages/agent-eval/src/lib/agents/eval-helper.mjs
@@ -17,7 +17,10 @@
  * It is AGENTIC and reuses the SAME harness: each assertion re-invokes the codegen
  * agent's runner (`__agent_eval__/run.mjs`, already shipped) IN this sandbox. The
  * judge explores the final state (cwd) or reads the materialized transcript file —
- * no fresh sandbox, no copying evidence around, no new harness.
+ * no fresh sandbox, no copying evidence around, no new harness. Codex runner
+ * re-invocations skip the native-default shell canary via the per-sandbox marker
+ * (~/.codex/agent-eval-canary.json, see codex/run.mjs) — only the first
+ * invocation pays it.
  *
  * Zero-dependency apart from `vitest` (already present in the fixture). Runs only
  * in-sandbox. The path constants below mirror shared.ts (the orchestrator writes
diff --git a/packages/agent-eval/src/lib/agents/plugin/orchestrator.ts b/packages/agent-eval/src/lib/agents/plugin/orchestrator.ts
index c17ed66..414554b 100644
--- a/packages/agent-eval/src/lib/agents/plugin/orchestrator.ts
+++ b/packages/agent-eval/src/lib/agents/plugin/orchestrator.ts
@@ -211,6 +211,7 @@ async function readRunnerResult(
         observedModel: status.observedModel ?? null,
         error: status.error ?? null,
         agentExitCode: status.agentExitCode ?? -1,
+        ...(status.modelRepair ? { modelRepair: status.modelRepair } : {}),
       };
     } catch {
       // fall through to the throw
@@ -265,7 +266,11 @@ export async function runWithDefinition(
       return { success: false, output: '', error: 'Aborted', duration: Date.now() - startTime };
     }
 
-    // 2. Create the sandbox.
+    // 2. Create the sandbox. One sandbox serves the codegen run AND every judge
+    //    re-invocation of the runner (eval-helper.mjs); codex's shell-canary
+    //    memoization (~/.codex/agent-eval-canary.json, see codex/run.mjs) relies
+    //    on that shared lifetime — a sandbox-per-invocation change would make
+    //    every judge assertion re-pay the canary exec.
     sandbox = await createSandbox({
       timeout: options.timeout,
       runtime: 'node24',
diff --git a/packages/agent-eval/src/lib/results.test.ts b/packages/agent-eval/src/lib/results.test.ts
index fab9cc2..4f182e5 100644
--- a/packages/agent-eval/src/lib/results.test.ts
+++ b/packages/agent-eval/src/lib/results.test.ts
@@ -48,6 +48,23 @@ describe('results utilities', () => {
       expect(runData.transcript).toBe('{"role":"assistant","content":"Hello"}');
       expect(runData.outputContent?.eval).toBe('test output');
       expect(runData.outputContent?.scripts?.build).toBe('build output');
+      // No repair ran → field omitted, mirroring observedModel's omit-when-absent shape.
+      expect(runData.result.modelRepair).toBeUndefined();
+    });
+
+    it('copies modelRepair through like observedModel (shell-tool repair evidence)', () => {
+      const agentResult: AgentRunResult = {
+        success: true,
+        output: 'Agent output',
+        duration: 45000,
+        observedModel: 'openai/gpt-5.6-sol',
+        modelRepair: 'gpt-5.6-sol',
+      };
+
+      const runData = agentResultToEvalRunData(agentResult);
+
+      expect(runData.result.observedModel).toBe('openai/gpt-5.6-sol');
+      expect(runData.result.modelRepair).toBe('gpt-5.6-sol');
     });
 
     it('converts failed agent result', () => {
@@ -243,7 +260,7 @@ describe('results utilities', () => {
 
       const evals = [
         createEvalSummary('eval-1', [
-          { result: { status: 'passed', duration: 10, observedModel: 'vercel/openai/gpt-5.5' } },
+          { result: { status: 'passed', duration: 10, observedModel: 'vercel/openai/gpt-5.5', modelRepair: 'gpt-5.5' } },
         ]),
       ];
 
@@ -266,6 +283,9 @@ describe('results utilities', () => {
       expect(resultJson.model).toBe('vercel/openai/gpt-5.5');
       expect(resultJson.requestedModel).toBeUndefined();
       expect(resultJson.observedModel).toBe('vercel/openai/gpt-5.5');
+      // Repair evidence must survive into the persisted result.json — it is the
+      // removal signal for the codex shell-tool workaround (see codex/run.mjs).
+      expect(resultJson.modelRepair).toBe('gpt-5.5');
     });
 
     it('does not collide when script is named "eval"', () => {
diff --git a/packages/agent-eval/src/lib/runner.test.ts b/packages/agent-eval/src/lib/runner.test.ts
index 1e6913e..7eba470 100644
--- a/packages/agent-eval/src/lib/runner.test.ts
+++ b/packages/agent-eval/src/lib/runner.test.ts
@@ -621,6 +621,7 @@ describe('runExperiment', () => {
           output: 'Agent output',
           duration: 1000,
           observedModel: 'runtime-model',
+          modelRepair: 'runtime-model',
           testResult: { success: true, output: 'Test passed' },
           scriptsResults: {},
         }),
@@ -661,6 +662,8 @@ describe('runExperiment', () => {
         modelPolicy: 'native-default',
       }));
       expect(results.evals[0].runs[0].result.observedModel).toBe('runtime-model');
+      // Shell-tool repair evidence follows the same path as observedModel.
+      expect(results.evals[0].runs[0].result.modelRepair).toBe('runtime-model');
     });
 
     it('passes response-only validation mode to the agent', async () => {