diff --git a/packages/processor/src/__tests__/shared.test.ts b/packages/processor/src/__tests__/shared.test.ts index 5363a02..200ae5e 100644 --- a/packages/processor/src/__tests__/shared.test.ts +++ b/packages/processor/src/__tests__/shared.test.ts @@ -6,7 +6,9 @@ import { buildInvestigateJsonRepairPrompt, buildRevalidateJsonRepairPrompt, classifyQuotaError, + formatMissingCodexBinaryError, formatJsonRepairFailureDebugText, + isMissingBinaryError, isTransientError, isUsingAiGateway, parseInvestigateResults, @@ -28,6 +30,11 @@ describe("isTransientError", () => { it("doesn't flag obvious permanent errors", () => { expect(isTransientError("ENOENT no such file")).toBe(false); + expect( + isTransientError( + "spawn /Users/me/project/node_modules/@openai/codex/vendor/aarch64-apple-darwin/codex/codex ENOENT", + ), + ).toBe(false); expect(isTransientError("invalid api key")).toBe(false); }); @@ -42,6 +49,49 @@ describe("isTransientError", () => { }); }); +describe("isMissingBinaryError", () => { + const vendorPath = + "/Users/me/project/node_modules/@openai/codex/vendor/aarch64-apple-darwin/codex/codex"; + + it("matches vendored Codex spawn ENOENT failures", () => { + expect(isMissingBinaryError(`spawn ${vendorPath} ENOENT`)).toBe(true); + expect( + isMissingBinaryError( + `Codex Exec exited with signal SIGKILL\nError: spawn ${vendorPath} ENOENT`, + ), + ).toBe(true); + }); + + it("does not match SIGKILL-only text or unrelated ENOENT failures", () => { + expect(isMissingBinaryError("Codex Exec exited with signal SIGKILL")).toBe(false); + expect(isMissingBinaryError("spawn /tmp/codex ENOENT")).toBe(false); + expect(isMissingBinaryError("ENOENT no such file")).toBe(false); + }); + + it("does not match transient or quota failures", () => { + expect(isMissingBinaryError("HTTP 429 too many requests")).toBe(false); + expect(isMissingBinaryError("ETIMEDOUT")).toBe(false); + expect(isMissingBinaryError("timeout waiting for Codex")).toBe(false); + expect(isMissingBinaryError("You've hit your usage limit.")).toBe(false); + expect(isMissingBinaryError("HTTP 429 insufficient_quota")).toBe(false); + }); +}); + +describe("formatMissingCodexBinaryError", () => { + it("includes the failing path and macOS quarantine remediation", () => { + const vendorPath = + "/Users/me/project/node_modules/@openai/codex/vendor/aarch64-apple-darwin/codex/codex"; + const formatted = formatMissingCodexBinaryError(`spawn ${vendorPath} ENOENT`); + + expect(formatted).toContain(vendorPath); + expect(formatted).toContain("Gatekeeper"); + expect(formatted).toContain("quarantine"); + expect(formatted).toContain("reinstall dependencies"); + expect(formatted).toContain("xattr -dr com.apple.quarantine"); + expect(formatted).not.toContain("produced no result"); + }); +}); + describe("classifyQuotaError", () => { // Strings in this block were extracted from the actual platform binaries // shipped via @anthropic-ai/claude-agent-sdk-darwin-arm64 (claude) and diff --git a/packages/processor/src/agents/codex-sdk.ts b/packages/processor/src/agents/codex-sdk.ts index 48a25f9..11a5f7a 100644 --- a/packages/processor/src/agents/codex-sdk.ts +++ b/packages/processor/src/agents/codex-sdk.ts @@ -19,7 +19,9 @@ import { buildRevalidateJsonRepairPrompt, buildRevalidatePrompt, classifyQuotaError, + formatMissingCodexBinaryError, formatJsonRepairFailureDebugText, + isMissingBinaryError, isTransientError, jsonRepairFailureError, MAX_ATTEMPTS, @@ -395,6 +397,32 @@ function readStderrTail(p: string | null): string | undefined { } } +function firstMissingCodexBinaryError( + ...messages: Array +): string | undefined { + return messages.find( + (msg): msg is string => typeof msg === "string" && isMissingBinaryError(msg), + ); +} + +function formatCodexNoResultError(params: { + resultKind: "result" | "revalidation result"; + lastError: string; + codexStderr?: string; +}): string { + const { resultKind, lastError, codexStderr } = params; + const combinedError = [lastError, codexStderr].filter(Boolean).join("\n"); + if (isMissingBinaryError(combinedError)) { + return formatMissingCodexBinaryError(combinedError); + } + + const stderrSuffix = codexStderr ? ` Stderr tail: ${codexStderr.slice(0, 800)}` : ""; + return ( + `Codex SDK produced no ${resultKind} after ${MAX_ATTEMPTS} attempt(s). ` + + `Last error: ${lastError || "(none captured)"}.${stderrSuffix}` + ); +} + function cleanupStderrLog(p: string | null): void { if (!p) return; try { @@ -780,7 +808,8 @@ export class CodexAgentSdkPlugin implements AgentPlugin { // Codex frequently exits silent on quota/auth errors — the SDK // returns an empty `response.completed` and the error only lives in // the wrapper-captured stderr. Read it BEFORE deciding to retry so - // a quota miss bails fast instead of burning all 3 attempts. + // a quota miss or missing binary bails fast instead of burning all + // 3 attempts. const stderrPeek = readStderrTail(invocation.stderrLog) ?? ""; const quotaSourceEarly = classifyQuotaError(lastError, "codex") || classifyQuotaError(stderrPeek, "codex"); @@ -792,6 +821,12 @@ export class CodexAgentSdkPlugin implements AgentPlugin { lastError || stderrPeek || "codex silent quota exit", ); } + const missingBinaryEarly = firstMissingCodexBinaryError(lastError, stderrPeek); + if (missingBinaryEarly) { + if (stderrPeek) sdkMeta.codexStderr = stderrPeek; + lastError = missingBinaryEarly; + break; + } // Silent-failure retry: gateway returned response.completed with 0 // tokens and no agent_message — codex CLI exited clean, but no work // happened. Retry as if it were a transient error. @@ -844,6 +879,9 @@ export class CodexAgentSdkPlugin implements AgentPlugin { if (quotaSourceFinal) { throw new QuotaExhaustedError(quotaSourceFinal, stderrTail); } + if (isMissingBinaryError(stderrTail)) { + throw new Error(formatMissingCodexBinaryError(stderrTail)); + } } // Empty-result check runs BEFORE parse so a silent failure throws @@ -851,12 +889,12 @@ export class CodexAgentSdkPlugin implements AgentPlugin { // with a misleading "no JSON in response" message — and the finally // block below cleans up regardless of which path we exit through. if (!resultText) { - const stderrSuffix = sdkMeta.codexStderr - ? ` Stderr tail: ${sdkMeta.codexStderr.slice(0, 800)}` - : ""; throw new Error( - `Codex SDK produced no result after ${MAX_ATTEMPTS} attempt(s). ` + - `Last error: ${lastError || "(none captured)"}.${stderrSuffix}`, + formatCodexNoResultError({ + resultKind: "result", + lastError, + codexStderr: sdkMeta.codexStderr, + }), ); } @@ -1053,7 +1091,7 @@ export class CodexAgentSdkPlugin implements AgentPlugin { if (agentMessages.length > 0) break; // See investigate() — read stderr first so we bail fast on quota - // misses instead of burning all 3 attempts. + // misses or missing binaries instead of burning all 3 attempts. const stderrPeek = readStderrTail(invocation.stderrLog) ?? ""; const quotaSourceEarly = classifyQuotaError(lastError, "codex") || classifyQuotaError(stderrPeek, "codex"); @@ -1064,6 +1102,12 @@ export class CodexAgentSdkPlugin implements AgentPlugin { lastError || stderrPeek || "codex silent quota exit", ); } + const missingBinaryEarly = firstMissingCodexBinaryError(lastError, stderrPeek); + if (missingBinaryEarly) { + if (stderrPeek) sdkMeta.codexStderr = stderrPeek; + lastError = missingBinaryEarly; + break; + } // Silent-failure retry: gateway returned response.completed with 0 // tokens and no agent_message — codex CLI exited clean, but no work // happened. Retry as if it were a transient error. @@ -1104,17 +1148,20 @@ export class CodexAgentSdkPlugin implements AgentPlugin { if (quotaSourceFinal) { throw new QuotaExhaustedError(quotaSourceFinal, stderrTail); } + if (isMissingBinaryError(stderrTail)) { + throw new Error(formatMissingCodexBinaryError(stderrTail)); + } } // Empty-result throw before parse so silent failures don't crash inside // parseRevalidateVerdicts and bypass the cleanup in finally. if (!resultText) { - const stderrSuffix = sdkMeta.codexStderr - ? ` Stderr tail: ${sdkMeta.codexStderr.slice(0, 800)}` - : ""; throw new Error( - `Codex SDK produced no revalidation result after ${MAX_ATTEMPTS} attempt(s). ` + - `Last error: ${lastError || "(none captured)"}.${stderrSuffix}`, + formatCodexNoResultError({ + resultKind: "revalidation result", + lastError, + codexStderr: sdkMeta.codexStderr, + }), ); } diff --git a/packages/processor/src/agents/shared.ts b/packages/processor/src/agents/shared.ts index 7c9d5a3..c748e24 100644 --- a/packages/processor/src/agents/shared.ts +++ b/packages/processor/src/agents/shared.ts @@ -200,6 +200,53 @@ export class QuotaExhaustedError extends Error { } } +const CODEX_VENDOR_BINARY_RE = + /@openai[\\/]codex[\\/]vendor[\\/][\s\S]*?[\\/]codex[\\/]codex(?:\.exe)?/i; + +function extractMissingCodexBinaryPath(msg: string): string | undefined { + const spawnMatch = msg.match( + /\bspawn\s+(.+?@openai[\\/]codex[\\/]vendor[\\/].+?[\\/]codex[\\/]codex(?:\.exe)?)\s+(?:ENOENT|$)/i, + ); + if (spawnMatch?.[1]) return spawnMatch[1].trim(); + + const pathMatch = msg.match( + /((?:[A-Za-z]:)?[^"'`\n\r]*?@openai[\\/]codex[\\/]vendor[\\/][^"'`\n\r]*?[\\/]codex[\\/]codex(?:\.exe)?)/i, + ); + return pathMatch?.[1]?.trim(); +} + +/** + * Detect the local failure mode where the vendored @openai/codex executable + * is gone by the time the SDK tries to spawn it. macOS Gatekeeper/XProtect + * can kill the binary first (SIGKILL) and quarantine/remove it, but SIGKILL + * alone is not enough to classify this: we require the vendored binary path + * plus ENOENT/no-such-file so ordinary process kills stay in the existing + * retry/error path. + */ +export function isMissingBinaryError(msg: string): boolean { + if (!msg) return false; + const hasMissingFileSignature = + /\bENOENT\b/i.test(msg) || /no such file or directory/i.test(msg); + return hasMissingFileSignature && CODEX_VENDOR_BINARY_RE.test(msg); +} + +export function formatMissingCodexBinaryError(msg: string): string { + const binaryPath = extractMissingCodexBinaryPath(msg); + const missingLine = binaryPath ? `\n\nMissing binary: ${binaryPath}` : ""; + const rawLine = msg ? `\n\nOriginal error: ${msg.slice(0, 1000)}` : ""; + + return ( + `Codex could not start because the bundled @openai/codex executable is missing or no longer spawnable.` + + missingLine + + `\n\nOn macOS, Gatekeeper may have killed or quarantined the vendored Codex binary. ` + + `That often appears first as "Codex Exec exited with signal SIGKILL", then the next spawn fails with ENOENT.` + + `\n\nTo fix it, reinstall dependencies so @openai/codex restores the vendored binary. ` + + `If you trust the installed package, you can also clear the macOS quarantine attribute from the restored package or binary with: ` + + `xattr -dr com.apple.quarantine ` + + rawLine + ); +} + /** * Vercel AI Gateway endpoints — duplicated from preflight.ts on purpose: * processor doesn't depend on the deepsec CLI package. Keep these in sync