From 4e8910b2dd6440eb30e38ff487b6fd8385f1069f Mon Sep 17 00:00:00 2001 From: qwen-code-dev-bot <269191875+qwen-code-dev-bot@users.noreply.github.com> Date: Thu, 23 Jul 2026 03:45:21 +0800 Subject: [PATCH] feat(cli): add an opt-in failure-taxonomy report categorizing tool-failure causes for unattended runs (#222) The run summary reports a single terminal reason and counts tool failures by name, but not why each call failed, so operators triaging a failed run had to read raw logs to tell a policy denial from a hook denial, an approval denial, a path escape, or a tool error. Add a privacy-safe, bounded failure-taxonomy report (opt-in --failure-taxonomy) that categorizes each failed tool call into a fixed taxonomy and reports the terminal reason class, in both the NDJSON headless stream and the plain-text path. The report is metadata only (category names, counts, terminal reason) and has no channel for error text, prompts, file contents, secrets, or raw tool payloads. --- src/agent.ts | 37 +++- src/headless-protocol.ts | 5 + src/index.ts | 26 +++ src/run-failure-taxonomy.ts | 122 +++++++++++++ .../integration/run-failure-taxonomy.test.ts | 163 ++++++++++++++++++ tests/unit/run-failure-taxonomy.test.ts | 104 +++++++++++ 6 files changed, 456 insertions(+), 1 deletion(-) create mode 100644 src/run-failure-taxonomy.ts create mode 100644 tests/integration/run-failure-taxonomy.test.ts create mode 100644 tests/unit/run-failure-taxonomy.test.ts diff --git a/src/agent.ts b/src/agent.ts index 8c68e27..db2fc1e 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -15,6 +15,7 @@ import { compactMessages, buildCompactedTranscript } from "./compaction.js"; import type { LoadedImage } from "./image-input.js"; import type { TurnImageCollector } from "./turn-checkpoint.js"; import type { BottleneckCollector } from "./run-bottleneck.js"; +import type { FailureTaxonomyCollector } from "./run-failure-taxonomy.js"; const MAX_ROUNDS = 30; @@ -54,6 +55,11 @@ export interface AgentOptions { // tool's execution wall-time and each approval gate's wait wall-time so the // caller can build a privacy-safe bottleneck report. Undefined ⇒ no collection. bottleneck?: BottleneckCollector; + // Optional failure-taxonomy collector. When present, the loop records the cause + // of each failed tool call (policy/hook/approval denial, path escape, tool + // error, unknown tool, folder-trust denial) so the caller can build a + // privacy-safe failure-taxonomy report. Undefined ⇒ no collection. + failureTaxonomy?: FailureTaxonomyCollector; } // Cumulative usage and cost reported after each round. `estimatedCostUsd` is an @@ -416,6 +422,7 @@ export async function runAgent( requestApproval, preToolUseHooks, opts.bottleneck, + opts.failureTaxonomy, opts.turnImages, ); sink.toolResult({ id: tc.id, name: tc.name, result, round }); @@ -457,10 +464,12 @@ async function executeToolCall( requestApproval: (name: string, args: Record) => Promise, preToolUseHooks: readonly PreToolUseHook[], bottleneck: BottleneckCollector | undefined, + failureTaxonomy: FailureTaxonomyCollector | undefined, turnImages?: TurnImageCollector, ): Promise { const tool = toolMap.get(tc.name); if (!tool) { + failureTaxonomy?.record("unknown_tool"); return { content: `Error: unknown tool "${tc.name}"`, isError: true }; } @@ -468,6 +477,7 @@ async function executeToolCall( // untrusted workspace fails closed for every mutating tool — yolo cannot widen // the boundary. Read-only tools (list/glob/grep/read) are always permitted. if (!mutatingAllowed && tool.category !== "read") { + failureTaxonomy?.record("folder_trust_denied"); return { content: folderTrustDenialMessage(), isError: true }; } @@ -485,6 +495,7 @@ async function executeToolCall( workspace: workspace.root, }); if (!decision.allowed) { + failureTaxonomy?.record("policy_denied"); return { content: policyDenialMessage(decision), isError: true }; } } @@ -507,6 +518,7 @@ async function executeToolCall( { cwd: workspace.root }, ); if (decision.denied) { + failureTaxonomy?.record("hook_denied"); return { content: decision.reason ? `Tool call denied by a user PreToolUse hook: ${decision.reason}` @@ -527,10 +539,14 @@ async function executeToolCall( const approved = await requestApproval(tc.name, parsed); bottleneck?.recordApproval(tc.name, Date.now() - approvalStartedAt); if (!approved) { + failureTaxonomy?.record("approval_denied"); return { content: "Tool execution denied by user", isError: true }; } } + // Detects a path escape so a thrown escape is categorized distinctly from a + // generic tool error in the failure taxonomy. + let pathEscape = false; try { const parsed = JSON.parse(tc.arguments); // Capture the file's pre-image before a mutating-file tool overwrites it, so @@ -544,15 +560,34 @@ async function executeToolCall( /* path escape is the tool's own error to surface */ } } + if (failureTaxonomy && tool.category === "mutate-file" && typeof parsed.path === "string") { + try { + workspace.resolveSafe(parsed.path); + } catch { + pathEscape = true; + } + } // Time only the tool's own execution (approval/policy/hook gates are measured // or rejected above); recorded even when the tool throws via the finally. const executeStartedAt = Date.now(); try { - return await tool.execute(parsed, workspace); + const result = await tool.execute(parsed, workspace); + // A tool that reports an error result without throwing is still a failed + // call; categorize it (a detected path escape takes precedence). + if (result.isError) { + failureTaxonomy?.record(pathEscape ? "path_escape" : "tool_error"); + } + return result; + } catch (err: unknown) { + failureTaxonomy?.record(pathEscape ? "path_escape" : "tool_error"); + const msg = err instanceof Error ? err.message : String(err); + return { content: `Tool error: ${msg}`, isError: true }; } finally { bottleneck?.recordTool(tc.name, Date.now() - executeStartedAt); } } catch (err: unknown) { + // A failure outside the tool's own execution (e.g. malformed arguments). + failureTaxonomy?.record("tool_error"); const msg = err instanceof Error ? err.message : String(err); return { content: `Tool error: ${msg}`, isError: true }; } diff --git a/src/headless-protocol.ts b/src/headless-protocol.ts index 75abefe..add5b6b 100644 --- a/src/headless-protocol.ts +++ b/src/headless-protocol.ts @@ -2,6 +2,7 @@ import type { AgentSink } from "./agent.js"; import type { ToolResult } from "./tools.js"; import type { RunSummary } from "./run-summary.js"; import type { BottleneckReport } from "./run-bottleneck.js"; +import type { FailureTaxonomyReport } from "./run-failure-taxonomy.js"; import { redactSecrets } from "./permission-impact.js"; // A stable, versioned newline-delimited JSON protocol for core run lifecycle @@ -80,6 +81,10 @@ export type HeadlessEvent = // report, emitted just before the terminal `complete`. Metadata only: tool // names, wall-time, and counts — never prompt/tool/file content. | { type: "bottleneck"; bottleneck: BottleneckReport } + // Opt-in (`--failure-taxonomy`) privacy-safe failure-taxonomy report, emitted + // just before the terminal `complete`. Metadata only: failure-cause categories, + // counts, and the terminal reason — never error text or prompt/tool/file content. + | { type: "failure_taxonomy"; failureTaxonomy: FailureTaxonomyReport } | { type: "complete"; ok: boolean; exitCode: number; rounds: number; reason: string }; export type HeadlessRecord = HeadlessEvent & { diff --git a/src/index.ts b/src/index.ts index 54244e9..79ca117 100644 --- a/src/index.ts +++ b/src/index.ts @@ -154,6 +154,7 @@ import { HeadlessWriter, createHeadlessSink, startEvent } from "./headless-proto import { redactSecrets, redactHomePath } from "./permission-impact.js"; import { buildRunSummary, formatRunSummary } from "./run-summary.js"; import { createBottleneckCollector, formatBottleneckReport } from "./run-bottleneck.js"; +import { createFailureTaxonomyCollector, formatFailureTaxonomyReport } from "./run-failure-taxonomy.js"; import { loadImageAttachments, imageRef } from "./image-input.js"; import type { LoadedImage } from "./image-input.js"; import { createTools } from "./tools.js"; @@ -378,6 +379,10 @@ program "--bottleneck", "Print a privacy-safe tool/approval wall-time bottleneck report for the run (unattended use)", ) + .option( + "--failure-taxonomy", + "Print a privacy-safe failure-cause taxonomy report for the run (unattended use)", + ) .option( "--budget ", "Spend budget in USD; stop before further provider calls once the estimated cost reaches it (also honors OMC_SPEND_BUDGET_USD)", @@ -1942,6 +1947,7 @@ program const startedAt = Date.now(); const turnImages = new TurnImageCollector(); const bottleneck = opts.bottleneck ? createBottleneckCollector() : null; + const failureTaxonomy = opts.failureTaxonomy ? createFailureTaxonomyCollector() : null; let result: AgentResult; try { result = await runAgent(opts.prompt, existingMessages, { @@ -1958,6 +1964,7 @@ program turnImages, preToolUseHooks, bottleneck: bottleneck?.collector, + failureTaxonomy: failureTaxonomy?.collector, }); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); @@ -1965,6 +1972,12 @@ program if (bottleneck) { writer.emit({ type: "bottleneck", bottleneck: bottleneck.build(Date.now() - startedAt) }); } + if (failureTaxonomy) { + writer.emit({ + type: "failure_taxonomy", + failureTaxonomy: failureTaxonomy.build(Date.now() - startedAt, "error"), + }); + } if (opts.summary) { writer.emit({ type: "summary", @@ -2012,6 +2025,12 @@ program if (bottleneck) { writer.emit({ type: "bottleneck", bottleneck: bottleneck.build(Date.now() - startedAt) }); } + if (failureTaxonomy) { + writer.emit({ + type: "failure_taxonomy", + failureTaxonomy: failureTaxonomy.build(Date.now() - startedAt, result.reason), + }); + } writer.emit({ type: "complete", ok: result.ok, @@ -2025,6 +2044,7 @@ program const startedAt = Date.now(); const turnImages = new TurnImageCollector(); const bottleneck = opts.bottleneck ? createBottleneckCollector() : null; + const failureTaxonomy = opts.failureTaxonomy ? createFailureTaxonomyCollector() : null; const result = await runAgent(opts.prompt, existingMessages, { config, workspace, @@ -2038,6 +2058,7 @@ program turnImages, preToolUseHooks, bottleneck: bottleneck?.collector, + failureTaxonomy: failureTaxonomy?.collector, }); sealSession(); recordTurnCheckpoint(turnImages); @@ -2065,6 +2086,11 @@ program if (bottleneck) { process.stdout.write("\n" + formatBottleneckReport(bottleneck.build(Date.now() - startedAt)) + "\n"); } + if (failureTaxonomy) { + process.stdout.write( + "\n" + formatFailureTaxonomyReport(failureTaxonomy.build(Date.now() - startedAt, result.reason)) + "\n", + ); + } process.exit(exitCode); } else { // Interactive REPL diff --git a/src/run-failure-taxonomy.ts b/src/run-failure-taxonomy.ts new file mode 100644 index 0000000..5656423 --- /dev/null +++ b/src/run-failure-taxonomy.ts @@ -0,0 +1,122 @@ +// A privacy-safe failure-taxonomy report for unattended (`-p`) runs. Where the run +// summary (#40) reports a single terminal reason and counts tool failures by tool +// name, this report categorizes each tool failure by CAUSE — why it failed — into a +// fixed, bounded taxonomy (policy denial, hook denial, approval denial, path +// escape, tool error, unknown tool, folder-trust denial), plus the terminal reason +// class. It is opt-in (`--failure-taxonomy`) and metadata-only: it carries category +// names and counts and has no channel for error text, prompts, file contents, +// secrets, or raw tool payloads. The schema is stable and deterministic so CI can +// retain and diff it. + +export const FAILURE_TAXONOMY_SCHEMA = "oh-my-cli.failure-taxonomy"; +export const FAILURE_TAXONOMY_VERSION = 1; + +// The fixed failure taxonomy, in canonical report order. Every tool failure is +// bucketed into exactly one category; anything unrecognized falls into "other" so +// the output stays bounded and stable even if a new failure path appears. +export const FAILURE_CATEGORIES = [ + "policy_denied", + "hook_denied", + "approval_denied", + "folder_trust_denied", + "path_escape", + "unknown_tool", + "tool_error", + "other", +] as const; + +export type FailureCategory = (typeof FAILURE_CATEGORIES)[number]; + +const CATEGORY_SET: ReadonlySet = new Set(FAILURE_CATEGORIES); + +export interface FailureTaxonomyReport { + schema: typeof FAILURE_TAXONOMY_SCHEMA; + v: typeof FAILURE_TAXONOMY_VERSION; + // Total run wall-time, for context. + elapsedMs: number; + // Terminal reason class (e.g. completed, provider_error, max_rounds, + // budget_reached) — the run-level outcome, distinct from per-tool failure causes. + reason: string; + // Total number of failed tool calls across the run. + totalFailures: number; + // Category → count, in canonical taxonomy order, including only non-zero + // categories. Empty when no tool call failed. + byCategory: Record; +} + +// Accumulates per-cause tool-failure counts across a run. +export interface FailureTaxonomyCollector { + record(category: FailureCategory): void; +} + +export function createFailureTaxonomyCollector(): { + collector: FailureTaxonomyCollector; + build: (elapsedMs: number, reason: string) => FailureTaxonomyReport; +} { + const counts = new Map(); + const collector: FailureTaxonomyCollector = { + record(category) { + const key: FailureCategory = CATEGORY_SET.has(category) ? category : "other"; + counts.set(key, (counts.get(key) ?? 0) + 1); + }, + }; + const build = (elapsedMs: number, reason: string): FailureTaxonomyReport => + buildFailureTaxonomyReport(elapsedMs, reason, counts); + return { collector, build }; +} + +function buildFailureTaxonomyReport( + elapsedMs: number, + reason: string, + counts: Map, +): FailureTaxonomyReport { + const byCategory: Record = {}; + let totalFailures = 0; + // Emit in canonical taxonomy order, only non-zero categories, for stable output. + for (const category of FAILURE_CATEGORIES) { + const n = counts.get(category) ?? 0; + if (n > 0) { + byCategory[category] = n; + totalFailures += n; + } + } + return { + schema: FAILURE_TAXONOMY_SCHEMA, + v: FAILURE_TAXONOMY_VERSION, + elapsedMs: Math.max(0, Math.round(elapsedMs)), + reason: sanitizeReason(reason), + totalFailures, + byCategory, + }; +} + +// The terminal reason is one of a small fixed set produced by the agent loop; bound +// and strip control characters defensively so the report can never carry a runaway +// or control-laced value. +function sanitizeReason(reason: string): string { + const cleaned = reason.replace(/[\u0000-\u001f\u007f]+/g, "_").trim(); + return cleaned.length <= 64 ? cleaned : cleaned.slice(0, 64); +} + +function formatElapsed(ms: number): string { + return `${(ms / 1000).toFixed(1)}s`; +} + +// A deterministic, human-readable rendering of the report. It contains only +// category names, counts, and the terminal reason — there is no channel for error +// text, prompt/tool/file content, or secrets. +export function formatFailureTaxonomyReport(report: FailureTaxonomyReport): string { + const lines: string[] = []; + lines.push(`Failure taxonomy (${report.schema} v${report.v})`); + lines.push(` elapsed: ${formatElapsed(report.elapsedMs)}`); + lines.push(` terminal: ${report.reason}`); + if (report.totalFailures === 0) { + lines.push(" failures: 0 (none)"); + } else { + lines.push(` failures: ${report.totalFailures}`); + for (const category of Object.keys(report.byCategory)) { + lines.push(` ${category.padEnd(20)} ${report.byCategory[category]}`); + } + } + return lines.join("\n"); +} diff --git a/tests/integration/run-failure-taxonomy.test.ts b/tests/integration/run-failure-taxonomy.test.ts new file mode 100644 index 0000000..702fddf --- /dev/null +++ b/tests/integration/run-failure-taxonomy.test.ts @@ -0,0 +1,163 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; +import { createFakeServer } from "../fake-provider.js"; +import type { FakeServer } from "../fake-provider.js"; +import { parseHeadlessStream } from "../../src/headless-protocol.js"; +import { spawn } from "node:child_process"; +import path from "node:path"; +import fs from "node:fs"; +import os from "node:os"; + +function runCli( + args: string[], + env: Record, + timeoutMs = 20_000, +): Promise<{ stdout: string; stderr: string; code: number | null }> { + return new Promise((resolve, reject) => { + const cliPath = path.resolve(import.meta.dirname, "../../dist/index.js"); + const proc = spawn("node", [cliPath, ...args], { + env: { ...process.env, ...env }, + timeout: timeoutMs, + }); + let stdout = ""; + let stderr = ""; + proc.stdout.on("data", (d) => { stdout += d; }); + proc.stderr.on("data", (d) => { stderr += d; }); + proc.on("close", (code) => resolve({ stdout, stderr, code })); + proc.on("error", reject); + }); +} + +describe("Integration: failure taxonomy (--failure-taxonomy)", () => { + let server: FakeServer; + let tmpDir: string; + let sessionDir: string; + let baseEnv: Record; + + beforeAll(async () => { + server = await createFakeServer(); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "oh-my-cli-failure-tax-")); + sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), "oh-my-cli-failure-tax-sess-")); + baseEnv = { + OPENAI_API_KEY: "fake-key", + OPENAI_BASE_URL: server.url, + OPENAI_MODEL: "fake-model", + HOME: sessionDir, + }; + }); + + afterAll(async () => { + await server.close(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + fs.rmSync(sessionDir, { recursive: true, force: true }); + }); + + beforeEach(() => { + server.requests.length = 0; + }); + + it("prints a failure taxonomy after a text run only when requested", async () => { + server.setResponses([ + { type: "tool_calls", toolCalls: [{ id: "call_1", name: "nonexistent_tool", arguments: "{}" }] }, + { type: "text", content: "done" }, + ]); + const withReport = await runCli( + ["-p", "do work", "--failure-taxonomy", "--approval-mode", "yolo", "--workspace", tmpDir], + baseEnv, + ); + expect(withReport.code).toBe(0); + expect(withReport.stdout).toContain("Failure taxonomy (oh-my-cli.failure-taxonomy v1)"); + expect(withReport.stdout).toContain("unknown_tool"); + + server.setResponses([ + { type: "tool_calls", toolCalls: [{ id: "call_1", name: "nonexistent_tool", arguments: "{}" }] }, + { type: "text", content: "done" }, + ]); + const plain = await runCli( + ["-p", "do work", "--approval-mode", "yolo", "--workspace", tmpDir], + baseEnv, + ); + expect(plain.code).toBe(0); + expect(plain.stdout).not.toContain("Failure taxonomy (oh-my-cli.failure-taxonomy v1)"); + }); + + it("categorizes an unknown-tool failure in the JSON stream", async () => { + server.setResponses([ + { type: "tool_calls", toolCalls: [{ id: "call_1", name: "nonexistent_tool", arguments: "{}" }] }, + { type: "text", content: "done" }, + ]); + const r = await runCli( + ["-p", "do work", "--output", "json", "--failure-taxonomy", "--approval-mode", "yolo", "--workspace", tmpDir], + baseEnv, + ); + expect(r.code).toBe(0); + + const recs = parseHeadlessStream(r.stdout); + const ft = recs.find((x) => x.type === "failure_taxonomy"); + expect(ft).toBeDefined(); + if (ft?.type === "failure_taxonomy") { + const report = ft.failureTaxonomy; + expect(report.schema).toBe("oh-my-cli.failure-taxonomy"); + expect(report.v).toBe(1); + expect(report.reason).toBe("completed"); + expect(report.totalFailures).toBe(1); + expect(report.byCategory).toEqual({ unknown_tool: 1 }); + } + }); + + it("categorizes a path-escape write failure distinctly from a generic tool error", async () => { + server.setResponses([ + { + type: "tool_calls", + toolCalls: [{ id: "call_1", name: "write", arguments: JSON.stringify({ path: "../../escape.txt", content: "hi" }) }], + }, + { type: "text", content: "done" }, + ]); + const r = await runCli( + ["-p", "do work", "--output", "json", "--failure-taxonomy", "--approval-mode", "yolo", "--workspace", tmpDir], + baseEnv, + ); + expect(r.code).toBe(0); + + const recs = parseHeadlessStream(r.stdout); + const ft = recs.find((x) => x.type === "failure_taxonomy"); + expect(ft).toBeDefined(); + if (ft?.type === "failure_taxonomy") { + expect(ft.failureTaxonomy.byCategory).toEqual({ path_escape: 1 }); + } + // The escape write must not have created a file outside the workspace. + expect(fs.existsSync(path.join(tmpDir, "../../escape.txt"))).toBe(false); + }); + + it("keeps secret-shaped tool arguments out of the failure taxonomy report", async () => { + const secret = "sk-abcdefghijklmnopqrstuvwxyz1234567890"; + server.setResponses([ + { + type: "tool_calls", + toolCalls: [{ id: "call_1", name: "nonexistent_tool", arguments: JSON.stringify({ apiKey: secret }) }], + }, + { type: "text", content: "done" }, + ]); + const r = await runCli( + ["-p", "do work", "--output", "json", "--failure-taxonomy", "--approval-mode", "yolo", "--workspace", tmpDir], + baseEnv, + ); + expect(r.code).toBe(0); + + const recs = parseHeadlessStream(r.stdout); + const ft = recs.find((x) => x.type === "failure_taxonomy"); + expect(ft).toBeDefined(); + // The report is metadata only: the secret-bearing argument never appears in it. + expect(JSON.stringify(ft)).not.toContain(secret); + }); + + it("reports zero failures for a clean run with no tool failures", async () => { + server.setResponses([{ type: "text", content: "just text" }]); + const r = await runCli( + ["-p", "hello", "--failure-taxonomy", "--workspace", tmpDir], + baseEnv, + ); + expect(r.code).toBe(0); + expect(r.stdout).toContain("Failure taxonomy (oh-my-cli.failure-taxonomy v1)"); + expect(r.stdout).toContain("failures: 0 (none)"); + }); +}); diff --git a/tests/unit/run-failure-taxonomy.test.ts b/tests/unit/run-failure-taxonomy.test.ts new file mode 100644 index 0000000..674ebdb --- /dev/null +++ b/tests/unit/run-failure-taxonomy.test.ts @@ -0,0 +1,104 @@ +import { describe, it, expect } from "vitest"; +import { + createFailureTaxonomyCollector, + formatFailureTaxonomyReport, + FAILURE_CATEGORIES, + FAILURE_TAXONOMY_SCHEMA, + FAILURE_TAXONOMY_VERSION, +} from "../../src/run-failure-taxonomy.js"; + +describe("createFailureTaxonomyCollector / build", () => { + it("counts each failure cause and reports the total and terminal reason", () => { + const { collector, build } = createFailureTaxonomyCollector(); + collector.record("policy_denied"); + collector.record("policy_denied"); + collector.record("tool_error"); + const report = build(5000, "provider_error"); + + expect(report.schema).toBe(FAILURE_TAXONOMY_SCHEMA); + expect(report.v).toBe(FAILURE_TAXONOMY_VERSION); + expect(report.elapsedMs).toBe(5000); + expect(report.reason).toBe("provider_error"); + expect(report.totalFailures).toBe(3); + expect(report.byCategory).toEqual({ policy_denied: 2, tool_error: 1 }); + }); + + it("emits byCategory in canonical taxonomy order regardless of insertion order", () => { + const { collector, build } = createFailureTaxonomyCollector(); + collector.record("tool_error"); + collector.record("approval_denied"); + collector.record("policy_denied"); + const report = build(100, "completed"); + // Canonical order: policy_denied before approval_denied before tool_error. + expect(Object.keys(report.byCategory)).toEqual(["policy_denied", "approval_denied", "tool_error"]); + }); + + it("buckets an unrecognized category into 'other'", () => { + const { collector, build } = createFailureTaxonomyCollector(); + // Cast to exercise the defensive path for a category outside the fixed set. + collector.record("brand_new_failure" as never); + const report = build(100, "completed"); + expect(report.byCategory).toEqual({ other: 1 }); + expect(report.totalFailures).toBe(1); + }); + + it("includes only non-zero categories", () => { + const { collector, build } = createFailureTaxonomyCollector(); + collector.record("hook_denied"); + const report = build(100, "completed"); + expect(Object.keys(report.byCategory)).toEqual(["hook_denied"]); + for (const category of FAILURE_CATEGORIES) { + if (category !== "hook_denied") expect(report.byCategory[category]).toBeUndefined(); + } + }); + + it("sanitizes control characters out of the terminal reason", () => { + const { build } = createFailureTaxonomyCollector(); + const report = build(100, "bad\u0000reason\u001f"); + expect(report.reason).toBe("bad_reason_"); + expect(report.reason).not.toMatch(/[\u0000-\u001f\u007f]/); + }); + + it("clamps a negative elapsed time to zero", () => { + const { build } = createFailureTaxonomyCollector(); + expect(build(-50, "completed").elapsedMs).toBe(0); + }); + + it("reports zero failures with an empty byCategory when nothing failed", () => { + const { build } = createFailureTaxonomyCollector(); + const report = build(1000, "completed"); + expect(report.totalFailures).toBe(0); + expect(report.byCategory).toEqual({}); + }); +}); + +describe("formatFailureTaxonomyReport", () => { + it("renders a human-readable taxonomy with per-category counts", () => { + const { collector, build } = createFailureTaxonomyCollector(); + collector.record("policy_denied"); + collector.record("policy_denied"); + collector.record("path_escape"); + const text = formatFailureTaxonomyReport(build(12345, "max_rounds")); + expect(text).toContain(`Failure taxonomy (${FAILURE_TAXONOMY_SCHEMA} v${FAILURE_TAXONOMY_VERSION})`); + expect(text).toContain("elapsed: 12.3s"); + expect(text).toContain("terminal: max_rounds"); + expect(text).toContain("failures: 3"); + expect(text).toContain("policy_denied"); + expect(text).toContain("2"); + expect(text).toContain("path_escape"); + }); + + it("renders an explicit no-failures line when nothing failed", () => { + const { build } = createFailureTaxonomyCollector(); + const text = formatFailureTaxonomyReport(build(1000, "completed")); + expect(text).toContain("failures: 0 (none)"); + }); + + it("carries no channel for error text or content (metadata only)", () => { + const { collector, build } = createFailureTaxonomyCollector(); + collector.record("tool_error"); + const text = formatFailureTaxonomyReport(build(100, "completed")); + expect(text).not.toContain("secret"); + expect(text).not.toContain("Error:"); + }); +});