feat(cli): add an opt-in failure-taxonomy report categorizing tool-failure causes for unattended runs (#222)

src/agent.ts

@@ -15,6 +15,7 @@ import { compactMessages, buildCompactedTranscript } from "./compaction.js";
 import type { LoadedImage } from "./image-input.js";
 import type { TurnImageCollector } from "./turn-checkpoint.js";
 import type { BottleneckCollector } from "./run-bottleneck.js";
+import type { FailureTaxonomyCollector } from "./run-failure-taxonomy.js";
 
 const MAX_ROUNDS = 30;
 
@@ -54,6 +55,11 @@ export interface AgentOptions {
   // tool's execution wall-time and each approval gate's wait wall-time so the
   // caller can build a privacy-safe bottleneck report. Undefined ⇒ no collection.
   bottleneck?: BottleneckCollector;
+  // Optional failure-taxonomy collector. When present, the loop records the cause
+  // of each failed tool call (policy/hook/approval denial, path escape, tool
+  // error, unknown tool, folder-trust denial) so the caller can build a
+  // privacy-safe failure-taxonomy report. Undefined ⇒ no collection.
+  failureTaxonomy?: FailureTaxonomyCollector;
 }
 
 // Cumulative usage and cost reported after each round. `estimatedCostUsd` is an
@@ -416,6 +422,7 @@ export async function runAgent(
         requestApproval,
         preToolUseHooks,
         opts.bottleneck,
+        opts.failureTaxonomy,
         opts.turnImages,
       );
       sink.toolResult({ id: tc.id, name: tc.name, result, round });
@@ -457,17 +464,20 @@ async function executeToolCall(
   requestApproval: (name: string, args: Record<string, unknown>) => Promise<boolean>,
   preToolUseHooks: readonly PreToolUseHook[],
   bottleneck: BottleneckCollector | undefined,
+  failureTaxonomy: FailureTaxonomyCollector | undefined,
   turnImages?: TurnImageCollector,
 ): Promise<ToolResult> {
   const tool = toolMap.get(tc.name);
   if (!tool) {
+    failureTaxonomy?.record("unknown_tool");
     return { content: `Error: unknown tool "${tc.name}"`, isError: true };
   }
 
   // Folder-trust enforcement runs first and regardless of approval mode, so an
   // untrusted workspace fails closed for every mutating tool — yolo cannot widen
   // the boundary. Read-only tools (list/glob/grep/read) are always permitted.
   if (!mutatingAllowed && tool.category !== "read") {
+    failureTaxonomy?.record("folder_trust_denied");
     return { content: folderTrustDenialMessage(), isError: true };
   }
 
@@ -485,6 +495,7 @@ async function executeToolCall(
       workspace: workspace.root,
     });
     if (!decision.allowed) {
+      failureTaxonomy?.record("policy_denied");
       return { content: policyDenialMessage(decision), isError: true };
     }
   }
@@ -507,6 +518,7 @@ async function executeToolCall(
       { cwd: workspace.root },
     );
     if (decision.denied) {
+      failureTaxonomy?.record("hook_denied");
       return {
         content: decision.reason
           ? `Tool call denied by a user PreToolUse hook: ${decision.reason}`
@@ -527,10 +539,14 @@ async function executeToolCall(
     const approved = await requestApproval(tc.name, parsed);
     bottleneck?.recordApproval(tc.name, Date.now() - approvalStartedAt);
     if (!approved) {
+      failureTaxonomy?.record("approval_denied");
       return { content: "Tool execution denied by user", isError: true };
     }
   }
 
+  // Detects a path escape so a thrown escape is categorized distinctly from a
+  // generic tool error in the failure taxonomy.
+  let pathEscape = false;
   try {
     const parsed = JSON.parse(tc.arguments);
     // Capture the file's pre-image before a mutating-file tool overwrites it, so
@@ -544,15 +560,34 @@ async function executeToolCall(
         /* path escape is the tool's own error to surface */
       }
     }
+    if (failureTaxonomy && tool.category === "mutate-file" && typeof parsed.path === "string") {
+      try {
+        workspace.resolveSafe(parsed.path);
+      } catch {
+        pathEscape = true;
+      }
+    }
     // Time only the tool's own execution (approval/policy/hook gates are measured
     // or rejected above); recorded even when the tool throws via the finally.
     const executeStartedAt = Date.now();
     try {
-      return await tool.execute(parsed, workspace);
+      const result = await tool.execute(parsed, workspace);
+      // A tool that reports an error result without throwing is still a failed
+      // call; categorize it (a detected path escape takes precedence).
+      if (result.isError) {
+        failureTaxonomy?.record(pathEscape ? "path_escape" : "tool_error");
+      }
+      return result;
+    } catch (err: unknown) {
+      failureTaxonomy?.record(pathEscape ? "path_escape" : "tool_error");
+      const msg = err instanceof Error ? err.message : String(err);
+      return { content: `Tool error: ${msg}`, isError: true };
     } finally {
       bottleneck?.recordTool(tc.name, Date.now() - executeStartedAt);
     }
   } catch (err: unknown) {
+    // A failure outside the tool's own execution (e.g. malformed arguments).
+    failureTaxonomy?.record("tool_error");
     const msg = err instanceof Error ? err.message : String(err);
     return { content: `Tool error: ${msg}`, isError: true };
   }

src/headless-protocol.ts

@@ -2,6 +2,7 @@ import type { AgentSink } from "./agent.js";
 import type { ToolResult } from "./tools.js";
 import type { RunSummary } from "./run-summary.js";
 import type { BottleneckReport } from "./run-bottleneck.js";
+import type { FailureTaxonomyReport } from "./run-failure-taxonomy.js";
 import { redactSecrets } from "./permission-impact.js";
 
 // A stable, versioned newline-delimited JSON protocol for core run lifecycle
@@ -80,6 +81,10 @@ export type HeadlessEvent =
   // report, emitted just before the terminal `complete`. Metadata only: tool
   // names, wall-time, and counts — never prompt/tool/file content.
   | { type: "bottleneck"; bottleneck: BottleneckReport }
+  // Opt-in (`--failure-taxonomy`) privacy-safe failure-taxonomy report, emitted
+  // just before the terminal `complete`. Metadata only: failure-cause categories,
+  // counts, and the terminal reason — never error text or prompt/tool/file content.
+  | { type: "failure_taxonomy"; failureTaxonomy: FailureTaxonomyReport }
   | { type: "complete"; ok: boolean; exitCode: number; rounds: number; reason: string };
 
 export type HeadlessRecord = HeadlessEvent & {

src/index.ts

@@ -154,6 +154,7 @@ import { HeadlessWriter, createHeadlessSink, startEvent } from "./headless-proto
 import { redactSecrets, redactHomePath } from "./permission-impact.js";
 import { buildRunSummary, formatRunSummary } from "./run-summary.js";
 import { createBottleneckCollector, formatBottleneckReport } from "./run-bottleneck.js";
+import { createFailureTaxonomyCollector, formatFailureTaxonomyReport } from "./run-failure-taxonomy.js";
 import { loadImageAttachments, imageRef } from "./image-input.js";
 import type { LoadedImage } from "./image-input.js";
 import { createTools } from "./tools.js";
@@ -378,6 +379,10 @@ program
     "--bottleneck",
     "Print a privacy-safe tool/approval wall-time bottleneck report for the run (unattended use)",
   )
+  .option(
+    "--failure-taxonomy",
+    "Print a privacy-safe failure-cause taxonomy report for the run (unattended use)",
+  )
   .option(
     "--budget <usd>",
     "Spend budget in USD; stop before further provider calls once the estimated cost reaches it (also honors OMC_SPEND_BUDGET_USD)",
@@ -1942,6 +1947,7 @@ program
           const startedAt = Date.now();
           const turnImages = new TurnImageCollector();
           const bottleneck = opts.bottleneck ? createBottleneckCollector() : null;
+          const failureTaxonomy = opts.failureTaxonomy ? createFailureTaxonomyCollector() : null;
           let result: AgentResult;
           try {
             result = await runAgent(opts.prompt, existingMessages, {
@@ -1958,13 +1964,20 @@ program
               turnImages,
               preToolUseHooks,
               bottleneck: bottleneck?.collector,
+              failureTaxonomy: failureTaxonomy?.collector,
             });
           } catch (err: unknown) {
             const msg = err instanceof Error ? err.message : String(err);
             writer.emit({ type: "error", stage: "internal", message: redactSecrets(msg).text });
             if (bottleneck) {
               writer.emit({ type: "bottleneck", bottleneck: bottleneck.build(Date.now() - startedAt) });
             }
+            if (failureTaxonomy) {
+              writer.emit({
+                type: "failure_taxonomy",
+                failureTaxonomy: failureTaxonomy.build(Date.now() - startedAt, "error"),
+              });
+            }
             if (opts.summary) {
               writer.emit({
                 type: "summary",
@@ -2012,6 +2025,12 @@ program
           if (bottleneck) {
             writer.emit({ type: "bottleneck", bottleneck: bottleneck.build(Date.now() - startedAt) });
           }
+          if (failureTaxonomy) {
+            writer.emit({
+              type: "failure_taxonomy",
+              failureTaxonomy: failureTaxonomy.build(Date.now() - startedAt, result.reason),
+            });
+          }
           writer.emit({
             type: "complete",
             ok: result.ok,
@@ -2025,6 +2044,7 @@ program
         const startedAt = Date.now();
         const turnImages = new TurnImageCollector();
         const bottleneck = opts.bottleneck ? createBottleneckCollector() : null;
+        const failureTaxonomy = opts.failureTaxonomy ? createFailureTaxonomyCollector() : null;
         const result = await runAgent(opts.prompt, existingMessages, {
           config,
           workspace,
@@ -2038,6 +2058,7 @@ program
           turnImages,
           preToolUseHooks,
           bottleneck: bottleneck?.collector,
+          failureTaxonomy: failureTaxonomy?.collector,
         });
         sealSession();
         recordTurnCheckpoint(turnImages);
@@ -2065,6 +2086,11 @@ program
         if (bottleneck) {
           process.stdout.write("\n" + formatBottleneckReport(bottleneck.build(Date.now() - startedAt)) + "\n");
         }
+        if (failureTaxonomy) {
+          process.stdout.write(
+            "\n" + formatFailureTaxonomyReport(failureTaxonomy.build(Date.now() - startedAt, result.reason)) + "\n",
+          );
+        }
         process.exit(exitCode);
       } else {
         // Interactive REPL

src/run-failure-taxonomy.ts

@@ -0,0 +1,122 @@
+// A privacy-safe failure-taxonomy report for unattended (`-p`) runs. Where the run
+// summary (#40) reports a single terminal reason and counts tool failures by tool
+// name, this report categorizes each tool failure by CAUSE — why it failed — into a
+// fixed, bounded taxonomy (policy denial, hook denial, approval denial, path
+// escape, tool error, unknown tool, folder-trust denial), plus the terminal reason
+// class. It is opt-in (`--failure-taxonomy`) and metadata-only: it carries category
+// names and counts and has no channel for error text, prompts, file contents,
+// secrets, or raw tool payloads. The schema is stable and deterministic so CI can
+// retain and diff it.
+
+export const FAILURE_TAXONOMY_SCHEMA = "oh-my-cli.failure-taxonomy";
+export const FAILURE_TAXONOMY_VERSION = 1;
+
+// The fixed failure taxonomy, in canonical report order. Every tool failure is
+// bucketed into exactly one category; anything unrecognized falls into "other" so
+// the output stays bounded and stable even if a new failure path appears.
+export const FAILURE_CATEGORIES = [
+  "policy_denied",
+  "hook_denied",
+  "approval_denied",
+  "folder_trust_denied",
+  "path_escape",
+  "unknown_tool",
+  "tool_error",
+  "other",
+] as const;
+
+export type FailureCategory = (typeof FAILURE_CATEGORIES)[number];
+
+const CATEGORY_SET: ReadonlySet<string> = new Set(FAILURE_CATEGORIES);
+
+export interface FailureTaxonomyReport {
+  schema: typeof FAILURE_TAXONOMY_SCHEMA;
+  v: typeof FAILURE_TAXONOMY_VERSION;
+  // Total run wall-time, for context.
+  elapsedMs: number;
+  // Terminal reason class (e.g. completed, provider_error, max_rounds,
+  // budget_reached) — the run-level outcome, distinct from per-tool failure causes.
+  reason: string;
+  // Total number of failed tool calls across the run.
+  totalFailures: number;
+  // Category → count, in canonical taxonomy order, including only non-zero
+  // categories. Empty when no tool call failed.
+  byCategory: Record<string, number>;
+}
+
+// Accumulates per-cause tool-failure counts across a run.
+export interface FailureTaxonomyCollector {
+  record(category: FailureCategory): void;
+}
+
+export function createFailureTaxonomyCollector(): {
+  collector: FailureTaxonomyCollector;
+  build: (elapsedMs: number, reason: string) => FailureTaxonomyReport;
+} {
+  const counts = new Map<FailureCategory, number>();
+  const collector: FailureTaxonomyCollector = {
+    record(category) {
+      const key: FailureCategory = CATEGORY_SET.has(category) ? category : "other";
+      counts.set(key, (counts.get(key) ?? 0) + 1);
+    },
+  };
+  const build = (elapsedMs: number, reason: string): FailureTaxonomyReport =>
+    buildFailureTaxonomyReport(elapsedMs, reason, counts);
+  return { collector, build };
+}
+
+function buildFailureTaxonomyReport(
+  elapsedMs: number,
+  reason: string,
+  counts: Map<FailureCategory, number>,
+): FailureTaxonomyReport {
+  const byCategory: Record<string, number> = {};
+  let totalFailures = 0;
+  // Emit in canonical taxonomy order, only non-zero categories, for stable output.
+  for (const category of FAILURE_CATEGORIES) {
+    const n = counts.get(category) ?? 0;
+    if (n > 0) {
+      byCategory[category] = n;
+      totalFailures += n;
+    }
+  }
+  return {
+    schema: FAILURE_TAXONOMY_SCHEMA,
+    v: FAILURE_TAXONOMY_VERSION,
+    elapsedMs: Math.max(0, Math.round(elapsedMs)),
+    reason: sanitizeReason(reason),
+    totalFailures,
+    byCategory,
+  };
+}
+
+// The terminal reason is one of a small fixed set produced by the agent loop; bound
+// and strip control characters defensively so the report can never carry a runaway
+// or control-laced value.
+function sanitizeReason(reason: string): string {
+  const cleaned = reason.replace(/[\u0000-\u001f\u007f]+/g, "_").trim();
+  return cleaned.length <= 64 ? cleaned : cleaned.slice(0, 64);
+}
+
+function formatElapsed(ms: number): string {
+  return `${(ms / 1000).toFixed(1)}s`;
+}
+
+// A deterministic, human-readable rendering of the report. It contains only
+// category names, counts, and the terminal reason — there is no channel for error
+// text, prompt/tool/file content, or secrets.
+export function formatFailureTaxonomyReport(report: FailureTaxonomyReport): string {
+  const lines: string[] = [];
+  lines.push(`Failure taxonomy (${report.schema} v${report.v})`);
+  lines.push(`  elapsed:   ${formatElapsed(report.elapsedMs)}`);
+  lines.push(`  terminal:  ${report.reason}`);
+  if (report.totalFailures === 0) {
+    lines.push("  failures:  0 (none)");
+  } else {
+    lines.push(`  failures:  ${report.totalFailures}`);
+    for (const category of Object.keys(report.byCategory)) {
+      lines.push(`    ${category.padEnd(20)} ${report.byCategory[category]}`);
+    }
+  }
+  return lines.join("\n");
+}