feat(cli): add reproducible task-fixture replay for deterministic unattended-run evaluation (#224)

src/agent.ts

@@ -3,6 +3,7 @@ import type { SessionMessage } from "./session.js";
 import type { ToolDef, ToolResult } from "./tools.js";
 import { createTools, toolSchemasForOpenAI } from "./tools.js";
 import { streamChat } from "./provider.js";
+import type { StreamProvider } from "./provider.js";
 import type { Workspace } from "./workspace.js";
 import type { ApprovalMode } from "./approval.js";
 import { needsApproval, promptApproval } from "./approval.js";
@@ -60,6 +61,10 @@ export interface AgentOptions {
   // error, unknown tool, folder-trust denial) so the caller can build a
   // privacy-safe failure-taxonomy report. Undefined ⇒ no collection.
   failureTaxonomy?: FailureTaxonomyCollector;
+  // Optional provider stream override. Defaults to the network `streamChat`;
+  // the deterministic task-fixture replay (#224) supplies a scripted provider so
+  // the same fixture always produces the same run. Undefined ⇒ network provider.
+  streamProvider?: StreamProvider;
 }
 
 // Cumulative usage and cost reported after each round. `estimatedCostUsd` is an
@@ -204,6 +209,9 @@ export async function runAgent(
   const toolMap = new Map(tools.map((t) => [t.name, t]));
   const schemas = toolSchemasForOpenAI(tools);
   const preToolUseHooks = opts.preToolUseHooks ?? [];
+  // Provider stream: the network `streamChat` by default, or a deterministic
+  // scripted provider for task-fixture replay (#224).
+  const stream = opts.streamProvider ?? streamChat;
 
   const messages: SessionMessage[] = [...existingMessages];
 
@@ -311,7 +319,7 @@ export async function runAgent(
     const assistantToolCalls: Array<{ id: string; name: string; arguments: string }> = [];
 
     try {
-      for await (const event of streamChat(opts.config, messages, { tools: schemas })) {
+      for await (const event of stream(opts.config, messages, { tools: schemas })) {
         if (event.type === "text") {
           assistantText += event.delta;
           sink.assistantDelta(event.delta);

src/index.ts

@@ -155,6 +155,7 @@ import { redactSecrets, redactHomePath } from "./permission-impact.js";
 import { buildRunSummary, formatRunSummary } from "./run-summary.js";
 import { createBottleneckCollector, formatBottleneckReport } from "./run-bottleneck.js";
 import { createFailureTaxonomyCollector, formatFailureTaxonomyReport } from "./run-failure-taxonomy.js";
+import { readTaskFixtureFile, fixtureStreamProvider, type TaskFixture } from "./task-fixture.js";
 import { loadImageAttachments, imageRef } from "./image-input.js";
 import type { LoadedImage } from "./image-input.js";
 import { createTools } from "./tools.js";
@@ -383,6 +384,10 @@ program
     "--failure-taxonomy",
     "Print a privacy-safe failure-cause taxonomy report for the run (unattended use)",
   )
+  .option(
+    "--replay-fixture <file>",
+    "Replay a deterministic task fixture (bounded prompt + scripted responses) for a reproducible unattended run",
+  )
   .option(
     "--budget <usd>",
     "Spend budget in USD; stop before further provider calls once the estimated cost reaches it (also honors OMC_SPEND_BUDGET_USD)",
@@ -1889,14 +1894,34 @@ program
         process.exit(2);
       }
 
-      if (opts.prompt) {
+      if (opts.prompt || opts.replayFixture) {
         // Non-interactive mode
         const format = String(opts.output ?? "text");
         if (format !== "text" && format !== "json") {
           process.stderr.write(`Error: invalid output format "${format}"\n`);
           process.exit(1);
         }
 
+        // Task-fixture replay (#224): load the fixture (fail closed) and drive the
+        // run from its bounded prompt and deterministic script instead of the
+        // network provider, so the same fixture reproduces the same run.
+        let replayFixture: TaskFixture | null = null;
+        if (opts.replayFixture) {
+          try {
+            replayFixture = readTaskFixtureFile(String(opts.replayFixture));
+          } catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            process.stderr.write(`${msg}\n`);
+            process.exit(2);
+          }
+        }
+        const runPrompt = replayFixture ? replayFixture.prompt : opts.prompt;
+        if (!runPrompt) {
+          process.stderr.write("Error: a prompt is required (use -p or --replay-fixture)\n");
+          process.exit(1);
+        }
+        const streamProvider = replayFixture ? fixtureStreamProvider(replayFixture) : undefined;
+
         // Capture a content-based checkpoint around this turn so a completed
         // turn can later be undone (and redone) without a Git reset. The
         // collector records each mutated file's pre-image before its tool runs;
@@ -1942,15 +1967,15 @@ program
           // Headless protocol: a versioned NDJSON event stream on stdout. The
           // terminal `complete` record's exitCode matches the process exit code.
           const writer = new HeadlessWriter(process.stdout);
-          writer.emit(startEvent({ sessionId, model: config.model, prompt: opts.prompt }));
+          writer.emit(startEvent({ sessionId, model: config.model, prompt: runPrompt }));
           const sink = createHeadlessSink(writer);
           const startedAt = Date.now();
           const turnImages = new TurnImageCollector();
           const bottleneck = opts.bottleneck ? createBottleneckCollector() : null;
           const failureTaxonomy = opts.failureTaxonomy ? createFailureTaxonomyCollector() : null;
           let result: AgentResult;
           try {
-            result = await runAgent(opts.prompt, existingMessages, {
+            result = await runAgent(runPrompt, existingMessages, {
               config,
               workspace,
               approvalMode,
@@ -1965,6 +1990,7 @@ program
               preToolUseHooks,
               bottleneck: bottleneck?.collector,
               failureTaxonomy: failureTaxonomy?.collector,
+              streamProvider,
             });
           } catch (err: unknown) {
             const msg = err instanceof Error ? err.message : String(err);
@@ -2045,7 +2071,7 @@ program
         const turnImages = new TurnImageCollector();
         const bottleneck = opts.bottleneck ? createBottleneckCollector() : null;
         const failureTaxonomy = opts.failureTaxonomy ? createFailureTaxonomyCollector() : null;
-        const result = await runAgent(opts.prompt, existingMessages, {
+        const result = await runAgent(runPrompt, existingMessages, {
           config,
           workspace,
           approvalMode,
@@ -2059,6 +2085,7 @@ program
           preToolUseHooks,
           bottleneck: bottleneck?.collector,
           failureTaxonomy: failureTaxonomy?.collector,
+          streamProvider,
         });
         sealSession();
         recordTurnCheckpoint(turnImages);

src/provider.ts

@@ -41,6 +41,15 @@ export interface StreamedRetry {
 
 export type StreamEvent = StreamedText | StreamedToolCall | StreamedUsage | StreamedRetry;
 
+// A provider call as a stream of events. `streamChat` is the network
+// implementation; tests and the deterministic task-fixture replay (#224) supply
+// their own implementation with the same signature.
+export type StreamProvider = (
+  config: Config,
+  messages: SessionMessage[],
+  options?: ProviderOptions,
+) => AsyncGenerator<StreamEvent>;
+
 // Bounded retry policy. Fixed so an unattended run can never hang: at most
 // RETRY_MAX_ATTEMPTS total tries, each wait capped at RETRY_MAX_DELAY_MS, so the
 // worst-case cumulative wait is bounded by their product.

src/task-fixture.ts

@@ -0,0 +1,186 @@
+// A versioned, deterministic task fixture for reproducible unattended-run
+// evaluation (roadmap #38). A fixture declares a bounded prompt and a
+// deterministic script of provider responses (text or tool calls); replaying it
+// drives one unattended run whose run-summary (#40) fields are reproducible — the
+// same fixture yields the same tool-call sequence, rounds, and token totals, so
+// before/after scorecards (#44) compare like-for-like inputs. Loading fails closed
+// with a redacted error, and the fixture format rejects raw-credential fields.
+
+import fs from "node:fs";
+import { z } from "zod";
+import type { Config } from "./config.js";
+import type { SessionMessage } from "./session.js";
+import type { ProviderOptions, StreamEvent, StreamProvider } from "./provider.js";
+
+export const TASK_FIXTURE_SCHEMA = "oh-my-cli.task-fixture";
+export const TASK_FIXTURE_VERSION = 1;
+export const SUPPORTED_TASK_FIXTURE_VERSIONS: readonly number[] = [1];
+
+// Deterministic usage emitted per scripted response so token totals are
+// reproducible across replays (mirrors the test fake provider's fixed usage).
+const FIXTURE_USAGE = { promptTokens: 5, completionTokens: 5, totalTokens: 10 };
+
+// Raw secret field names that must never appear in a fixture; rejected (not
+// ignored) so a plaintext secret cannot become a supported fixture path. Mirrors
+// the other settings contracts.
+const FORBIDDEN_FIXTURE_KEYS = [
+  "apiKey",
+  "apikey",
+  "api_key",
+  "key",
+  "token",
+  "secret",
+  "password",
+  "credential",
+];
+
+const FixtureToolCallSchema = z
+  .object({
+    id: z.string().min(1).max(256),
+    name: z.string().min(1).max(256),
+    arguments: z.string().max(65_536),
+  })
+  .strict();
+
+const FixtureStepSchema = z
+  .object({
+    type: z.enum(["text", "tool_calls"]),
+    content: z.string().max(65_536).optional(),
+    toolCalls: z.array(FixtureToolCallSchema).max(64).optional(),
+  })
+  .strict();
+
+const TaskFixtureBodySchema = z
+  .object({
+    schema: z.literal(TASK_FIXTURE_SCHEMA).optional(),
+    version: z.number().int(),
+    prompt: z.string().min(1).max(8_192),
+    script: z.array(FixtureStepSchema).min(1).max(256),
+  })
+  .strict();
+
+export interface TaskFixtureToolCall {
+  id: string;
+  name: string;
+  arguments: string;
+}
+
+export interface TaskFixtureStep {
+  type: "text" | "tool_calls";
+  content?: string;
+  toolCalls?: TaskFixtureToolCall[];
+}
+
+export interface TaskFixture {
+  version: number;
+  prompt: string;
+  script: TaskFixtureStep[];
+}
+
+function assertNoForbiddenKeys(obj: Record<string, unknown>, where: string): void {
+  for (const key of FORBIDDEN_FIXTURE_KEYS) {
+    if (key in obj) {
+      throw new Error(
+        `Task fixture error: ${where} contains raw credential field "${key}"; ` +
+          "reference secrets via the environment, not the fixture",
+      );
+    }
+  }
+}
+
+// Parse and validate a task fixture, failing closed on a malformed fixture, an
+// unsupported version, a step whose payload does not match its type, or a raw
+// credential field.
+export function parseTaskFixture(raw: unknown): TaskFixture {
+  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
+    throw new Error("Task fixture error: fixture must be a JSON object");
+  }
+  const obj = raw as Record<string, unknown>;
+  assertNoForbiddenKeys(obj, "fixture");
+  if (!("version" in obj)) {
+    throw new Error("Task fixture error: version is required");
+  }
+  // Reject credential fields in each raw step before schema validation so the
+  // error names the offending step rather than a generic "unrecognized key".
+  if (Array.isArray(obj.script)) {
+    obj.script.forEach((step, i) => {
+      if (step && typeof step === "object" && !Array.isArray(step)) {
+        assertNoForbiddenKeys(step as Record<string, unknown>, `script[${i}]`);
+      }
+    });
+  }
+  const parsed = TaskFixtureBodySchema.safeParse(obj);
+  if (!parsed.success) {
+    const issues = parsed.error.issues.map((issue) => issue.message).join("; ");
+    throw new Error(`Task fixture error: ${issues}`);
+  }
+  const body = parsed.data;
+  if (!SUPPORTED_TASK_FIXTURE_VERSIONS.includes(body.version)) {
+    throw new Error(
+      `Task fixture error: task fixture version ${body.version} is not supported; ` +
+        `supported: ${SUPPORTED_TASK_FIXTURE_VERSIONS.join(", ")}`,
+    );
+  }
+  const script: TaskFixtureStep[] = [];
+  body.script.forEach((step, i) => {
+    if (step.type === "text") {
+      if (typeof step.content !== "string") {
+        throw new Error(`Task fixture error: script[${i}] text step requires "content"`);
+      }
+      script.push({ type: "text", content: step.content });
+    } else {
+      if (!Array.isArray(step.toolCalls) || step.toolCalls.length === 0) {
+        throw new Error(`Task fixture error: script[${i}] tool_calls step requires non-empty "toolCalls"`);
+      }
+      script.push({
+        type: "tool_calls",
+        toolCalls: step.toolCalls.map((tc) => ({ id: tc.id, name: tc.name, arguments: tc.arguments })),
+      });
+    }
+  });
+  return { version: body.version, prompt: body.prompt, script };
+}
+
+// Read and parse a task fixture file, failing closed on a missing/unreadable file
+// or invalid JSON.
+export function readTaskFixtureFile(path: string): TaskFixture {
+  let raw: string;
+  try {
+    raw = fs.readFileSync(path, "utf8");
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    throw new Error(`Task fixture error: cannot read fixture file: ${msg}`);
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    throw new Error("Task fixture error: fixture file contains invalid JSON");
+  }
+  return parseTaskFixture(parsed);
+}
+
+// A deterministic stream provider that replays the fixture's scripted responses,
+// one per agent round, so the same fixture always yields the same tool-call
+// sequence, rounds, and token totals. When the script is exhausted it yields a
+// final empty-text response so the run terminates instead of looping.
+export function fixtureStreamProvider(fixture: TaskFixture): StreamProvider {
+  let callIndex = 0;
+  return async function* fixtureStream(
+    _config: Config,
+    _messages: SessionMessage[],
+    _options?: ProviderOptions,
+  ): AsyncGenerator<StreamEvent> {
+    const step: TaskFixtureStep =
+      callIndex < fixture.script.length ? fixture.script[callIndex] : { type: "text", content: "" };
+    callIndex++;
+    if (step.type === "text") {
+      yield { type: "text", delta: step.content ?? "" };
+    } else {
+      for (const tc of step.toolCalls ?? []) {
+        yield { type: "tool_call", id: tc.id, name: tc.name, arguments: tc.arguments };
+      }
+    }
+    yield { type: "usage", ...FIXTURE_USAGE };
+  };
+}