From fa051e627f0569a1e4c6805face9f4a54a69bb47 Mon Sep 17 00:00:00 2001 From: qwen-code-dev-bot <269191875+qwen-code-dev-bot@users.noreply.github.com> Date: Thu, 23 Jul 2026 04:44:07 +0800 Subject: [PATCH] feat(cli): add reproducible task-fixture replay for deterministic unattended-run evaluation (#224) Before/after evaluation of a model or configuration change was noisy because there was no shared, repeatable definition of the task being measured: re-running an ad hoc prompt is not reproducible. Add a versioned task-fixture format (bounded prompt + deterministic provider/tool script) and an opt-in --replay-fixture mode that drives one unattended run from the fixture via a deterministic in-process provider, so the same fixture always yields the same tool-call sequence, rounds, and token totals and feeds reproducible inputs to the run summary (#40) and scorecard (#44). Fixture loading fails closed and rejects raw-credential fields; the replay never touches the network provider. --- src/agent.ts | 10 +- src/index.ts | 35 ++++- src/provider.ts | 9 ++ src/task-fixture.ts | 186 +++++++++++++++++++++++++ tests/integration/task-fixture.test.ts | 156 +++++++++++++++++++++ tests/unit/task-fixture.test.ts | 164 ++++++++++++++++++++++ 6 files changed, 555 insertions(+), 5 deletions(-) create mode 100644 src/task-fixture.ts create mode 100644 tests/integration/task-fixture.test.ts create mode 100644 tests/unit/task-fixture.test.ts diff --git a/src/agent.ts b/src/agent.ts index db2fc1e..0a24250 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -3,6 +3,7 @@ import type { SessionMessage } from "./session.js"; import type { ToolDef, ToolResult } from "./tools.js"; import { createTools, toolSchemasForOpenAI } from "./tools.js"; import { streamChat } from "./provider.js"; +import type { StreamProvider } from "./provider.js"; import type { Workspace } from "./workspace.js"; import type { ApprovalMode } from "./approval.js"; import { needsApproval, promptApproval } from "./approval.js"; @@ -60,6 +61,10 @@ export interface AgentOptions { // error, unknown tool, folder-trust denial) so the caller can build a // privacy-safe failure-taxonomy report. Undefined ⇒ no collection. failureTaxonomy?: FailureTaxonomyCollector; + // Optional provider stream override. Defaults to the network `streamChat`; + // the deterministic task-fixture replay (#224) supplies a scripted provider so + // the same fixture always produces the same run. Undefined ⇒ network provider. + streamProvider?: StreamProvider; } // Cumulative usage and cost reported after each round. `estimatedCostUsd` is an @@ -204,6 +209,9 @@ export async function runAgent( const toolMap = new Map(tools.map((t) => [t.name, t])); const schemas = toolSchemasForOpenAI(tools); const preToolUseHooks = opts.preToolUseHooks ?? []; + // Provider stream: the network `streamChat` by default, or a deterministic + // scripted provider for task-fixture replay (#224). + const stream = opts.streamProvider ?? streamChat; const messages: SessionMessage[] = [...existingMessages]; @@ -311,7 +319,7 @@ export async function runAgent( const assistantToolCalls: Array<{ id: string; name: string; arguments: string }> = []; try { - for await (const event of streamChat(opts.config, messages, { tools: schemas })) { + for await (const event of stream(opts.config, messages, { tools: schemas })) { if (event.type === "text") { assistantText += event.delta; sink.assistantDelta(event.delta); diff --git a/src/index.ts b/src/index.ts index 79ca117..adf26e2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -155,6 +155,7 @@ import { redactSecrets, redactHomePath } from "./permission-impact.js"; import { buildRunSummary, formatRunSummary } from "./run-summary.js"; import { createBottleneckCollector, formatBottleneckReport } from "./run-bottleneck.js"; import { createFailureTaxonomyCollector, formatFailureTaxonomyReport } from "./run-failure-taxonomy.js"; +import { readTaskFixtureFile, fixtureStreamProvider, type TaskFixture } from "./task-fixture.js"; import { loadImageAttachments, imageRef } from "./image-input.js"; import type { LoadedImage } from "./image-input.js"; import { createTools } from "./tools.js"; @@ -383,6 +384,10 @@ program "--failure-taxonomy", "Print a privacy-safe failure-cause taxonomy report for the run (unattended use)", ) + .option( + "--replay-fixture ", + "Replay a deterministic task fixture (bounded prompt + scripted responses) for a reproducible unattended run", + ) .option( "--budget ", "Spend budget in USD; stop before further provider calls once the estimated cost reaches it (also honors OMC_SPEND_BUDGET_USD)", @@ -1889,7 +1894,7 @@ program process.exit(2); } - if (opts.prompt) { + if (opts.prompt || opts.replayFixture) { // Non-interactive mode const format = String(opts.output ?? "text"); if (format !== "text" && format !== "json") { @@ -1897,6 +1902,26 @@ program process.exit(1); } + // Task-fixture replay (#224): load the fixture (fail closed) and drive the + // run from its bounded prompt and deterministic script instead of the + // network provider, so the same fixture reproduces the same run. + let replayFixture: TaskFixture | null = null; + if (opts.replayFixture) { + try { + replayFixture = readTaskFixtureFile(String(opts.replayFixture)); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + process.stderr.write(`${msg}\n`); + process.exit(2); + } + } + const runPrompt = replayFixture ? replayFixture.prompt : opts.prompt; + if (!runPrompt) { + process.stderr.write("Error: a prompt is required (use -p or --replay-fixture)\n"); + process.exit(1); + } + const streamProvider = replayFixture ? fixtureStreamProvider(replayFixture) : undefined; + // Capture a content-based checkpoint around this turn so a completed // turn can later be undone (and redone) without a Git reset. The // collector records each mutated file's pre-image before its tool runs; @@ -1942,7 +1967,7 @@ program // Headless protocol: a versioned NDJSON event stream on stdout. The // terminal `complete` record's exitCode matches the process exit code. const writer = new HeadlessWriter(process.stdout); - writer.emit(startEvent({ sessionId, model: config.model, prompt: opts.prompt })); + writer.emit(startEvent({ sessionId, model: config.model, prompt: runPrompt })); const sink = createHeadlessSink(writer); const startedAt = Date.now(); const turnImages = new TurnImageCollector(); @@ -1950,7 +1975,7 @@ program const failureTaxonomy = opts.failureTaxonomy ? createFailureTaxonomyCollector() : null; let result: AgentResult; try { - result = await runAgent(opts.prompt, existingMessages, { + result = await runAgent(runPrompt, existingMessages, { config, workspace, approvalMode, @@ -1965,6 +1990,7 @@ program preToolUseHooks, bottleneck: bottleneck?.collector, failureTaxonomy: failureTaxonomy?.collector, + streamProvider, }); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); @@ -2045,7 +2071,7 @@ program const turnImages = new TurnImageCollector(); const bottleneck = opts.bottleneck ? createBottleneckCollector() : null; const failureTaxonomy = opts.failureTaxonomy ? createFailureTaxonomyCollector() : null; - const result = await runAgent(opts.prompt, existingMessages, { + const result = await runAgent(runPrompt, existingMessages, { config, workspace, approvalMode, @@ -2059,6 +2085,7 @@ program preToolUseHooks, bottleneck: bottleneck?.collector, failureTaxonomy: failureTaxonomy?.collector, + streamProvider, }); sealSession(); recordTurnCheckpoint(turnImages); diff --git a/src/provider.ts b/src/provider.ts index 1ef0662..a25612b 100644 --- a/src/provider.ts +++ b/src/provider.ts @@ -41,6 +41,15 @@ export interface StreamedRetry { export type StreamEvent = StreamedText | StreamedToolCall | StreamedUsage | StreamedRetry; +// A provider call as a stream of events. `streamChat` is the network +// implementation; tests and the deterministic task-fixture replay (#224) supply +// their own implementation with the same signature. +export type StreamProvider = ( + config: Config, + messages: SessionMessage[], + options?: ProviderOptions, +) => AsyncGenerator; + // Bounded retry policy. Fixed so an unattended run can never hang: at most // RETRY_MAX_ATTEMPTS total tries, each wait capped at RETRY_MAX_DELAY_MS, so the // worst-case cumulative wait is bounded by their product. diff --git a/src/task-fixture.ts b/src/task-fixture.ts new file mode 100644 index 0000000..c8f544e --- /dev/null +++ b/src/task-fixture.ts @@ -0,0 +1,186 @@ +// A versioned, deterministic task fixture for reproducible unattended-run +// evaluation (roadmap #38). A fixture declares a bounded prompt and a +// deterministic script of provider responses (text or tool calls); replaying it +// drives one unattended run whose run-summary (#40) fields are reproducible — the +// same fixture yields the same tool-call sequence, rounds, and token totals, so +// before/after scorecards (#44) compare like-for-like inputs. Loading fails closed +// with a redacted error, and the fixture format rejects raw-credential fields. + +import fs from "node:fs"; +import { z } from "zod"; +import type { Config } from "./config.js"; +import type { SessionMessage } from "./session.js"; +import type { ProviderOptions, StreamEvent, StreamProvider } from "./provider.js"; + +export const TASK_FIXTURE_SCHEMA = "oh-my-cli.task-fixture"; +export const TASK_FIXTURE_VERSION = 1; +export const SUPPORTED_TASK_FIXTURE_VERSIONS: readonly number[] = [1]; + +// Deterministic usage emitted per scripted response so token totals are +// reproducible across replays (mirrors the test fake provider's fixed usage). +const FIXTURE_USAGE = { promptTokens: 5, completionTokens: 5, totalTokens: 10 }; + +// Raw secret field names that must never appear in a fixture; rejected (not +// ignored) so a plaintext secret cannot become a supported fixture path. Mirrors +// the other settings contracts. +const FORBIDDEN_FIXTURE_KEYS = [ + "apiKey", + "apikey", + "api_key", + "key", + "token", + "secret", + "password", + "credential", +]; + +const FixtureToolCallSchema = z + .object({ + id: z.string().min(1).max(256), + name: z.string().min(1).max(256), + arguments: z.string().max(65_536), + }) + .strict(); + +const FixtureStepSchema = z + .object({ + type: z.enum(["text", "tool_calls"]), + content: z.string().max(65_536).optional(), + toolCalls: z.array(FixtureToolCallSchema).max(64).optional(), + }) + .strict(); + +const TaskFixtureBodySchema = z + .object({ + schema: z.literal(TASK_FIXTURE_SCHEMA).optional(), + version: z.number().int(), + prompt: z.string().min(1).max(8_192), + script: z.array(FixtureStepSchema).min(1).max(256), + }) + .strict(); + +export interface TaskFixtureToolCall { + id: string; + name: string; + arguments: string; +} + +export interface TaskFixtureStep { + type: "text" | "tool_calls"; + content?: string; + toolCalls?: TaskFixtureToolCall[]; +} + +export interface TaskFixture { + version: number; + prompt: string; + script: TaskFixtureStep[]; +} + +function assertNoForbiddenKeys(obj: Record, where: string): void { + for (const key of FORBIDDEN_FIXTURE_KEYS) { + if (key in obj) { + throw new Error( + `Task fixture error: ${where} contains raw credential field "${key}"; ` + + "reference secrets via the environment, not the fixture", + ); + } + } +} + +// Parse and validate a task fixture, failing closed on a malformed fixture, an +// unsupported version, a step whose payload does not match its type, or a raw +// credential field. +export function parseTaskFixture(raw: unknown): TaskFixture { + if (raw === null || typeof raw !== "object" || Array.isArray(raw)) { + throw new Error("Task fixture error: fixture must be a JSON object"); + } + const obj = raw as Record; + assertNoForbiddenKeys(obj, "fixture"); + if (!("version" in obj)) { + throw new Error("Task fixture error: version is required"); + } + // Reject credential fields in each raw step before schema validation so the + // error names the offending step rather than a generic "unrecognized key". + if (Array.isArray(obj.script)) { + obj.script.forEach((step, i) => { + if (step && typeof step === "object" && !Array.isArray(step)) { + assertNoForbiddenKeys(step as Record, `script[${i}]`); + } + }); + } + const parsed = TaskFixtureBodySchema.safeParse(obj); + if (!parsed.success) { + const issues = parsed.error.issues.map((issue) => issue.message).join("; "); + throw new Error(`Task fixture error: ${issues}`); + } + const body = parsed.data; + if (!SUPPORTED_TASK_FIXTURE_VERSIONS.includes(body.version)) { + throw new Error( + `Task fixture error: task fixture version ${body.version} is not supported; ` + + `supported: ${SUPPORTED_TASK_FIXTURE_VERSIONS.join(", ")}`, + ); + } + const script: TaskFixtureStep[] = []; + body.script.forEach((step, i) => { + if (step.type === "text") { + if (typeof step.content !== "string") { + throw new Error(`Task fixture error: script[${i}] text step requires "content"`); + } + script.push({ type: "text", content: step.content }); + } else { + if (!Array.isArray(step.toolCalls) || step.toolCalls.length === 0) { + throw new Error(`Task fixture error: script[${i}] tool_calls step requires non-empty "toolCalls"`); + } + script.push({ + type: "tool_calls", + toolCalls: step.toolCalls.map((tc) => ({ id: tc.id, name: tc.name, arguments: tc.arguments })), + }); + } + }); + return { version: body.version, prompt: body.prompt, script }; +} + +// Read and parse a task fixture file, failing closed on a missing/unreadable file +// or invalid JSON. +export function readTaskFixtureFile(path: string): TaskFixture { + let raw: string; + try { + raw = fs.readFileSync(path, "utf8"); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + throw new Error(`Task fixture error: cannot read fixture file: ${msg}`); + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error("Task fixture error: fixture file contains invalid JSON"); + } + return parseTaskFixture(parsed); +} + +// A deterministic stream provider that replays the fixture's scripted responses, +// one per agent round, so the same fixture always yields the same tool-call +// sequence, rounds, and token totals. When the script is exhausted it yields a +// final empty-text response so the run terminates instead of looping. +export function fixtureStreamProvider(fixture: TaskFixture): StreamProvider { + let callIndex = 0; + return async function* fixtureStream( + _config: Config, + _messages: SessionMessage[], + _options?: ProviderOptions, + ): AsyncGenerator { + const step: TaskFixtureStep = + callIndex < fixture.script.length ? fixture.script[callIndex] : { type: "text", content: "" }; + callIndex++; + if (step.type === "text") { + yield { type: "text", delta: step.content ?? "" }; + } else { + for (const tc of step.toolCalls ?? []) { + yield { type: "tool_call", id: tc.id, name: tc.name, arguments: tc.arguments }; + } + } + yield { type: "usage", ...FIXTURE_USAGE }; + }; +} diff --git a/tests/integration/task-fixture.test.ts b/tests/integration/task-fixture.test.ts new file mode 100644 index 0000000..b4a5da6 --- /dev/null +++ b/tests/integration/task-fixture.test.ts @@ -0,0 +1,156 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; +import { createFakeServer } from "../fake-provider.js"; +import type { FakeServer } from "../fake-provider.js"; +import { parseHeadlessStream } from "../../src/headless-protocol.js"; +import { spawn } from "node:child_process"; +import path from "node:path"; +import fs from "node:fs"; +import os from "node:os"; + +function runCli( + args: string[], + env: Record, + timeoutMs = 20_000, +): Promise<{ stdout: string; stderr: string; code: number | null }> { + return new Promise((resolve, reject) => { + const cliPath = path.resolve(import.meta.dirname, "../../dist/index.js"); + const proc = spawn("node", [cliPath, ...args], { + env: { ...process.env, ...env }, + timeout: timeoutMs, + }); + let stdout = ""; + let stderr = ""; + proc.stdout.on("data", (d) => { stdout += d; }); + proc.stderr.on("data", (d) => { stderr += d; }); + proc.on("close", (code) => resolve({ stdout, stderr, code })); + proc.on("error", reject); + }); +} + +// The deterministic summary fields a reproducible replay must fix (everything +// except wall-time and the per-run session evidence pointers). +function deterministicFields(summary: Record): unknown { + const { elapsedMs: _e, evidence: _v, ...rest } = summary; + void _e; + void _v; + return rest; +} + +describe("Integration: task-fixture replay (--replay-fixture)", () => { + let server: FakeServer; + let tmpDir: string; + let sessionDir: string; + let baseEnv: Record; + let fixturePath: string; + + beforeAll(async () => { + server = await createFakeServer(); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "oh-my-cli-fixture-")); + sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), "oh-my-cli-fixture-sess-")); + baseEnv = { + OPENAI_API_KEY: "fake-key", + OPENAI_BASE_URL: server.url, + OPENAI_MODEL: "fake-model", + HOME: sessionDir, + }; + fixturePath = path.join(tmpDir, "fixture.json"); + fs.writeFileSync( + fixturePath, + JSON.stringify({ + schema: "oh-my-cli.task-fixture", + version: 1, + prompt: "write a file", + script: [ + { + type: "tool_calls", + toolCalls: [{ id: "c1", name: "write", arguments: JSON.stringify({ path: "replay.txt", content: "hi" }) }], + }, + { type: "text", content: "done" }, + ], + }), + ); + }); + + afterAll(async () => { + await server.close(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + fs.rmSync(sessionDir, { recursive: true, force: true }); + }); + + beforeEach(() => { + server.requests.length = 0; + }); + + function replay(): Promise<{ stdout: string; stderr: string; code: number | null }> { + return runCli( + [ + "--replay-fixture", fixturePath, + "--output", "json", + "--summary", + "--approval-mode", "yolo", + "--workspace", tmpDir, + ], + baseEnv, + ); + } + + it("replays a fixture deterministically: two replays yield identical summaries", async () => { + const r1 = await replay(); + expect(r1.code).toBe(0); + const r2 = await replay(); + expect(r2.code).toBe(0); + + const s1 = parseHeadlessStream(r1.stdout).find((x) => x.type === "summary"); + const s2 = parseHeadlessStream(r2.stdout).find((x) => x.type === "summary"); + expect(s1).toBeDefined(); + expect(s2).toBeDefined(); + if (s1?.type === "summary" && s2?.type === "summary") { + // The deterministic fields match exactly across the two replays. + expect(deterministicFields(s1.summary as unknown as Record)).toEqual( + deterministicFields(s2.summary as unknown as Record), + ); + // The fixture's scripted tool call is reflected in the summary. + const sum = s1.summary as unknown as { toolCalls: { byName: Record }; outcome: string }; + expect(sum.outcome).toBe("success"); + expect(sum.toolCalls.byName).toEqual({ write: 1 }); + } + // The replay never hit the network provider (the fixture provider drove it). + expect(server.requests.length).toBe(0); + // The scripted write actually ran. + expect(fs.existsSync(path.join(tmpDir, "replay.txt"))).toBe(true); + }); + + it("exits 2 (fail closed) on an unsupported fixture version", async () => { + const bad = path.join(tmpDir, "bad-version.json"); + fs.writeFileSync(bad, JSON.stringify({ version: 99, prompt: "x", script: [{ type: "text", content: "y" }] })); + const r = await runCli( + ["--replay-fixture", bad, "--output", "json", "--approval-mode", "yolo", "--workspace", tmpDir], + baseEnv, + ); + expect(r.code).toBe(2); + expect(r.stderr).toContain("not supported"); + }); + + it("exits 2 (fail closed) on a fixture carrying a raw credential field", async () => { + const bad = path.join(tmpDir, "bad-secret.json"); + fs.writeFileSync( + bad, + JSON.stringify({ version: 1, prompt: "x", script: [{ type: "text", content: "y" }], apiKey: "leaked" }), + ); + const r = await runCli( + ["--replay-fixture", bad, "--output", "json", "--approval-mode", "yolo", "--workspace", tmpDir], + baseEnv, + ); + expect(r.code).toBe(2); + expect(r.stderr).toContain("raw credential field"); + }); + + it("exits 2 on a missing fixture file", async () => { + const r = await runCli( + ["--replay-fixture", path.join(tmpDir, "nope.json"), "--output", "json", "--workspace", tmpDir], + baseEnv, + ); + expect(r.code).toBe(2); + expect(r.stderr).toContain("cannot read fixture file"); + }); +}); diff --git a/tests/unit/task-fixture.test.ts b/tests/unit/task-fixture.test.ts new file mode 100644 index 0000000..4b84635 --- /dev/null +++ b/tests/unit/task-fixture.test.ts @@ -0,0 +1,164 @@ +import { describe, it, expect, afterAll } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + parseTaskFixture, + readTaskFixtureFile, + fixtureStreamProvider, + TASK_FIXTURE_SCHEMA, + TASK_FIXTURE_VERSION, + SUPPORTED_TASK_FIXTURE_VERSIONS, +} from "../../src/task-fixture.js"; +import type { StreamEvent, StreamProvider, Config } from "../../src/provider.js"; + +const tmpDirs: string[] = []; +function tmpDir(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), "omc-task-fixture-")); + tmpDirs.push(d); + return d; +} +function writeFixture(obj: unknown): string { + const p = path.join(tmpDir(), "fixture.json"); + fs.writeFileSync(p, JSON.stringify(obj)); + return p; +} + +const VALID = { + schema: TASK_FIXTURE_SCHEMA, + version: 1, + prompt: "do the thing", + script: [ + { type: "tool_calls", toolCalls: [{ id: "c1", name: "write", arguments: "{\"path\":\"a.txt\",\"content\":\"hi\"}" }] }, + { type: "text", content: "done" }, + ], +}; + +async function drain(gen: AsyncGenerator): Promise { + const events: StreamEvent[] = []; + for await (const e of gen) events.push(e); + return events; +} + +// Simulate a full replay: call the provider once per round until a round produces +// no tool call (the terminal text response). +async function replayAll(provider: StreamProvider): Promise { + const rounds: StreamEvent[][] = []; + const config = {} as Config; + for (let i = 0; i < 100; i++) { + const events = await drain(provider(config, [], undefined)); + rounds.push(events); + if (!events.some((e) => e.type === "tool_call")) break; + } + return rounds; +} + +afterAll(() => { + for (const d of tmpDirs) fs.rmSync(d, { recursive: true, force: true }); +}); + +describe("parseTaskFixture", () => { + it("parses a valid fixture with tool_calls and text steps", () => { + const fixture = parseTaskFixture(VALID); + expect(fixture.version).toBe(TASK_FIXTURE_VERSION); + expect(fixture.prompt).toBe("do the thing"); + expect(fixture.script).toHaveLength(2); + expect(fixture.script[0].type).toBe("tool_calls"); + expect(fixture.script[0].toolCalls?.[0].name).toBe("write"); + expect(fixture.script[1]).toEqual({ type: "text", content: "done" }); + expect(SUPPORTED_TASK_FIXTURE_VERSIONS).toContain(TASK_FIXTURE_VERSION); + }); + + it("fails closed when version is missing", () => { + const { version, ...rest } = VALID; + void version; + expect(() => parseTaskFixture(rest)).toThrow(/version is required/); + }); + + it("fails closed on an unsupported version", () => { + expect(() => parseTaskFixture({ ...VALID, version: 99 })).toThrow(/not supported/); + }); + + it("fails closed on a non-object fixture", () => { + expect(() => parseTaskFixture([])).toThrow(/must be a JSON object/); + }); + + it("fails closed on an unknown top-level key", () => { + expect(() => parseTaskFixture({ ...VALID, extra: 1 })).toThrow(/unrecognized|unknown/i); + }); + + it("fails closed on a text step without content", () => { + expect(() => + parseTaskFixture({ ...VALID, script: [{ type: "text" }] }), + ).toThrow(/content/); + }); + + it("fails closed on a tool_calls step without toolCalls", () => { + expect(() => + parseTaskFixture({ ...VALID, script: [{ type: "tool_calls" }] }), + ).toThrow(/toolCalls/); + }); + + it("fails closed on a raw credential field at the top level", () => { + expect(() => parseTaskFixture({ ...VALID, apiKey: "leaked" })).toThrow(/raw credential field/); + }); + + it("fails closed on a raw credential field in a step", () => { + expect(() => + parseTaskFixture({ ...VALID, script: [{ type: "text", content: "x", token: "leaked" }] }), + ).toThrow(/raw credential field/); + }); +}); + +describe("readTaskFixtureFile", () => { + it("reads and parses a valid fixture file", () => { + const p = writeFixture(VALID); + expect(readTaskFixtureFile(p).prompt).toBe("do the thing"); + }); + + it("fails closed on a missing file", () => { + expect(() => readTaskFixtureFile(path.join(tmpDir(), "nope.json"))).toThrow(/cannot read fixture file/); + }); + + it("fails closed on invalid JSON", () => { + const p = path.join(tmpDir(), "bad.json"); + fs.writeFileSync(p, "{ not json"); + expect(() => readTaskFixtureFile(p)).toThrow(/invalid JSON/); + }); +}); + +describe("fixtureStreamProvider", () => { + it("replays scripted responses in order with deterministic usage", async () => { + const provider = fixtureStreamProvider(parseTaskFixture(VALID)); + const rounds = await replayAll(provider); + // Round 0: the write tool call + usage. Round 1: the final text + usage. + expect(rounds).toHaveLength(2); + expect(rounds[0].find((e) => e.type === "tool_call")).toMatchObject({ name: "write" }); + expect(rounds[0].find((e) => e.type === "usage")).toMatchObject({ totalTokens: 10 }); + expect(rounds[1].find((e) => e.type === "text")).toMatchObject({ delta: "done" }); + }); + + it("is deterministic: two replays of the same fixture are identical", async () => { + const fixture = parseTaskFixture(VALID); + const a = await replayAll(fixtureStreamProvider(fixture)); + const b = await replayAll(fixtureStreamProvider(fixture)); + expect(JSON.stringify(a)).toBe(JSON.stringify(b)); + }); + + it("terminates when the script is exhausted (final empty text)", async () => { + // A script of only tool calls would loop; the provider must yield a terminal + // empty-text response once the script runs out so the run ends. + const fixture = parseTaskFixture({ + version: 1, + prompt: "loop", + script: [{ type: "tool_calls", toolCalls: [{ id: "c1", name: "read", arguments: "{}" }] }], + }); + const provider = fixtureStreamProvider(fixture); + const rounds = await replayAll(provider); + // Round 0: the read tool call. Round 1: terminal empty text (no tool call). + expect(rounds).toHaveLength(2); + const last = rounds[rounds.length - 1]; + expect(last.some((e) => e.type === "tool_call")).toBe(false); + expect(last.find((e) => e.type === "text")).toMatchObject({ delta: "" }); + }); +});