From 827747b561f0af358595b99427132ea40d428eac Mon Sep 17 00:00:00 2001 From: "hiraoku.shinichi" Date: Sun, 5 Jul 2026 10:59:32 +0900 Subject: [PATCH 1/2] =?UTF-8?q?kaizen:=20=E5=8D=98=E4=B8=80=E3=81=AE=20bui?= =?UTF-8?q?lder-agent=20=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E4=B8=BB?= =?UTF-8?q?=E8=A6=81=E3=83=A2=E3=82=B8=E3=83=A5=E3=83=BC=E3=83=AB=E5=88=A5?= =?UTF-8?q?=E3=81=AE=20TypeScript=20=E3=83=86=E3=82=B9=E3=83=88=E3=81=B8?= =?UTF-8?q?=E5=88=86=E5=89=B2=E3=81=97=E3=81=BE=E3=81=97=E3=81=9F=E3=80=82?= =?UTF-8?q?=20(#75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/agents/AgentRunner.test.ts | 584 +++++++++++ test/artifacts.test.ts | 67 ++ test/builder-agent.test.js | 1387 -------------------------- test/builder/BuilderAgent.test.ts | 125 +++ test/cli.test.ts | 269 +++++ test/helpers.ts | 119 +++ test/review/SelfReview.test.ts | 33 + test/types/BuildRequest.test.ts | 30 + test/types/BuildResult.test.ts | 98 ++ test/types/DiscoveredIssue.test.ts | 42 + test/types/KaizenLoopPayload.test.ts | 72 ++ test/types/contracts.test.ts | 67 ++ 12 files changed, 1506 insertions(+), 1387 deletions(-) create mode 100644 test/agents/AgentRunner.test.ts create mode 100644 test/artifacts.test.ts delete mode 100644 test/builder-agent.test.js create mode 100644 test/builder/BuilderAgent.test.ts create mode 100644 test/cli.test.ts create mode 100644 test/helpers.ts create mode 100644 test/review/SelfReview.test.ts create mode 100644 test/types/BuildRequest.test.ts create mode 100644 test/types/BuildResult.test.ts create mode 100644 test/types/DiscoveredIssue.test.ts create mode 100644 test/types/KaizenLoopPayload.test.ts create mode 100644 test/types/contracts.test.ts diff --git a/test/agents/AgentRunner.test.ts b/test/agents/AgentRunner.test.ts new file mode 100644 index 0000000..6b1e823 --- /dev/null +++ b/test/agents/AgentRunner.test.ts @@ -0,0 +1,584 @@ +import assert from "node:assert/strict"; +import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, it } from "node:test"; +import { runImplementationAgent } from "../../dist/index.js"; + +describe("AgentRunner provider selection", () => { + it("supports the kaizen-loop contract with the codex backend", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + const argsPath = join(dir, "codex-args.json"); + await mkdir(binDir); + await writeFile(join(binDir, "package.json"), '{"type":"module"}', "utf8"); + const fakeCodexPath = join(binDir, "codex"); + + await writeFile( + fakeCodexPath, + `#!/usr/bin/env node +(async () => { +const { writeFileSync } = await import("node:fs"); +const args = process.argv.slice(2); +writeFileSync(${JSON.stringify(argsPath)}, JSON.stringify(args)); +const outputIndex = args.indexOf("--output-last-message"); +writeFileSync(args[outputIndex + 1], JSON.stringify({ + status: "fixed", + summary: "implemented with codex", + notes: "checked" +})); +})(); +`, + "utf8" + ); + await chmod(fakeCodexPath, 0o755); + + const result = await runImplementationAgent({ + agent: "codex", + prompt: "Fix issue #1", + workspaceDir: dir, + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}` + } + }); + const args = JSON.parse(await readFile(argsPath, "utf8")); + + assert.equal(result.exitCode, 0); + assert.equal(result.payload.status, "fixed"); + assert.equal(result.payload.summary, "implemented with codex"); + assert.match(result.payload.notes, /checked/); + assert.match(result.payload.notes, /codex: exitCode=0, status=selected, failureClass=none, fallbackReason=none, payloadSource=last-message/); + assert.match(result.payload.notes, /Selected backend: codex/); + assert.match(result.payload.notes, /Final payload source: last-message/); + assert.deepEqual(args.slice(0, 5), ["exec", "--json", "--sandbox", "workspace-write", "-C"]); + assert.equal(args.includes("--ask-for-approval"), false); + }); + + it("falls back to the next preferred backend when an agent fails without a payload", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + await mkdir(binDir); + const fakeCodexPath = join(binDir, "codex"); + const fakeClaudePath = join(binDir, "claude"); + + await writeFile( + fakeCodexPath, + `#!/usr/bin/env node +console.error("codex is not authenticated"); +process.exit(1); +`, + "utf8" + ); + await writeFile( + fakeClaudePath, + `#!/usr/bin/env node +console.log(JSON.stringify({ + result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"implemented by fallback\",\"notes\":\"checked\"}\n```")} +})); +`, + "utf8" + ); + await chmod(fakeCodexPath, 0o755); + await chmod(fakeClaudePath, 0o755); + + const result = await runImplementationAgent({ + agent: "codex,claude", + prompt: "Fix issue #1", + workspaceDir: dir, + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}` + } + }); + + assert.equal(result.exitCode, 0); + assert.equal(result.payload.summary, "implemented by fallback"); + assert.match(result.payload.notes, /codex: exitCode=1, status=fallback, failureClass=auth_failed/); + assert.match(result.payload.notes, /claude: exitCode=0, status=selected/); + }); + + it("returns aggregated attempt output when all preferred backends fail without a payload", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + await mkdir(binDir); + const fakeCodexPath = join(binDir, "codex"); + const fakeClaudePath = join(binDir, "claude"); + + await writeFile( + fakeCodexPath, + `#!/usr/bin/env node +console.error("codex failed " + "x".repeat(2500)); +process.exit(1); +`, + "utf8" + ); + await writeFile( + fakeClaudePath, + `#!/usr/bin/env node +console.error("claude failed"); +process.exit(1); +`, + "utf8" + ); + await chmod(fakeCodexPath, 0o755); + await chmod(fakeClaudePath, 0o755); + + const result = await runImplementationAgent({ + agent: "codex,claude", + prompt: "Fix issue #1", + workspaceDir: dir, + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}` + } + }); + + assert.equal(result.exitCode, 1); + assert.equal(result.payload, undefined); + assert.match(result.providerEvidence, /Provider evidence:/); + assert.match(result.providerEvidence, /codex: exitCode=1, status=fallback, failureClass=invalid_payload, fallbackReason=invalid_payload, payloadSource=none/); + assert.match(result.providerEvidence, /claude: exitCode=1, status=fallback, failureClass=invalid_payload, fallbackReason=invalid_payload, payloadSource=none/); + assert.match(result.raw, /Agent "claude" exited with code 1/); + assert.match(result.raw, /claude failed/); + }); + + it("runs custom providers from KAIZEN_AGENT_PROVIDERS", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + const argsPath = join(dir, "opencode-args.json"); + await mkdir(binDir); + await writeFile(join(binDir, "package.json"), '{"type":"module"}', "utf8"); + const fakeOpenCodePath = join(binDir, "opencode-go"); + + await writeFile( + fakeOpenCodePath, + `#!/usr/bin/env node +(async () => { +const { writeFileSync } = await import("node:fs"); +const args = process.argv.slice(2); +writeFileSync(${JSON.stringify(argsPath)}, JSON.stringify(args)); +console.log(JSON.stringify({ + status: "fixed", + summary: "implemented by custom provider", + notes: "checked" +})); +})(); +`, + "utf8" + ); + await chmod(fakeOpenCodePath, 0o755); + + const result = await runImplementationAgent({ + agent: "opencode-go", + prompt: "Fix issue #1", + workspaceDir: dir, + model: "zai-coder", + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}`, + KAIZEN_AGENT_PROVIDERS: JSON.stringify({ + "opencode-go": { + command: "opencode-go", + args: ["run", "--cwd", "{{workspaceDir}}", "--model", "{{model}}", "{{prompt}}"], + output: "stdout" + } + }) + } + }); + const args = JSON.parse(await readFile(argsPath, "utf8")); + + assert.equal(result.payload.status, "fixed"); + assert.equal(result.payload.summary, "implemented by custom provider"); + assert.deepEqual(args, ["run", "--cwd", dir, "--model", "zai-coder", "Fix issue #1"]); + }); + + it("omits custom provider flag-value pairs when a placeholder value is empty", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + const argsPath = join(dir, "zai-args.json"); + await mkdir(binDir); + await writeFile(join(binDir, "package.json"), '{"type":"module"}', "utf8"); + const fakeZaiPath = join(binDir, "zai"); + + await writeFile( + fakeZaiPath, + `#!/usr/bin/env node +(async () => { +const { writeFileSync } = await import("node:fs"); +const args = process.argv.slice(2); +writeFileSync(${JSON.stringify(argsPath)}, JSON.stringify(args)); +console.log(JSON.stringify({ + status: "fixed", + summary: "implemented without model", + notes: "checked" +})); +})(); +`, + "utf8" + ); + await chmod(fakeZaiPath, 0o755); + + await runImplementationAgent({ + agent: "zai", + prompt: "Fix issue #1", + workspaceDir: dir, + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}`, + KAIZEN_AGENT_PROVIDERS: JSON.stringify({ + zai: { + command: "zai", + args: ["agent", "--workspace", "{{workspaceDir}}", "--model", "{{model}}", "{{prompt}}"], + output: "stdout" + } + }) + } + }); + const args = JSON.parse(await readFile(argsPath, "utf8")); + + assert.deepEqual(args, ["agent", "--workspace", dir, "Fix issue #1"]); + }); + + it("loads custom providers from KAIZEN_AGENT_PROVIDERS_FILE and applies prompt templates", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + const argsPath = join(dir, "hermes-args.json"); + const providerConfigPath = join(dir, "providers.json"); + await mkdir(binDir); + await writeFile(join(binDir, "package.json"), '{"type":"module"}', "utf8"); + const fakeHermesPath = join(binDir, "hermes-agent"); + + await writeFile( + fakeHermesPath, + `#!/usr/bin/env node +(async () => { +const { writeFileSync } = await import("node:fs"); +const args = process.argv.slice(2); +writeFileSync(${JSON.stringify(argsPath)}, JSON.stringify(args)); +console.log(JSON.stringify({ + status: "fixed", + summary: "implemented by hermes-style provider", + notes: "checked" +})); +})(); +`, + "utf8" + ); + await writeFile( + providerConfigPath, + JSON.stringify({ + providers: { + "hermes-agent": { + command: "hermes-agent", + args: ["run", "--input", "{{prompt}}"], + promptTemplate: "Hermes task:\n{{prompt}}", + output: "stdout" + } + } + }), + "utf8" + ); + await chmod(fakeHermesPath, 0o755); + + const result = await runImplementationAgent({ + agent: "hermes-agent", + prompt: "Fix issue #1", + workspaceDir: dir, + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}`, + KAIZEN_AGENT_PROVIDERS_FILE: providerConfigPath + } + }); + const args = JSON.parse(await readFile(argsPath, "utf8")); + + assert.equal(result.payload.status, "fixed"); + assert.equal(result.payload.summary, "implemented by hermes-style provider"); + assert.deepEqual(args, ["run", "--input", "Hermes task:\nFix issue #1"]); + }); +}); + +describe("AgentRunner fallback classification", () => { + it("falls back when a provider health check fails with a fallbackable class", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + await mkdir(binDir); + const fakeHermesPath = join(binDir, "hermes-agent"); + const fakeClaudePath = join(binDir, "claude"); + + await writeFile( + fakeHermesPath, + `#!/usr/bin/env node +if (process.argv[2] === "health") { + console.error("401 unauthorized"); + process.exit(1); +} +console.log(JSON.stringify({ + status: "fixed", + summary: "primary should not run", + notes: "checked" +})); +`, + "utf8" + ); + await writeFile( + fakeClaudePath, + `#!/usr/bin/env node +console.log(JSON.stringify({ + result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"implemented after health-check fallback\",\"notes\":\"checked\"}\n```")} +})); +`, + "utf8" + ); + await chmod(fakeHermesPath, 0o755); + await chmod(fakeClaudePath, 0o755); + + const result = await runImplementationAgent({ + agent: "hermes-agent,claude", + prompt: "Fix issue #1", + workspaceDir: dir, + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}`, + KAIZEN_AGENT_PROVIDERS: JSON.stringify({ + "hermes-agent": { + command: "hermes-agent", + args: ["run", "{{prompt}}"], + healthCheck: { args: ["health"] }, + output: "stdout" + } + }) + } + }); + + assert.equal(result.payload.summary, "implemented after health-check fallback"); + assert.match(result.payload.notes, /Provider evidence/); + assert.match(result.payload.notes, /hermes-agent: exitCode=1, status=fallback, failureClass=auth_failed/); + assert.match(result.payload.notes, /Selected backend: claude/); + }); + + it("classifies fallbackable provider failure patterns", async () => { + const cases = [ + { name: "command_missing", command: "missing-kaizen-provider-command", args: [], pattern: /failureClass=command_missing/ }, + { name: "auth_failed", command: process.execPath, args: ["-e", "console.error('login required'); process.exit(1);"], pattern: /failureClass=auth_failed/ }, + { name: "rate_limited", command: process.execPath, args: ["-e", "console.error('429 too many requests'); process.exit(1);"], pattern: /failureClass=rate_limited/ }, + { name: "invalid_payload", command: process.execPath, args: ["-e", "console.error('not json'); process.exit(1);"], pattern: /failureClass=invalid_payload/ }, + { name: "timeout", command: process.execPath, args: ["-e", "setTimeout(() => {}, 1000);"], timeoutMs: 10, pattern: /failureClass=timeout/ } + ]; + + for (const failureCase of cases) { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const result = await runImplementationAgent({ + agent: `${failureCase.name}-provider,fallback`, + prompt: "Fix issue #1", + workspaceDir: dir, + env: { + ...process.env, + KAIZEN_AGENT_PROVIDERS: JSON.stringify({ + [`${failureCase.name}-provider`]: { + command: failureCase.command, + args: failureCase.args, + output: "stdout", + ...(failureCase.timeoutMs ? { timeoutMs: failureCase.timeoutMs } : {}) + }, + fallback: { + command: process.execPath, + args: ["-e", "console.log(JSON.stringify({status:'fixed',summary:'fallback selected',notes:'checked'}));"], + output: "stdout" + } + }) + } + }); + + assert.equal(result.payload.status, "fixed"); + assert.match(result.payload.notes, failureCase.pattern); + assert.match(result.payload.notes, /fallback: exitCode=0, status=selected/); + } + }); + + it("stops fallback for provider-blocked failures unless the provider opts in", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + await mkdir(binDir); + const fakeCodexPath = join(binDir, "codex"); + const fakeClaudePath = join(binDir, "claude"); + + await writeFile( + fakeCodexPath, + `#!/usr/bin/env node +console.error("content policy safety refusal"); +process.exit(1); +`, + "utf8" + ); + await writeFile( + fakeClaudePath, + `#!/usr/bin/env node +console.log(JSON.stringify({ + result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"should not fallback\",\"notes\":\"checked\"}\n```")} +})); +`, + "utf8" + ); + await chmod(fakeCodexPath, 0o755); + await chmod(fakeClaudePath, 0o755); + + const result = await runImplementationAgent({ + agent: "codex,claude", + prompt: "Fix issue #1", + workspaceDir: dir, + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}` + } + }); + + assert.equal(result.exitCode, 1); + assert.equal(result.payload, undefined); + assert.match(result.raw, /Failure class: provider_blocked/); + assert.doesNotMatch(result.raw, /should not fallback/); + }); + + it("falls back when a provider emits an unrelated safety log", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + await mkdir(binDir); + const fakeCodexPath = join(binDir, "codex"); + const fakeClaudePath = join(binDir, "claude"); + + await writeFile( + fakeCodexPath, + `#!/usr/bin/env node +console.error("project safety check failed"); +process.exit(1); +`, + "utf8" + ); + await writeFile( + fakeClaudePath, + `#!/usr/bin/env node +console.log(JSON.stringify({ + result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"fallback after project safety check\",\"notes\":\"checked\"}\n```")} +})); +`, + "utf8" + ); + await chmod(fakeCodexPath, 0o755); + await chmod(fakeClaudePath, 0o755); + + const result = await runImplementationAgent({ + agent: "codex,claude", + prompt: "Fix issue #1", + workspaceDir: dir, + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}` + } + }); + + assert.equal(result.payload.status, "fixed"); + assert.equal(result.payload.summary, "fallback after project safety check"); + assert.match(result.payload.notes, /codex: exitCode=1, status=fallback, failureClass=invalid_payload/); + assert.match(result.payload.notes, /Selected backend: claude/); + }); + + it("falls back on provider-blocked failures when the provider opts in", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + await mkdir(binDir); + const fakeHermesPath = join(binDir, "hermes-agent"); + const fakeClaudePath = join(binDir, "claude"); + + await writeFile( + fakeHermesPath, + `#!/usr/bin/env node +console.error("provider blocked by content policy"); +process.exit(1); +`, + "utf8" + ); + await writeFile( + fakeClaudePath, + `#!/usr/bin/env node +console.log(JSON.stringify({ + result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"implemented after provider-blocked fallback\",\"notes\":\"checked\"}\n```")} +})); +`, + "utf8" + ); + await chmod(fakeHermesPath, 0o755); + await chmod(fakeClaudePath, 0o755); + + const result = await runImplementationAgent({ + agent: "hermes-agent,claude", + prompt: "Fix issue #1", + workspaceDir: dir, + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}`, + KAIZEN_AGENT_PROVIDERS: JSON.stringify({ + "hermes-agent": { + command: "hermes-agent", + args: ["run", "{{prompt}}"], + fallbackOn: [" provider_blocked "], + output: "stdout" + } + }) + } + }); + + assert.equal(result.payload.status, "fixed"); + assert.equal(result.payload.summary, "implemented after provider-blocked fallback"); + assert.match(result.payload.notes, /hermes-agent: exitCode=1, status=fallback, failureClass=provider_blocked, fallbackReason=provider_blocked/); + assert.match(result.payload.notes, /claude: exitCode=0, status=selected, failureClass=none, fallbackReason=none/); + assert.match(result.payload.notes, /Selected backend: claude/); + assert.match(result.payload.notes, /Final payload source: stdout/); + }); + + it("preserves structured blocked payloads when the codex backend exits non-zero", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + await mkdir(binDir); + await writeFile(join(binDir, "package.json"), '{"type":"module"}', "utf8"); + const fakeCodexPath = join(binDir, "codex"); + + await writeFile( + fakeCodexPath, + `#!/usr/bin/env node +(async () => { +const { writeFileSync } = await import("node:fs"); +const args = process.argv.slice(2); +const outputIndex = args.indexOf("--output-last-message"); +writeFileSync(args[outputIndex + 1], JSON.stringify({ + status: "blocked", + summary: "provider reported a structured block", + notes: "captured provider detail", + blockedReason: "provider limit reached", + discoveredIssues: [{ title: "Provider limit", severity: "medium" }] +})); +process.exit(2); +})(); +`, + "utf8" + ); + await chmod(fakeCodexPath, 0o755); + + const result = await runImplementationAgent({ + agent: "codex", + prompt: "Fix issue #1", + workspaceDir: dir, + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}` + } + }); + + assert.equal(result.exitCode, 2); + assert.equal(result.payload.status, "blocked"); + assert.equal(result.payload.summary, "provider reported a structured block"); + assert.equal(result.payload.notes, "captured provider detail"); + assert.equal(result.payload.blockedReason, "provider limit reached"); + assert.deepEqual(result.payload.discoveredIssues, [{ title: "Provider limit", severity: "medium" }]); + }); +}); diff --git a/test/artifacts.test.ts b/test/artifacts.test.ts new file mode 100644 index 0000000..fc73905 --- /dev/null +++ b/test/artifacts.test.ts @@ -0,0 +1,67 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, it } from "node:test"; +import { writeBuildArtifacts } from "../dist/artifacts.js"; +import { BuilderAgent } from "../dist/index.js"; +import { createAdapter, failingReview, passingReview } from "./helpers.ts"; + +describe("artifacts", () => { + it("preserves artifacts for each implementation iteration", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const outDir = join(dir, "out"); + await mkdir(join(outDir, "iterations", "3"), { recursive: true }); + await writeFile(join(outDir, "iterations", "3", "stale.json"), "stale", "utf8"); + + const adapter = createAdapter({ reviews: [failingReview, passingReview] }); + adapter.implement = async () => ({ + summary: "Implemented the first version.", + changedFiles: ["src/feature.js"], + residualNotes: ["Tests still need to be added."], + discoveredIssues: [{ title: "Verifier warning needs follow-up", repo: "verifier" }] + }); + adapter.improve = async ({ implementation }) => ({ + summary: "Added targeted regression coverage.", + changedFiles: [...implementation.changedFiles, "test/feature.test.js"], + residualNotes: [], + discoveredIssues: [{ title: "Builder docs need a note", repo: "builder-agent" }] + }); + + const result = await new BuilderAgent(adapter).build({ + task: "Implement a small feature.", + maxIterations: 2 + }); + await writeBuildArtifacts(outDir, result); + + const resultText = await readFile(join(outDir, "build-result.json"), "utf8"); + const writtenResult = JSON.parse(resultText); + const latestReview = JSON.parse(await readFile(join(outDir, "self-review.json"), "utf8")); + const iteration1Summary = JSON.parse(await readFile(join(outDir, "iterations", "1", "implementation-summary.json"), "utf8")); + const iteration1ChangedFiles = JSON.parse(await readFile(join(outDir, "iterations", "1", "changed-files.json"), "utf8")); + const iteration1DiscoveredIssues = JSON.parse(await readFile(join(outDir, "iterations", "1", "discovered-issues.json"), "utf8")); + const iteration1Review = JSON.parse(await readFile(join(outDir, "iterations", "1", "self-review.json"), "utf8")); + const iteration1Instructions = JSON.parse(await readFile(join(outDir, "iterations", "1", "improvement-instructions.json"), "utf8")); + const iteration1ResidualNotes = JSON.parse(await readFile(join(outDir, "iterations", "1", "residual-notes.json"), "utf8")); + const iteration2Summary = JSON.parse(await readFile(join(outDir, "iterations", "2", "implementation-summary.json"), "utf8")); + const iteration2ChangedFiles = JSON.parse(await readFile(join(outDir, "iterations", "2", "changed-files.json"), "utf8")); + const iteration2DiscoveredIssues = JSON.parse(await readFile(join(outDir, "iterations", "2", "discovered-issues.json"), "utf8")); + const iteration2Review = JSON.parse(await readFile(join(outDir, "iterations", "2", "self-review.json"), "utf8")); + + assert.equal(writtenResult.status, "ready"); + assert.equal(writtenResult.iterations, 2); + assert.equal(latestReview.passed, true); + assert.equal(iteration1Summary.summary, "Implemented the first version."); + assert.deepEqual(iteration1ChangedFiles, ["src/feature.js"]); + assert.deepEqual(iteration1DiscoveredIssues, [{ title: "Verifier warning needs follow-up", repo: "verifier" }]); + assert.equal(iteration1Review.passed, false); + assert.deepEqual(iteration1Instructions, ["Add targeted tests for the requested behavior."]); + assert.deepEqual(iteration1ResidualNotes, ["Tests still need to be added."]); + assert.equal(iteration2Summary.summary, "Added targeted regression coverage."); + assert.deepEqual(iteration2ChangedFiles, ["src/feature.js", "test/feature.test.js"]); + assert.deepEqual(iteration2DiscoveredIssues, [{ title: "Builder docs need a note", repo: "builder-agent" }]); + assert.equal(iteration2Review.passed, true); + assert.equal(Object.hasOwn(writtenResult, "iterationArtifacts"), false); + await assert.rejects(readFile(join(outDir, "iterations", "3", "stale.json"), "utf8")); + }); +}); diff --git a/test/builder-agent.test.js b/test/builder-agent.test.js deleted file mode 100644 index 77ea753..0000000 --- a/test/builder-agent.test.js +++ /dev/null @@ -1,1387 +0,0 @@ -import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; -import { spawn } from "node:child_process"; -import { chmod, mkdir, mkdtemp, readdir, readFile, stat, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { promisify } from "node:util"; -import { describe, it } from "node:test"; -import { BuilderAgent, normalizeAgent, normalizeAgents, normalizeBuildRequest, normalizeBuildResult, normalizeKaizenLoopPayload, normalizeSelfReview } from "../dist/index.js"; - -const execFileAsync = promisify(execFile); - -const passingReview = { - score: 90, - confidence: 0.8, - dimensions: { - requirementFit: 90, - architectureQuality: 90, - implementationQuality: 90, - testQuality: 90, - maintainability: 90 - }, - mustFix: [], - shouldFix: [], - niceToHave: [], - improvementInstructions: [], - passed: false -}; - -const failingReview = { - score: 70, - confidence: 0.65, - dimensions: { - requirementFit: 70, - architectureQuality: 80, - implementationQuality: 70, - testQuality: 60, - maintainability: 80 - }, - mustFix: ["Add tests for the requested behavior."], - shouldFix: [], - niceToHave: [], - improvementInstructions: ["Add targeted tests for the requested behavior."], - passed: false -}; - -describe("BuilderAgent", () => { - it("returns ready when the first self-review passes", async () => { - const adapter = createAdapter({ reviews: [passingReview] }); - const result = await new BuilderAgent(adapter).build({ task: "Implement a small feature." }); - - assert.equal(result.status, "ready"); - assert.equal(result.iterations, 1); - assert.equal(result.review.passed, true); - assert.deepEqual(result.taskUnderstanding, { - summary: "analysis", - constraints: [] - }); - assert.deepEqual(result.changedFiles, ["src/feature.js"]); - }); - - it("falls back to normalized request details when analysis has no summary", async () => { - const adapter = createAdapter({ reviews: [passingReview] }); - adapter.analyzeTask = async () => ({}); - - const result = await new BuilderAgent(adapter).build({ - task: " Implement a small feature. ", - goal: " Preserve verifier handoff evidence. ", - constraints: [" Keep the change additive. "] - }); - - assert.deepEqual(result.taskUnderstanding, { - summary: "Task: Implement a small feature.", - goal: "Preserve verifier handoff evidence.", - constraints: ["Keep the change additive."] - }); - }); - - it("captures request constraints before later adapter hooks can mutate them", async () => { - const adapter = createAdapter({ reviews: [passingReview] }); - adapter.createPlan = async ({ request }) => { - request.constraints.push("Mutated during planning."); - return { summary: "Implement the requested change." }; - }; - - const result = await new BuilderAgent(adapter).build({ - task: "Implement a small feature.", - constraints: ["Keep the change additive."] - }); - - assert.deepEqual(result.taskUnderstanding.constraints, ["Keep the change additive."]); - }); - - it("runs improve and re-reviews until the threshold is met", async () => { - const adapter = createAdapter({ reviews: [failingReview, passingReview] }); - const result = await new BuilderAgent(adapter).build({ - task: "Implement a small feature.", - maxIterations: 2 - }); - - assert.equal(result.status, "ready"); - assert.equal(result.iterations, 2); - assert.equal(adapter.calls.improve, 1); - assert.deepEqual(result.changedFiles, ["src/feature.js", "test/feature.test.js"]); - }); - - it("stores immutable snapshots for completed iteration artifacts", async () => { - const mutableReview = { - ...failingReview, - mustFix: [...failingReview.mustFix], - improvementInstructions: [...failingReview.improvementInstructions] - }; - const adapter = createAdapter({ reviews: [mutableReview, passingReview] }); - const originalImprove = adapter.improve; - adapter.improve = async (input) => { - input.review.mustFix.push("mutated review"); - input.instructions.push("mutated instruction"); - return originalImprove(input); - }; - - const result = await new BuilderAgent(adapter).build({ - task: "Implement a small feature.", - maxIterations: 2 - }); - - assert.equal(result.status, "ready"); - assert.deepEqual(result.iterationArtifacts[0].review.mustFix, ["Add tests for the requested behavior."]); - assert.deepEqual(result.iterationArtifacts[0].improvementInstructions, ["Add targeted tests for the requested behavior."]); - }); - - it("preserves completed iteration artifacts when a later adapter step fails", async () => { - const adapter = createAdapter({ reviews: [failingReview] }); - adapter.improve = async () => { - throw new Error("adapter improve failed"); - }; - - const result = await new BuilderAgent(adapter).build({ - task: "Implement a small feature.", - maxIterations: 2 - }); - - assert.equal(result.status, "failed"); - assert.equal(result.iterationArtifacts.length, 1); - assert.equal(result.iterationArtifacts[0].implementationSummary, "Changed files: src/feature.js"); - }); - - it("returns blocked when maxIterations is reached without passing", async () => { - const adapter = createAdapter({ reviews: [failingReview, failingReview] }); - const result = await new BuilderAgent(adapter).build({ - task: "Implement a small feature.", - maxIterations: 2 - }); - - assert.equal(result.status, "blocked"); - assert.equal(result.iterations, 2); - assert.equal(result.review.passed, false); - assert.match(result.residualNotes[0], /did not pass/); - }); - - it("returns failed when the adapter contract is incomplete", async () => { - const result = await new BuilderAgent({}).build({ task: "Implement a small feature." }); - - assert.equal(result.status, "failed"); - assert.match(result.review.mustFix[0], /missing required method/); - }); -}); - -describe("validation", () => { - it("normalizes build request defaults", () => { - assert.deepEqual(normalizeBuildRequest({ task: " Do work. " }), { - task: "Do work.", - constraints: [], - threshold: 85, - maxIterations: 3 - }); - }); - - it("normalizes agent defaults to codex first", () => { - assert.deepEqual(normalizeAgents(undefined), ["codex", "claude"]); - assert.equal(normalizeAgent(undefined), "codex"); - }); - - it("normalizes custom provider fallbacks to codex first", () => { - assert.deepEqual(normalizeAgents("opencode-go"), ["opencode-go", "codex", "claude"]); - }); - - it("overrides an incorrect passed flag from self-review input", () => { - const review = normalizeSelfReview({ ...passingReview, passed: false }, 85); - - assert.equal(review.passed, true); - }); - - it("rejects unknown build request fields", () => { - assert.throws( - () => normalizeBuildRequest({ task: "Do work.", extra: true }), - /unknown field/ - ); - }); - - it("requires self-review input to match the published schema shape", () => { - assert.throws( - () => normalizeSelfReview({ ...passingReview, passed: undefined }, 85), - /passed must be a boolean/ - ); - assert.throws( - () => normalizeSelfReview({ - ...passingReview, - dimensions: { ...passingReview.dimensions, security: 90 } - }, 85), - /unknown field/ - ); - }); - - it("normalizes build result artifacts with the published schema shape", () => { - const result = normalizeBuildResult({ - status: "ready", - iterations: 1, - taskUnderstanding: { - summary: "Understand the requested behavior before implementation.", - constraints: ["Keep the change focused."] - }, - planSummary: "Implement the requested change.", - changedFiles: ["src/feature.js"], - review: passingReview, - residualNotes: [] - }); - - assert.equal(result.review.passed, true); - assert.deepEqual(result.taskUnderstanding, { - summary: "Understand the requested behavior before implementation.", - constraints: ["Keep the change focused."] - }); - assert.deepEqual(result.discoveredIssues, []); - assert.throws( - () => normalizeBuildResult({ ...result, extra: true }), - /unknown field/ - ); - assert.throws( - () => normalizeBuildResult({ - status: "ready", - iterations: 1, - planSummary: "Implement the requested change.", - changedFiles: ["src/feature.js"], - review: passingReview, - residualNotes: [] - }), - /taskUnderstanding is required/ - ); - assert.throws( - () => normalizeBuildResult({ - ...result, - taskUnderstanding: { - summary: "Understand the requested behavior before implementation." - } - }), - /taskUnderstanding\.constraints is required/ - ); - }); - - it("documents builder handoff as reviewable evidence, not approval", async () => { - const [readme, skill, implementPrompt, selfReviewPrompt, improvePrompt] = await Promise.all([ - readFile("README.md", "utf8"), - readFile("SKILL.md", "utf8"), - readFile("prompts/implement.md", "utf8"), - readFile("prompts/self-review.md", "utf8"), - readFile("prompts/improve.md", "utf8") - ]); - const handoffGuidance = `${readme}\n${skill}\n${implementPrompt}\n${selfReviewPrompt}\n${improvePrompt}`; - - assert.doesNotMatch(readme, /approved task/i); - assert.match(readme, /accepted issue or scoped task/i); - assert.match(handoffGuidance, /what changed/i); - assert.match(handoffGuidance, /why/i); - assert.match(handoffGuidance, /verification run or skipped/i); - assert.match(handoffGuidance, /residual risk/i); - assert.match(handoffGuidance, /reviewer notes/i); - assert.match(handoffGuidance, /not approval/i); - }); - - it("normalizes discovered issues in build results", () => { - const result = normalizeBuildResult({ - status: "ready", - iterations: 1, - taskUnderstanding: { - summary: "Understand the requested behavior before implementation.", - constraints: ["Keep the change focused."] - }, - planSummary: "Implement the requested change.", - changedFiles: ["src/feature.js"], - review: passingReview, - residualNotes: [], - discoveredIssues: [ - { - title: " Verifier false-positive on legacy status text ", - repo: "verifier", - body: "Observed during the run.", - expected: "The verifier should ignore plain status words in summaries.", - evidence: "verifier.log", - labels: ["kaizen", "kaizen"] - } - ] - }); - - assert.deepEqual(result.discoveredIssues, [ - { - title: "Verifier false-positive on legacy status text", - repo: "verifier", - body: "Observed during the run.", - expected: "The verifier should ignore plain status words in summaries.", - evidence: "verifier.log", - labels: ["kaizen"] - } - ]); - }); - - it("keeps discovered issues optional in the published build result schema", async () => { - const schema = JSON.parse(await readFile("schemas/build-result.schema.json", "utf8")); - - assert.equal(schema.properties.taskUnderstanding.type, "object"); - assert.equal(schema.required.includes("taskUnderstanding"), true); - assert.equal(schema.properties.discoveredIssues.type, "array"); - assert.equal(schema.required.includes("discoveredIssues"), false); - }); - - it("normalizes kaizen-loop payloads with the published schema shape", () => { - const payload = normalizeKaizenLoopPayload({ - status: "partial", - summary: " Implemented most of the change. ", - notes: "Ran targeted checks.", - discoveredIssues: [ - { - title: " Missing verifier diagnostic ", - repo: " verifier ", - labels: ["kaizen", "kaizen"] - } - ] - }); - - assert.deepEqual(payload, { - status: "partial", - summary: " Implemented most of the change. ", - notes: "Ran targeted checks.", - discoveredIssues: [ - { - title: "Missing verifier diagnostic", - repo: "verifier", - labels: ["kaizen"] - } - ] - }); - }); - - it("rejects malformed kaizen-loop discovered issues explicitly", () => { - assert.throws( - () => normalizeKaizenLoopPayload({ - status: "fixed", - summary: "Implemented.", - notes: "", - discoveredIssues: [{ repo: "verifier" }] - }), - /discoveredIssues\[0\]\.title/ - ); - assert.throws( - () => normalizeKaizenLoopPayload({ - status: "fixed", - summary: "Implemented.", - notes: "", - discoveredIssues: [{ title: "Bad routing", repo: 123 }] - }), - /discoveredIssues\[0\]\.repo must be a string/ - ); - assert.throws( - () => normalizeKaizenLoopPayload({ - status: "blocked", - summary: "Blocked.", - notes: "", - blockedReason: false - }), - /blockedReason must be a string/ - ); - }); - - it("publishes the kaizen-loop payload schema", async () => { - const schema = JSON.parse(await readFile("schemas/kaizen-loop-payload.schema.json", "utf8")); - - assert.deepEqual(schema.properties.status.enum, ["fixed", "partial", "blocked"]); - assert.equal(schema.properties.discoveredIssues.items.properties.repo.type, "string"); - assert.equal(schema.required.includes("discoveredIssues"), false); - }); -}); - -describe("TypeScript build boundaries", () => { - it("keeps source modules in TypeScript", async () => { - const sourceFiles = await listFiles("src"); - - assert.equal(sourceFiles.some((file) => file.endsWith(".js")), false); - assert.equal(sourceFiles.some((file) => file.endsWith(".ts")), true); - }); - - it("emits declarations for reusable builder contracts and runners", async () => { - const [packageJsonText, entrypoint, contracts, buildRequest, kaizenLoopPayload, builderAgent, agentRunner, kaizenLoop, cli, cliStat] = await Promise.all([ - readFile("package.json", "utf8"), - readFile("dist/index.d.ts", "utf8"), - readFile("dist/types/contracts.d.ts", "utf8"), - readFile("dist/types/BuildRequest.d.ts", "utf8"), - readFile("dist/types/KaizenLoopPayload.d.ts", "utf8"), - readFile("dist/builder/BuilderAgent.d.ts", "utf8"), - readFile("dist/agents/AgentRunner.d.ts", "utf8"), - readFile("dist/kaizen-loop.d.ts", "utf8"), - readFile("dist/cli.js", "utf8"), - stat("dist/cli.js") - ]); - const packageJson = JSON.parse(packageJsonText); - - assert.equal(packageJson.main, "./dist/index.js"); - assert.equal(packageJson.bin["builder-agent"], "./dist/cli.js"); - assert.equal(packageJson.types, "./dist/index.d.ts"); - assert.match(cli, /^#!\/usr\/bin\/env node/); - assert.notEqual(cliStat.mode & 0o111, 0); - assert.match(entrypoint, /export type \{[^}]*BuildRequest[^}]*BuilderAdapter[^}]*\} from "\.\/types\/contracts\.js"/); - assert.match(contracts, /export interface BuilderAdapter/); - assert.match(contracts, /export interface TaskUnderstanding/); - assert.match(contracts, /taskUnderstanding: TaskUnderstanding/); - assert.match(contracts, /export interface KaizenLoopPayload/); - assert.match(buildRequest, /normalizeBuildRequest\(input: BuildRequestInput\): BuildRequest/); - assert.match(kaizenLoopPayload, /normalizeKaizenLoopPayload\(input: unknown\): KaizenLoopPayload/); - assert.match(builderAgent, /build\(input: BuildRequestInput\): Promise/); - assert.match(agentRunner, /runImplementationAgent\([^)]*AgentRunInput[^)]*\): Promise/); - assert.match(kaizenLoop, /runKaizenLoopBuilder\([^)]*KaizenLoopBuilderIO[^)]*\): Promise/); - }); -}); - -describe("CLI", () => { - it("runs the build command and writes structured artifacts", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const requestPath = join(dir, "request.json"); - const adapterPath = join(dir, "adapter.mjs"); - const outDir = join(dir, "out"); - - await writeFile( - requestPath, - JSON.stringify({ task: "Implement a small feature.", maxIterations: 1 }, null, 2), - "utf8" - ); - await writeFile( - adapterPath, - ` -export default { - async analyzeTask() { - return {}; - }, - async createPlan() { - return { summary: "Implement the requested change." }; - }, - async implement() { - return { changedFiles: ["src/feature.js"], residualNotes: [] }; - }, - async selfReview() { - return ${JSON.stringify(passingReview)}; - }, - async improve() { - throw new Error("improve should not be called"); - } -}; -`, - "utf8" - ); - - const { stdout } = await execFileAsync(process.execPath, [ - "dist/cli.js", - "build", - "--request", - requestPath, - "--adapter", - adapterPath, - "--out", - outDir - ]); - const output = JSON.parse(stdout); - const result = JSON.parse(await readFile(join(outDir, "build-result.json"), "utf8")); - const review = JSON.parse(await readFile(join(outDir, "self-review.json"), "utf8")); - - assert.equal(output.status, "ready"); - assert.equal(result.status, "ready"); - assert.deepEqual(result.taskUnderstanding, { - summary: "Task: Implement a small feature.", - constraints: [] - }); - assert.equal(review.passed, true); - }); - - it("preserves artifacts for each implementation iteration", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const requestPath = join(dir, "request.json"); - const adapterPath = join(dir, "adapter.mjs"); - const outDir = join(dir, "out"); - - await writeFile( - requestPath, - JSON.stringify({ task: "Implement a small feature.", maxIterations: 2 }, null, 2), - "utf8" - ); - await mkdir(join(outDir, "iterations", "3"), { recursive: true }); - await writeFile(join(outDir, "iterations", "3", "stale.json"), "stale", "utf8"); - await writeFile( - adapterPath, - ` -let reviewCount = 0; - -export default { - async analyzeTask() { - return {}; - }, - async createPlan() { - return { summary: "Implement the requested change." }; - }, - async implement() { - return { - summary: "Implemented the first version.", - changedFiles: ["src/feature.js"], - residualNotes: ["Tests still need to be added."], - discoveredIssues: [{ title: "Verifier warning needs follow-up", repo: "verifier" }] - }; - }, - async selfReview() { - reviewCount += 1; - return reviewCount === 1 ? ${JSON.stringify(failingReview)} : ${JSON.stringify(passingReview)}; - }, - async improve({ implementation }) { - return { - summary: "Added targeted regression coverage.", - changedFiles: [...implementation.changedFiles, "test/feature.test.js"], - residualNotes: [], - discoveredIssues: [{ title: "Builder docs need a note", repo: "builder-agent" }] - }; - } -}; -`, - "utf8" - ); - - const { stdout } = await execFileAsync(process.execPath, [ - "dist/cli.js", - "build", - "--request", - requestPath, - "--adapter", - adapterPath, - "--out", - outDir - ]); - const output = JSON.parse(stdout); - const resultText = await readFile(join(outDir, "build-result.json"), "utf8"); - const result = JSON.parse(resultText); - const latestReview = JSON.parse(await readFile(join(outDir, "self-review.json"), "utf8")); - const iteration1Summary = JSON.parse(await readFile(join(outDir, "iterations", "1", "implementation-summary.json"), "utf8")); - const iteration1ChangedFiles = JSON.parse(await readFile(join(outDir, "iterations", "1", "changed-files.json"), "utf8")); - const iteration1DiscoveredIssues = JSON.parse(await readFile(join(outDir, "iterations", "1", "discovered-issues.json"), "utf8")); - const iteration1Review = JSON.parse(await readFile(join(outDir, "iterations", "1", "self-review.json"), "utf8")); - const iteration1Instructions = JSON.parse(await readFile(join(outDir, "iterations", "1", "improvement-instructions.json"), "utf8")); - const iteration1ResidualNotes = JSON.parse(await readFile(join(outDir, "iterations", "1", "residual-notes.json"), "utf8")); - const iteration2Summary = JSON.parse(await readFile(join(outDir, "iterations", "2", "implementation-summary.json"), "utf8")); - const iteration2ChangedFiles = JSON.parse(await readFile(join(outDir, "iterations", "2", "changed-files.json"), "utf8")); - const iteration2DiscoveredIssues = JSON.parse(await readFile(join(outDir, "iterations", "2", "discovered-issues.json"), "utf8")); - const iteration2Review = JSON.parse(await readFile(join(outDir, "iterations", "2", "self-review.json"), "utf8")); - - assert.equal(output.status, "ready"); - assert.equal(result.status, "ready"); - assert.equal(result.iterations, 2); - assert.equal(latestReview.passed, true); - assert.equal(iteration1Summary.summary, "Implemented the first version."); - assert.deepEqual(iteration1ChangedFiles, ["src/feature.js"]); - assert.deepEqual(iteration1DiscoveredIssues, [{ title: "Verifier warning needs follow-up", repo: "verifier" }]); - assert.equal(iteration1Review.passed, false); - assert.deepEqual(iteration1Instructions, ["Add targeted tests for the requested behavior."]); - assert.deepEqual(iteration1ResidualNotes, ["Tests still need to be added."]); - assert.equal(iteration2Summary.summary, "Added targeted regression coverage."); - assert.deepEqual(iteration2ChangedFiles, ["src/feature.js", "test/feature.test.js"]); - assert.deepEqual(iteration2DiscoveredIssues, [{ title: "Builder docs need a note", repo: "builder-agent" }]); - assert.equal(iteration2Review.passed, true); - assert.equal(Object.hasOwn(result, "iterationArtifacts"), false); - await assert.rejects(readFile(join(outDir, "iterations", "3", "stale.json"), "utf8")); - }); - - it("supports the kaizen-loop stdin/result-file contract", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const binDir = join(dir, "bin"); - const resultPath = join(dir, "build-result.json"); - const argsPath = join(dir, "claude-args.json"); - await mkdir(binDir); - const fakeCodexPath = join(binDir, "codex"); - const fakeClaudePath = join(binDir, "claude"); - - await writeFile( - fakeCodexPath, - `#!/usr/bin/env node -console.error("codex fallback disabled for this fixture"); -process.exit(1); -`, - "utf8" - ); - await writeFile( - fakeClaudePath, - `#!/usr/bin/env node -(async () => { -const { writeFileSync } = await import("node:fs"); -const args = process.argv.slice(2); -writeFileSync(${JSON.stringify(argsPath)}, JSON.stringify(args)); -console.log(JSON.stringify({ - result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"implemented\",\"notes\":\"checked\",\"discoveredIssues\":[{\"title\":\"Verifier false positive\",\"repo\":\"verifier\",\"evidence\":\"log excerpt\"}]}\n```")} -})); -})(); -`, - "utf8" - ); - await chmod(fakeCodexPath, 0o755); - await chmod(fakeClaudePath, 0o755); - - const { stdout } = await spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_BUILD_RESULT_PATH: resultPath, - KAIZEN_WORKSPACE_DIR: dir, - KAIZEN_PREFERRED_AGENT: "claude" - } - }); - - const output = JSON.parse(stdout); - const result = JSON.parse(await readFile(resultPath, "utf8")); - const args = JSON.parse(await readFile(argsPath, "utf8")); - const allowedToolsIndex = args.indexOf("--allowedTools"); - assert.notEqual(allowedToolsIndex, -1); - const allowedTools = args[allowedToolsIndex + 1]; - - assert.equal(output.status, "fixed"); - assert.equal(result.status, "fixed"); - assert.equal(result.summary, "implemented"); - assert.doesNotMatch(allowedTools, /Bash\(git add:\*\)/); - assert.doesNotMatch(allowedTools, /Bash\(git commit:\*\)/); - assert.match(allowedTools, /Bash\(npm:\*\)/); - assert.match(allowedTools, /\bRead\b/); - assert.match(allowedTools, /\bWrite\b/); - assert.match(allowedTools, /\bEdit\b/); - assert.match(allowedTools, /\bGlob\b/); - assert.match(allowedTools, /\bGrep\b/); - assert.deepEqual(result.discoveredIssues, [ - { - title: "Verifier false positive", - repo: "verifier", - evidence: "log excerpt" - } - ]); - }); - - it("supports the kaizen-loop contract with the codex backend", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const binDir = join(dir, "bin"); - const resultPath = join(dir, "build-result.json"); - const argsPath = join(dir, "codex-args.json"); - await mkdir(binDir); - await writeFile(join(binDir, "package.json"), '{"type":"module"}', "utf8"); - const fakeCodexPath = join(binDir, "codex"); - - await writeFile( - fakeCodexPath, - `#!/usr/bin/env node -(async () => { -const { writeFileSync } = await import("node:fs"); -const args = process.argv.slice(2); -writeFileSync(${JSON.stringify(argsPath)}, JSON.stringify(args)); -const outputIndex = args.indexOf("--output-last-message"); -writeFileSync(args[outputIndex + 1], JSON.stringify({ - status: "fixed", - summary: "implemented with codex", - notes: "checked" -})); -})(); -`, - "utf8" - ); - await chmod(fakeCodexPath, 0o755); - - const { stdout } = await spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_BUILD_RESULT_PATH: resultPath, - KAIZEN_WORKSPACE_DIR: dir, - KAIZEN_PREFERRED_AGENT: "codex" - } - }); - - const output = JSON.parse(stdout); - const result = JSON.parse(await readFile(resultPath, "utf8")); - const args = JSON.parse(await readFile(argsPath, "utf8")); - - assert.equal(output.status, "fixed"); - assert.equal(result.status, "fixed"); - assert.equal(result.summary, "implemented with codex"); - assert.match(result.notes, /checked/); - assert.match(result.notes, /codex: exitCode=0, status=selected, failureClass=none, fallbackReason=none, payloadSource=last-message/); - assert.match(result.notes, /Selected backend: codex/); - assert.match(result.notes, /Final payload source: last-message/); - assert.deepEqual(args.slice(0, 5), ["exec", "--json", "--sandbox", "workspace-write", "-C"]); - assert.equal(args.includes("--ask-for-approval"), false); - }); - - it("treats malformed kaizen-loop provider payloads as invalid instead of dropping discovered issues", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const binDir = join(dir, "bin"); - const resultPath = join(dir, "build-result.json"); - await mkdir(binDir); - const fakeClaudePath = join(binDir, "claude"); - - await writeFile( - fakeClaudePath, - `#!/usr/bin/env node -console.log(JSON.stringify({ - result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"implemented\",\"notes\":\"checked\",\"discoveredIssues\":[{\"repo\":\"verifier\"}]}\n```")} -})); -`, - "utf8" - ); - await chmod(fakeClaudePath, 0o755); - - await assert.rejects( - spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_BUILD_RESULT_PATH: resultPath, - KAIZEN_WORKSPACE_DIR: dir, - KAIZEN_PREFERRED_AGENT: "claude" - } - }), - /Command exited with 2/ - ); - - const result = JSON.parse(await readFile(resultPath, "utf8")); - - assert.equal(result.status, "blocked"); - assert.equal(result.summary, "Builder agent exited with code 1."); - assert.match(result.notes, /Agent "claude" exited with code 0/); - assert.match(result.notes, /Failure class: invalid_payload/); - assert.deepEqual(result.discoveredIssues, []); - }); - - it("falls back to the next preferred backend when an agent fails without a payload", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const binDir = join(dir, "bin"); - const resultPath = join(dir, "build-result.json"); - await mkdir(binDir); - const fakeCodexPath = join(binDir, "codex"); - const fakeClaudePath = join(binDir, "claude"); - - await writeFile( - fakeCodexPath, - `#!/usr/bin/env node -console.error("codex is not authenticated"); -process.exit(1); -`, - "utf8" - ); - await writeFile( - fakeClaudePath, - `#!/usr/bin/env node -console.log(JSON.stringify({ - result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"implemented by fallback\",\"notes\":\"checked\"}\n```")} -})); -`, - "utf8" - ); - await chmod(fakeCodexPath, 0o755); - await chmod(fakeClaudePath, 0o755); - - const { stdout } = await spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_BUILD_RESULT_PATH: resultPath, - KAIZEN_WORKSPACE_DIR: dir, - KAIZEN_PREFERRED_AGENT: "codex,claude" - } - }); - - const output = JSON.parse(stdout); - const result = JSON.parse(await readFile(resultPath, "utf8")); - - assert.equal(output.status, "fixed"); - assert.equal(result.summary, "implemented by fallback"); - }); - - it("returns aggregated attempt output when all preferred backends fail without a payload", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const binDir = join(dir, "bin"); - const resultPath = join(dir, "build-result.json"); - await mkdir(binDir); - const fakeCodexPath = join(binDir, "codex"); - const fakeClaudePath = join(binDir, "claude"); - - await writeFile( - fakeCodexPath, - `#!/usr/bin/env node -console.error("codex failed " + "x".repeat(2500)); -process.exit(1); -`, - "utf8" - ); - await writeFile( - fakeClaudePath, - `#!/usr/bin/env node -console.error("claude failed"); -process.exit(1); -`, - "utf8" - ); - await chmod(fakeCodexPath, 0o755); - await chmod(fakeClaudePath, 0o755); - - await assert.rejects( - spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_BUILD_RESULT_PATH: resultPath, - KAIZEN_WORKSPACE_DIR: dir, - KAIZEN_PREFERRED_AGENT: "codex,claude" - } - }), - /Command exited with 2/ - ); - - const result = JSON.parse(await readFile(resultPath, "utf8")); - - assert.equal(result.status, "blocked"); - assert.equal(result.summary, "Builder agent exited with code 1."); - assert.match(result.notes, /Provider evidence:/); - assert.match(result.notes, /codex: exitCode=1, status=fallback, failureClass=invalid_payload, fallbackReason=invalid_payload, payloadSource=none/); - assert.match(result.notes, /claude: exitCode=1, status=fallback, failureClass=invalid_payload, fallbackReason=invalid_payload, payloadSource=none/); - assert.match(result.notes, /Raw output tail:/); - assert.match(result.notes, /Agent "claude" exited with code 1/); - assert.match(result.notes, /claude failed/); - }); - - it("runs custom providers from KAIZEN_AGENT_PROVIDERS", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const binDir = join(dir, "bin"); - const resultPath = join(dir, "build-result.json"); - const argsPath = join(dir, "opencode-args.json"); - await mkdir(binDir); - await writeFile(join(binDir, "package.json"), '{"type":"module"}', "utf8"); - const fakeOpenCodePath = join(binDir, "opencode-go"); - - await writeFile( - fakeOpenCodePath, - `#!/usr/bin/env node -(async () => { -const { writeFileSync } = await import("node:fs"); -const args = process.argv.slice(2); -writeFileSync(${JSON.stringify(argsPath)}, JSON.stringify(args)); -console.log(JSON.stringify({ - status: "fixed", - summary: "implemented by custom provider", - notes: "checked" -})); -})(); -`, - "utf8" - ); - await chmod(fakeOpenCodePath, 0o755); - - const { stdout } = await spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_BUILD_RESULT_PATH: resultPath, - KAIZEN_WORKSPACE_DIR: dir, - KAIZEN_PREFERRED_AGENT: "opencode-go", - KAIZEN_AGENT_MODEL: "zai-coder", - KAIZEN_AGENT_PROVIDERS: JSON.stringify({ - "opencode-go": { - command: "opencode-go", - args: ["run", "--cwd", "{{workspaceDir}}", "--model", "{{model}}", "{{prompt}}"], - output: "stdout" - } - }) - } - }); - - const output = JSON.parse(stdout); - const args = JSON.parse(await readFile(argsPath, "utf8")); - - assert.equal(output.status, "fixed"); - assert.equal(output.summary, "implemented by custom provider"); - assert.deepEqual(args, ["run", "--cwd", dir, "--model", "zai-coder", "Fix issue #1"]); - }); - - it("omits custom provider flag-value pairs when a placeholder value is empty", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const binDir = join(dir, "bin"); - const resultPath = join(dir, "build-result.json"); - const argsPath = join(dir, "zai-args.json"); - await mkdir(binDir); - await writeFile(join(binDir, "package.json"), '{"type":"module"}', "utf8"); - const fakeZaiPath = join(binDir, "zai"); - - await writeFile( - fakeZaiPath, - `#!/usr/bin/env node -(async () => { -const { writeFileSync } = await import("node:fs"); -const args = process.argv.slice(2); -writeFileSync(${JSON.stringify(argsPath)}, JSON.stringify(args)); -console.log(JSON.stringify({ - status: "fixed", - summary: "implemented without model", - notes: "checked" -})); -})(); -`, - "utf8" - ); - await chmod(fakeZaiPath, 0o755); - - await spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_BUILD_RESULT_PATH: resultPath, - KAIZEN_WORKSPACE_DIR: dir, - KAIZEN_PREFERRED_AGENT: "zai", - KAIZEN_AGENT_PROVIDERS: JSON.stringify({ - zai: { - command: "zai", - args: ["agent", "--workspace", "{{workspaceDir}}", "--model", "{{model}}", "{{prompt}}"], - output: "stdout" - } - }) - } - }); - - const args = JSON.parse(await readFile(argsPath, "utf8")); - - assert.deepEqual(args, ["agent", "--workspace", dir, "Fix issue #1"]); - }); - - it("loads custom providers from KAIZEN_AGENT_PROVIDERS_FILE and applies prompt templates", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const binDir = join(dir, "bin"); - const resultPath = join(dir, "build-result.json"); - const argsPath = join(dir, "hermes-args.json"); - const providerConfigPath = join(dir, "providers.json"); - await mkdir(binDir); - await writeFile(join(binDir, "package.json"), '{"type":"module"}', "utf8"); - const fakeHermesPath = join(binDir, "hermes-agent"); - - await writeFile( - fakeHermesPath, - `#!/usr/bin/env node -(async () => { -const { writeFileSync } = await import("node:fs"); -const args = process.argv.slice(2); -writeFileSync(${JSON.stringify(argsPath)}, JSON.stringify(args)); -console.log(JSON.stringify({ - status: "fixed", - summary: "implemented by hermes-style provider", - notes: "checked" -})); -})(); -`, - "utf8" - ); - await writeFile( - providerConfigPath, - JSON.stringify({ - providers: { - "hermes-agent": { - command: "hermes-agent", - args: ["run", "--input", "{{prompt}}"], - promptTemplate: "Hermes task:\n{{prompt}}", - output: "stdout" - } - } - }), - "utf8" - ); - await chmod(fakeHermesPath, 0o755); - - const { stdout } = await spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_BUILD_RESULT_PATH: resultPath, - KAIZEN_WORKSPACE_DIR: dir, - KAIZEN_PREFERRED_AGENT: "hermes-agent", - KAIZEN_AGENT_PROVIDERS_FILE: providerConfigPath - } - }); - - const output = JSON.parse(stdout); - const args = JSON.parse(await readFile(argsPath, "utf8")); - - assert.equal(output.status, "fixed"); - assert.equal(output.summary, "implemented by hermes-style provider"); - assert.deepEqual(args, ["run", "--input", "Hermes task:\nFix issue #1"]); - }); - - it("falls back when a provider health check fails with a fallbackable class", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const binDir = join(dir, "bin"); - const resultPath = join(dir, "build-result.json"); - await mkdir(binDir); - const fakeHermesPath = join(binDir, "hermes-agent"); - const fakeClaudePath = join(binDir, "claude"); - - await writeFile( - fakeHermesPath, - `#!/usr/bin/env node -if (process.argv[2] === "health") { - console.error("401 unauthorized"); - process.exit(1); -} -console.log(JSON.stringify({ - status: "fixed", - summary: "primary should not run", - notes: "checked" -})); -`, - "utf8" - ); - await writeFile( - fakeClaudePath, - `#!/usr/bin/env node -console.log(JSON.stringify({ - result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"implemented after health-check fallback\",\"notes\":\"checked\"}\n```")} -})); -`, - "utf8" - ); - await chmod(fakeHermesPath, 0o755); - await chmod(fakeClaudePath, 0o755); - - const { stdout } = await spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_BUILD_RESULT_PATH: resultPath, - KAIZEN_WORKSPACE_DIR: dir, - KAIZEN_PREFERRED_AGENT: "hermes-agent,claude", - KAIZEN_AGENT_PROVIDERS: JSON.stringify({ - "hermes-agent": { - command: "hermes-agent", - args: ["run", "{{prompt}}"], - healthCheck: { args: ["health"] }, - output: "stdout" - } - }) - } - }); - - const output = JSON.parse(stdout); - const result = JSON.parse(await readFile(resultPath, "utf8")); - - assert.equal(output.status, "fixed"); - assert.equal(result.summary, "implemented after health-check fallback"); - assert.match(result.notes, /Provider evidence/); - assert.match(result.notes, /hermes-agent: exitCode=1, status=fallback, failureClass=auth_failed/); - assert.match(result.notes, /Selected backend: claude/); - }); - - it("stops fallback for provider-blocked failures unless the provider opts in", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const binDir = join(dir, "bin"); - const resultPath = join(dir, "build-result.json"); - await mkdir(binDir); - const fakeCodexPath = join(binDir, "codex"); - const fakeClaudePath = join(binDir, "claude"); - - await writeFile( - fakeCodexPath, - `#!/usr/bin/env node -console.error("content policy safety refusal"); -process.exit(1); -`, - "utf8" - ); - await writeFile( - fakeClaudePath, - `#!/usr/bin/env node -console.log(JSON.stringify({ - result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"should not fallback\",\"notes\":\"checked\"}\n```")} -})); -`, - "utf8" - ); - await chmod(fakeCodexPath, 0o755); - await chmod(fakeClaudePath, 0o755); - - await assert.rejects( - spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_BUILD_RESULT_PATH: resultPath, - KAIZEN_WORKSPACE_DIR: dir, - KAIZEN_PREFERRED_AGENT: "codex,claude" - } - }), - /Command exited with 2/ - ); - - const result = JSON.parse(await readFile(resultPath, "utf8")); - - assert.equal(result.status, "blocked"); - assert.equal(result.summary, "Builder agent exited with code 1."); - assert.match(result.notes, /Failure class: provider_blocked/); - assert.doesNotMatch(result.notes, /should not fallback/); - }); - - it("falls back when a provider emits an unrelated safety log", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const binDir = join(dir, "bin"); - const resultPath = join(dir, "build-result.json"); - await mkdir(binDir); - const fakeCodexPath = join(binDir, "codex"); - const fakeClaudePath = join(binDir, "claude"); - - await writeFile( - fakeCodexPath, - `#!/usr/bin/env node -console.error("project safety check failed"); -process.exit(1); -`, - "utf8" - ); - await writeFile( - fakeClaudePath, - `#!/usr/bin/env node -console.log(JSON.stringify({ - result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"fallback after project safety check\",\"notes\":\"checked\"}\n```")} -})); -`, - "utf8" - ); - await chmod(fakeCodexPath, 0o755); - await chmod(fakeClaudePath, 0o755); - - await spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_BUILD_RESULT_PATH: resultPath, - KAIZEN_WORKSPACE_DIR: dir, - KAIZEN_PREFERRED_AGENT: "codex,claude" - } - }); - - const result = JSON.parse(await readFile(resultPath, "utf8")); - - assert.equal(result.status, "fixed"); - assert.equal(result.summary, "fallback after project safety check"); - assert.match(result.notes, /codex: exitCode=1, status=fallback, failureClass=invalid_payload/); - assert.match(result.notes, /Selected backend: claude/); - }); - - it("falls back on provider-blocked failures when the provider opts in", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const binDir = join(dir, "bin"); - const resultPath = join(dir, "build-result.json"); - await mkdir(binDir); - const fakeHermesPath = join(binDir, "hermes-agent"); - const fakeClaudePath = join(binDir, "claude"); - - await writeFile( - fakeHermesPath, - `#!/usr/bin/env node -console.error("provider blocked by content policy"); -process.exit(1); -`, - "utf8" - ); - await writeFile( - fakeClaudePath, - `#!/usr/bin/env node -console.log(JSON.stringify({ - result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"implemented after provider-blocked fallback\",\"notes\":\"checked\"}\n```")} -})); -`, - "utf8" - ); - await chmod(fakeHermesPath, 0o755); - await chmod(fakeClaudePath, 0o755); - - await spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_BUILD_RESULT_PATH: resultPath, - KAIZEN_WORKSPACE_DIR: dir, - KAIZEN_PREFERRED_AGENT: "hermes-agent,claude", - KAIZEN_AGENT_PROVIDERS: JSON.stringify({ - "hermes-agent": { - command: "hermes-agent", - args: ["run", "{{prompt}}"], - fallbackOn: [" provider_blocked "], - output: "stdout" - } - }) - } - }); - - const result = JSON.parse(await readFile(resultPath, "utf8")); - - assert.equal(result.status, "fixed"); - assert.equal(result.summary, "implemented after provider-blocked fallback"); - assert.match(result.notes, /hermes-agent: exitCode=1, status=fallback, failureClass=provider_blocked, fallbackReason=provider_blocked/); - assert.match(result.notes, /claude: exitCode=0, status=selected, failureClass=none, fallbackReason=none/); - assert.match(result.notes, /Selected backend: claude/); - assert.match(result.notes, /Final payload source: stdout/); - }); - - it("preserves structured blocked payloads when the codex backend exits non-zero", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const binDir = join(dir, "bin"); - const resultPath = join(dir, "build-result.json"); - await mkdir(binDir); - await writeFile(join(binDir, "package.json"), '{"type":"module"}', "utf8"); - const fakeCodexPath = join(binDir, "codex"); - - await writeFile( - fakeCodexPath, - `#!/usr/bin/env node -(async () => { -const { writeFileSync } = await import("node:fs"); -const args = process.argv.slice(2); -const outputIndex = args.indexOf("--output-last-message"); -writeFileSync(args[outputIndex + 1], JSON.stringify({ - status: "blocked", - summary: "provider reported a structured block", - notes: "captured provider detail", - blockedReason: "provider limit reached", - discoveredIssues: [{ title: "Provider limit", severity: "medium" }] -})); -process.exit(2); -})(); -`, - "utf8" - ); - await chmod(fakeCodexPath, 0o755); - - await assert.rejects( - spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_BUILD_RESULT_PATH: resultPath, - KAIZEN_WORKSPACE_DIR: dir, - KAIZEN_PREFERRED_AGENT: "codex" - } - }), - /Command exited with 2/ - ); - - const result = JSON.parse(await readFile(resultPath, "utf8")); - - assert.equal(result.status, "blocked"); - assert.equal(result.summary, "provider reported a structured block"); - assert.equal(result.notes, "captured provider detail"); - assert.equal(result.blockedReason, "provider limit reached"); - assert.deepEqual(result.discoveredIssues, [{ title: "Provider limit", severity: "medium" }]); - }); - - it("creates the kaizen-loop result directory when it is missing", async () => { - const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); - const binDir = join(dir, "bin"); - const resultPath = join(dir, ".kaizen", "builder", "build-result.json"); - await mkdir(binDir); - const fakeClaudePath = join(binDir, "claude"); - - await writeFile( - fakeClaudePath, - `#!/usr/bin/env node -console.log(JSON.stringify({ - result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"implemented\",\"notes\":\"checked\"}\n```")} -})); -`, - "utf8" - ); - await chmod(fakeClaudePath, 0o755); - - await spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_BUILD_RESULT_PATH: resultPath, - KAIZEN_WORKSPACE_DIR: dir, - KAIZEN_PREFERRED_AGENT: "claude" - } - }); - - const result = JSON.parse(await readFile(resultPath, "utf8")); - assert.equal(result.status, "fixed"); - }); -}); - -function createAdapter({ reviews }) { - const calls = { - improve: 0 - }; - - return { - calls, - - async analyzeTask() { - return { summary: "analysis" }; - }, - - async createPlan() { - return { summary: "Implement the requested change and update tests." }; - }, - - async implement() { - return { - changedFiles: ["src/feature.js"], - residualNotes: [] - }; - }, - - async selfReview() { - return reviews.shift(); - }, - - async improve({ implementation }) { - calls.improve += 1; - return { - changedFiles: [...implementation.changedFiles, "test/feature.test.js"], - residualNotes: [] - }; - } - }; -} - -function spawnWithInput(command, args, input, options) { - return new Promise((resolve, reject) => { - const child = spawn(command, args, { - ...options, - stdio: ["pipe", "pipe", "pipe"] - }); - let stdout = ""; - let stderr = ""; - - child.stdout.setEncoding("utf8"); - child.stderr.setEncoding("utf8"); - child.stdout.on("data", (chunk) => { - stdout += chunk; - }); - child.stderr.on("data", (chunk) => { - stderr += chunk; - }); - child.on("error", reject); - child.on("close", (code) => { - if (code === 0) { - resolve({ stdout, stderr }); - } else { - reject(new Error(`Command exited with ${code}: ${stderr}${stdout}`)); - } - }); - child.stdin.end(input); - }); -} - -async function listFiles(root) { - const entries = await readdir(root, { withFileTypes: true }); - const files = []; - - for (const entry of entries) { - const path = join(root, entry.name); - if (entry.isDirectory()) { - files.push(...await listFiles(path)); - } else { - files.push(path); - } - } - - return files; -} diff --git a/test/builder/BuilderAgent.test.ts b/test/builder/BuilderAgent.test.ts new file mode 100644 index 0000000..50cfee8 --- /dev/null +++ b/test/builder/BuilderAgent.test.ts @@ -0,0 +1,125 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { BuilderAgent } from "../../dist/index.js"; +import { createAdapter, failingReview, passingReview } from "../helpers.ts"; + +describe("BuilderAgent", () => { + it("returns ready when the first self-review passes", async () => { + const adapter = createAdapter({ reviews: [passingReview] }); + const result = await new BuilderAgent(adapter).build({ task: "Implement a small feature." }); + + assert.equal(result.status, "ready"); + assert.equal(result.iterations, 1); + assert.equal(result.review.passed, true); + assert.deepEqual(result.taskUnderstanding, { + summary: "analysis", + constraints: [] + }); + assert.deepEqual(result.changedFiles, ["src/feature.js"]); + }); + + it("falls back to normalized request details when analysis has no summary", async () => { + const adapter = createAdapter({ reviews: [passingReview] }); + adapter.analyzeTask = async () => ({}); + + const result = await new BuilderAgent(adapter).build({ + task: " Implement a small feature. ", + goal: " Preserve verifier handoff evidence. ", + constraints: [" Keep the change additive. "] + }); + + assert.deepEqual(result.taskUnderstanding, { + summary: "Task: Implement a small feature.", + goal: "Preserve verifier handoff evidence.", + constraints: ["Keep the change additive."] + }); + }); + + it("captures request constraints before later adapter hooks can mutate them", async () => { + const adapter = createAdapter({ reviews: [passingReview] }); + adapter.createPlan = async ({ request }) => { + request.constraints.push("Mutated during planning."); + return { summary: "Implement the requested change." }; + }; + + const result = await new BuilderAgent(adapter).build({ + task: "Implement a small feature.", + constraints: ["Keep the change additive."] + }); + + assert.deepEqual(result.taskUnderstanding.constraints, ["Keep the change additive."]); + }); + + it("runs improve and re-reviews until the threshold is met", async () => { + const adapter = createAdapter({ reviews: [failingReview, passingReview] }); + const result = await new BuilderAgent(adapter).build({ + task: "Implement a small feature.", + maxIterations: 2 + }); + + assert.equal(result.status, "ready"); + assert.equal(result.iterations, 2); + assert.equal(adapter.calls.improve, 1); + assert.deepEqual(result.changedFiles, ["src/feature.js", "test/feature.test.js"]); + }); + + it("stores immutable snapshots for completed iteration artifacts", async () => { + const mutableReview = { + ...failingReview, + mustFix: [...failingReview.mustFix], + improvementInstructions: [...failingReview.improvementInstructions] + }; + const adapter = createAdapter({ reviews: [mutableReview, passingReview] }); + const originalImprove = adapter.improve; + adapter.improve = async (input) => { + input.review.mustFix.push("mutated review"); + input.instructions.push("mutated instruction"); + return originalImprove(input); + }; + + const result = await new BuilderAgent(adapter).build({ + task: "Implement a small feature.", + maxIterations: 2 + }); + + assert.equal(result.status, "ready"); + assert.deepEqual(result.iterationArtifacts[0].review.mustFix, ["Add tests for the requested behavior."]); + assert.deepEqual(result.iterationArtifacts[0].improvementInstructions, ["Add targeted tests for the requested behavior."]); + }); + + it("preserves completed iteration artifacts when a later adapter step fails", async () => { + const adapter = createAdapter({ reviews: [failingReview] }); + adapter.improve = async () => { + throw new Error("adapter improve failed"); + }; + + const result = await new BuilderAgent(adapter).build({ + task: "Implement a small feature.", + maxIterations: 2 + }); + + assert.equal(result.status, "failed"); + assert.equal(result.iterationArtifacts.length, 1); + assert.equal(result.iterationArtifacts[0].implementationSummary, "Changed files: src/feature.js"); + }); + + it("returns blocked when maxIterations is reached without passing", async () => { + const adapter = createAdapter({ reviews: [failingReview, failingReview] }); + const result = await new BuilderAgent(adapter).build({ + task: "Implement a small feature.", + maxIterations: 2 + }); + + assert.equal(result.status, "blocked"); + assert.equal(result.iterations, 2); + assert.equal(result.review.passed, false); + assert.match(result.residualNotes[0], /did not pass/); + }); + + it("returns failed when the adapter contract is incomplete", async () => { + const result = await new BuilderAgent({}).build({ task: "Implement a small feature." }); + + assert.equal(result.status, "failed"); + assert.match(result.review.mustFix[0], /missing required method/); + }); +}); diff --git a/test/cli.test.ts b/test/cli.test.ts new file mode 100644 index 0000000..ed25e60 --- /dev/null +++ b/test/cli.test.ts @@ -0,0 +1,269 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { describe, it } from "node:test"; +import { failingReview, passingReview, spawnWithInput } from "./helpers.ts"; + +const execFileAsync = promisify(execFile); + +describe("CLI", () => { + it("runs the build command and writes structured artifacts", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const requestPath = join(dir, "request.json"); + const adapterPath = join(dir, "adapter.mjs"); + const outDir = join(dir, "out"); + + await writeFile( + requestPath, + JSON.stringify({ task: "Implement a small feature.", maxIterations: 1 }, null, 2), + "utf8" + ); + await writeFile( + adapterPath, + ` +export default { + async analyzeTask() { + return {}; + }, + async createPlan() { + return { summary: "Implement the requested change." }; + }, + async implement() { + return { changedFiles: ["src/feature.js"], residualNotes: [] }; + }, + async selfReview() { + return ${JSON.stringify(passingReview)}; + }, + async improve() { + throw new Error("improve should not be called"); + } +}; +`, + "utf8" + ); + + const { stdout } = await execFileAsync(process.execPath, [ + "dist/cli.js", + "build", + "--request", + requestPath, + "--adapter", + adapterPath, + "--out", + outDir + ]); + const output = JSON.parse(stdout); + const result = JSON.parse(await readFile(join(outDir, "build-result.json"), "utf8")); + const review = JSON.parse(await readFile(join(outDir, "self-review.json"), "utf8")); + + assert.equal(output.status, "ready"); + assert.equal(result.status, "ready"); + assert.deepEqual(result.taskUnderstanding, { + summary: "Task: Implement a small feature.", + constraints: [] + }); + assert.equal(review.passed, true); + }); + + it("supports the kaizen-loop stdin/result-file contract", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + const resultPath = join(dir, "build-result.json"); + const argsPath = join(dir, "claude-args.json"); + await mkdir(binDir); + const fakeCodexPath = join(binDir, "codex"); + const fakeClaudePath = join(binDir, "claude"); + + await writeFile( + fakeCodexPath, + `#!/usr/bin/env node +console.error("codex fallback disabled for this fixture"); +process.exit(1); +`, + "utf8" + ); + await writeFile( + fakeClaudePath, + `#!/usr/bin/env node +(async () => { +const { writeFileSync } = await import("node:fs"); +const args = process.argv.slice(2); +writeFileSync(${JSON.stringify(argsPath)}, JSON.stringify(args)); +console.log(JSON.stringify({ + result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"implemented\",\"notes\":\"checked\",\"discoveredIssues\":[{\"title\":\"Verifier false positive\",\"repo\":\"verifier\",\"evidence\":\"log excerpt\"}]}\n```")} +})); +})(); +`, + "utf8" + ); + await chmod(fakeCodexPath, 0o755); + await chmod(fakeClaudePath, 0o755); + + const { stdout } = await spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}`, + KAIZEN_BUILD_RESULT_PATH: resultPath, + KAIZEN_WORKSPACE_DIR: dir, + KAIZEN_PREFERRED_AGENT: "claude" + } + }); + + const output = JSON.parse(stdout); + const result = JSON.parse(await readFile(resultPath, "utf8")); + const args = JSON.parse(await readFile(argsPath, "utf8")); + const allowedToolsIndex = args.indexOf("--allowedTools"); + assert.notEqual(allowedToolsIndex, -1); + const allowedTools = args[allowedToolsIndex + 1]; + + assert.equal(output.status, "fixed"); + assert.equal(result.status, "fixed"); + assert.equal(result.summary, "implemented"); + assert.doesNotMatch(allowedTools, /Bash\(git add:\*\)/); + assert.doesNotMatch(allowedTools, /Bash\(git commit:\*\)/); + assert.match(allowedTools, /Bash\(npm:\*\)/); + assert.match(allowedTools, /\bRead\b/); + assert.match(allowedTools, /\bWrite\b/); + assert.match(allowedTools, /\bEdit\b/); + assert.match(allowedTools, /\bGlob\b/); + assert.match(allowedTools, /\bGrep\b/); + assert.deepEqual(result.discoveredIssues, [ + { + title: "Verifier false positive", + repo: "verifier", + evidence: "log excerpt" + } + ]); + }); + + it("returns exit code 2 for blocked build results", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const requestPath = join(dir, "request.json"); + const adapterPath = join(dir, "adapter.mjs"); + + await writeFile( + requestPath, + JSON.stringify({ task: "Implement a small feature.", maxIterations: 1 }, null, 2), + "utf8" + ); + await writeFile( + adapterPath, + ` +export default { + async analyzeTask() { + return {}; + }, + async createPlan() { + return { summary: "Implement the requested change." }; + }, + async implement() { + return { changedFiles: ["src/feature.js"], residualNotes: [] }; + }, + async selfReview() { + return ${JSON.stringify(failingReview)}; + }, + async improve() { + throw new Error("improve should not be called"); + } +}; +`, + "utf8" + ); + + await assert.rejects( + execFileAsync(process.execPath, [ + "dist/cli.js", + "build", + "--request", + requestPath, + "--adapter", + adapterPath, + "--out", + join(dir, "out") + ]), + (error) => error.code === 2 && /"status": "blocked"/.test(error.stdout) + ); + }); + + it("returns exit code 3 for command parsing errors", async () => { + await assert.rejects( + execFileAsync(process.execPath, ["dist/cli.js", "unknown-command"]), + (error) => error.code === 3 && /Unknown command: unknown-command/.test(error.stderr) + ); + }); + + it("treats malformed kaizen-loop provider payloads as invalid instead of dropping discovered issues", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + const resultPath = join(dir, "build-result.json"); + await mkdir(binDir); + const fakeClaudePath = join(binDir, "claude"); + + await writeFile( + fakeClaudePath, + `#!/usr/bin/env node +console.log(JSON.stringify({ + result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"implemented\",\"notes\":\"checked\",\"discoveredIssues\":[{\"repo\":\"verifier\"}]}\n```")} +})); +`, + "utf8" + ); + await chmod(fakeClaudePath, 0o755); + + await assert.rejects( + spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}`, + KAIZEN_BUILD_RESULT_PATH: resultPath, + KAIZEN_WORKSPACE_DIR: dir, + KAIZEN_PREFERRED_AGENT: "claude" + } + }), + /Command exited with 2/ + ); + + const result = JSON.parse(await readFile(resultPath, "utf8")); + + assert.equal(result.status, "blocked"); + assert.equal(result.summary, "Builder agent exited with code 1."); + assert.match(result.notes, /Agent "claude" exited with code 0/); + assert.match(result.notes, /Failure class: invalid_payload/); + assert.deepEqual(result.discoveredIssues, []); + }); + + it("creates the kaizen-loop result directory when it is missing", async () => { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + const resultPath = join(dir, ".kaizen", "builder", "build-result.json"); + await mkdir(binDir); + const fakeClaudePath = join(binDir, "claude"); + + await writeFile( + fakeClaudePath, + `#!/usr/bin/env node +console.log(JSON.stringify({ + result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"implemented\",\"notes\":\"checked\"}\n```")} +})); +`, + "utf8" + ); + await chmod(fakeClaudePath, 0o755); + + await spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}`, + KAIZEN_BUILD_RESULT_PATH: resultPath, + KAIZEN_WORKSPACE_DIR: dir, + KAIZEN_PREFERRED_AGENT: "claude" + } + }); + + const result = JSON.parse(await readFile(resultPath, "utf8")); + assert.equal(result.status, "fixed"); + }); +}); diff --git a/test/helpers.ts b/test/helpers.ts new file mode 100644 index 0000000..8e7c207 --- /dev/null +++ b/test/helpers.ts @@ -0,0 +1,119 @@ +import { spawn } from "node:child_process"; +import { readdir } from "node:fs/promises"; +import { join } from "node:path"; + +export const passingReview = { + score: 90, + confidence: 0.8, + dimensions: { + requirementFit: 90, + architectureQuality: 90, + implementationQuality: 90, + testQuality: 90, + maintainability: 90 + }, + mustFix: [], + shouldFix: [], + niceToHave: [], + improvementInstructions: [], + passed: false +}; + +export const failingReview = { + score: 70, + confidence: 0.65, + dimensions: { + requirementFit: 70, + architectureQuality: 80, + implementationQuality: 70, + testQuality: 60, + maintainability: 80 + }, + mustFix: ["Add tests for the requested behavior."], + shouldFix: [], + niceToHave: [], + improvementInstructions: ["Add targeted tests for the requested behavior."], + passed: false +}; + +export function createAdapter({ reviews }) { + const calls = { + improve: 0 + }; + + return { + calls, + + async analyzeTask() { + return { summary: "analysis" }; + }, + + async createPlan() { + return { summary: "Implement the requested change and update tests." }; + }, + + async implement() { + return { + changedFiles: ["src/feature.js"], + residualNotes: [] + }; + }, + + async selfReview() { + return reviews.shift(); + }, + + async improve({ implementation }) { + calls.improve += 1; + return { + changedFiles: [...implementation.changedFiles, "test/feature.test.js"], + residualNotes: [] + }; + } + }; +} + +export function spawnWithInput(command, args, input, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + ...options, + stdio: ["pipe", "pipe", "pipe"] + }); + let stdout = ""; + let stderr = ""; + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) { + resolve({ stdout, stderr }); + } else { + reject(new Error(`Command exited with ${code}: ${stderr}${stdout}`)); + } + }); + child.stdin.end(input); + }); +} + +export async function listFiles(root) { + const entries = await readdir(root, { withFileTypes: true }); + const files = []; + + for (const entry of entries) { + const path = join(root, entry.name); + if (entry.isDirectory()) { + files.push(...await listFiles(path)); + } else { + files.push(path); + } + } + + return files; +} diff --git a/test/review/SelfReview.test.ts b/test/review/SelfReview.test.ts new file mode 100644 index 0000000..2681fa8 --- /dev/null +++ b/test/review/SelfReview.test.ts @@ -0,0 +1,33 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { isReviewPassed, normalizeSelfReview } from "../../dist/index.js"; +import { passingReview } from "../helpers.ts"; + +describe("SelfReview", () => { + it("recomputes passing from score, must-fix items, and confidence", () => { + assert.equal(isReviewPassed({ ...passingReview, score: 85, confidence: 0.7, mustFix: [] }, 85), true); + assert.equal(isReviewPassed({ ...passingReview, score: 84, confidence: 1, mustFix: [] }, 85), false); + assert.equal(isReviewPassed({ ...passingReview, score: 100, confidence: 1, mustFix: ["Fix it."] }, 85), false); + assert.equal(isReviewPassed({ ...passingReview, score: 100, confidence: 0.69, mustFix: [] }, 85), false); + }); + + it("overrides an incorrect passed flag from self-review input", () => { + const review = normalizeSelfReview({ ...passingReview, passed: false }, 85); + + assert.equal(review.passed, true); + }); + + it("requires self-review input to match the published schema shape", () => { + assert.throws( + () => normalizeSelfReview({ ...passingReview, passed: undefined }, 85), + /passed must be a boolean/ + ); + assert.throws( + () => normalizeSelfReview({ + ...passingReview, + dimensions: { ...passingReview.dimensions, security: 90 } + }, 85), + /unknown field/ + ); + }); +}); diff --git a/test/types/BuildRequest.test.ts b/test/types/BuildRequest.test.ts new file mode 100644 index 0000000..62f4e09 --- /dev/null +++ b/test/types/BuildRequest.test.ts @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { normalizeAgent, normalizeAgents, normalizeBuildRequest } from "../../dist/index.js"; + +describe("BuildRequest", () => { + it("normalizes build request defaults", () => { + assert.deepEqual(normalizeBuildRequest({ task: " Do work. " }), { + task: "Do work.", + constraints: [], + threshold: 85, + maxIterations: 3 + }); + }); + + it("normalizes agent defaults to codex first", () => { + assert.deepEqual(normalizeAgents(undefined), ["codex", "claude"]); + assert.equal(normalizeAgent(undefined), "codex"); + }); + + it("normalizes custom provider fallbacks to codex first", () => { + assert.deepEqual(normalizeAgents("opencode-go"), ["opencode-go", "codex", "claude"]); + }); + + it("rejects unknown build request fields", () => { + assert.throws( + () => normalizeBuildRequest({ task: "Do work.", extra: true }), + /unknown field/ + ); + }); +}); diff --git a/test/types/BuildResult.test.ts b/test/types/BuildResult.test.ts new file mode 100644 index 0000000..a448035 --- /dev/null +++ b/test/types/BuildResult.test.ts @@ -0,0 +1,98 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { describe, it } from "node:test"; +import { normalizeBuildResult } from "../../dist/index.js"; +import { passingReview } from "../helpers.ts"; + +describe("BuildResult", () => { + it("normalizes build result artifacts with the published schema shape", () => { + const result = normalizeBuildResult({ + status: "ready", + iterations: 1, + taskUnderstanding: { + summary: "Understand the requested behavior before implementation.", + constraints: ["Keep the change focused."] + }, + planSummary: "Implement the requested change.", + changedFiles: ["src/feature.js"], + review: passingReview, + residualNotes: [] + }); + + assert.equal(result.review.passed, true); + assert.deepEqual(result.taskUnderstanding, { + summary: "Understand the requested behavior before implementation.", + constraints: ["Keep the change focused."] + }); + assert.deepEqual(result.discoveredIssues, []); + assert.throws( + () => normalizeBuildResult({ ...result, extra: true }), + /unknown field/ + ); + assert.throws( + () => normalizeBuildResult({ + status: "ready", + iterations: 1, + planSummary: "Implement the requested change.", + changedFiles: ["src/feature.js"], + review: passingReview, + residualNotes: [] + }), + /taskUnderstanding is required/ + ); + assert.throws( + () => normalizeBuildResult({ + ...result, + taskUnderstanding: { + summary: "Understand the requested behavior before implementation." + } + }), + /taskUnderstanding\.constraints is required/ + ); + }); + + it("normalizes discovered issues in build results", () => { + const result = normalizeBuildResult({ + status: "ready", + iterations: 1, + taskUnderstanding: { + summary: "Understand the requested behavior before implementation.", + constraints: ["Keep the change focused."] + }, + planSummary: "Implement the requested change.", + changedFiles: ["src/feature.js"], + review: passingReview, + residualNotes: [], + discoveredIssues: [ + { + title: " Verifier false-positive on legacy status text ", + repo: "verifier", + body: "Observed during the run.", + expected: "The verifier should ignore plain status words in summaries.", + evidence: "verifier.log", + labels: ["kaizen", "kaizen"] + } + ] + }); + + assert.deepEqual(result.discoveredIssues, [ + { + title: "Verifier false-positive on legacy status text", + repo: "verifier", + body: "Observed during the run.", + expected: "The verifier should ignore plain status words in summaries.", + evidence: "verifier.log", + labels: ["kaizen"] + } + ]); + }); + + it("keeps discovered issues optional in the published build result schema", async () => { + const schema = JSON.parse(await readFile("schemas/build-result.schema.json", "utf8")); + + assert.equal(schema.properties.taskUnderstanding.type, "object"); + assert.equal(schema.required.includes("taskUnderstanding"), true); + assert.equal(schema.properties.discoveredIssues.type, "array"); + assert.equal(schema.required.includes("discoveredIssues"), false); + }); +}); diff --git a/test/types/DiscoveredIssue.test.ts b/test/types/DiscoveredIssue.test.ts new file mode 100644 index 0000000..a080dda --- /dev/null +++ b/test/types/DiscoveredIssue.test.ts @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { normalizeDiscoveredIssues } from "../../dist/index.js"; + +describe("DiscoveredIssue", () => { + it("normalizes discovered issue text fields and labels", () => { + assert.deepEqual( + normalizeDiscoveredIssues( + [ + { + title: " Verifier false-positive on legacy status text ", + repo: " verifier ", + body: " Observed during the run. ", + expected: " The verifier should ignore plain status words in summaries. ", + evidence: " verifier.log ", + severity: " P2 ", + labels: ["kaizen", "kaizen", " follow-up "] + } + ], + { label: "discoveredIssues" } + ), + [ + { + title: "Verifier false-positive on legacy status text", + repo: "verifier", + body: "Observed during the run.", + expected: "The verifier should ignore plain status words in summaries.", + evidence: "verifier.log", + severity: "P2", + labels: ["kaizen", "follow-up"] + } + ] + ); + }); + + it("requires each discovered issue title to be present", () => { + assert.throws( + () => normalizeDiscoveredIssues([{ repo: "verifier" }], { label: "discoveredIssues" }), + /discoveredIssues\[0\]\.title/ + ); + }); +}); diff --git a/test/types/KaizenLoopPayload.test.ts b/test/types/KaizenLoopPayload.test.ts new file mode 100644 index 0000000..9e1bc8d --- /dev/null +++ b/test/types/KaizenLoopPayload.test.ts @@ -0,0 +1,72 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { describe, it } from "node:test"; +import { normalizeKaizenLoopPayload } from "../../dist/index.js"; + +describe("KaizenLoopPayload", () => { + it("normalizes kaizen-loop payloads with the published schema shape", () => { + const payload = normalizeKaizenLoopPayload({ + status: "partial", + summary: " Implemented most of the change. ", + notes: "Ran targeted checks.", + discoveredIssues: [ + { + title: " Missing verifier diagnostic ", + repo: " verifier ", + labels: ["kaizen", "kaizen"] + } + ] + }); + + assert.deepEqual(payload, { + status: "partial", + summary: " Implemented most of the change. ", + notes: "Ran targeted checks.", + discoveredIssues: [ + { + title: "Missing verifier diagnostic", + repo: "verifier", + labels: ["kaizen"] + } + ] + }); + }); + + it("rejects malformed kaizen-loop discovered issues explicitly", () => { + assert.throws( + () => normalizeKaizenLoopPayload({ + status: "fixed", + summary: "Implemented.", + notes: "", + discoveredIssues: [{ repo: "verifier" }] + }), + /discoveredIssues\[0\]\.title/ + ); + assert.throws( + () => normalizeKaizenLoopPayload({ + status: "fixed", + summary: "Implemented.", + notes: "", + discoveredIssues: [{ title: "Bad routing", repo: 123 }] + }), + /discoveredIssues\[0\]\.repo must be a string/ + ); + assert.throws( + () => normalizeKaizenLoopPayload({ + status: "blocked", + summary: "Blocked.", + notes: "", + blockedReason: false + }), + /blockedReason must be a string/ + ); + }); + + it("publishes the kaizen-loop payload schema", async () => { + const schema = JSON.parse(await readFile("schemas/kaizen-loop-payload.schema.json", "utf8")); + + assert.deepEqual(schema.properties.status.enum, ["fixed", "partial", "blocked"]); + assert.equal(schema.properties.discoveredIssues.items.properties.repo.type, "string"); + assert.equal(schema.required.includes("discoveredIssues"), false); + }); +}); diff --git a/test/types/contracts.test.ts b/test/types/contracts.test.ts new file mode 100644 index 0000000..c5e61b9 --- /dev/null +++ b/test/types/contracts.test.ts @@ -0,0 +1,67 @@ +import assert from "node:assert/strict"; +import { readFile, stat } from "node:fs/promises"; +import { describe, it } from "node:test"; +import { listFiles } from "../helpers.ts"; + +describe("TypeScript build boundaries", () => { + it("keeps source modules in TypeScript", async () => { + const sourceFiles = await listFiles("src"); + + assert.equal(sourceFiles.some((file) => file.endsWith(".js")), false); + assert.equal(sourceFiles.some((file) => file.endsWith(".ts")), true); + }); + + it("emits declarations for reusable builder contracts and runners", async () => { + const [packageJsonText, entrypoint, contracts, buildRequest, kaizenLoopPayload, builderAgent, agentRunner, kaizenLoop, cli, cliStat] = await Promise.all([ + readFile("package.json", "utf8"), + readFile("dist/index.d.ts", "utf8"), + readFile("dist/types/contracts.d.ts", "utf8"), + readFile("dist/types/BuildRequest.d.ts", "utf8"), + readFile("dist/types/KaizenLoopPayload.d.ts", "utf8"), + readFile("dist/builder/BuilderAgent.d.ts", "utf8"), + readFile("dist/agents/AgentRunner.d.ts", "utf8"), + readFile("dist/kaizen-loop.d.ts", "utf8"), + readFile("dist/cli.js", "utf8"), + stat("dist/cli.js") + ]); + const packageJson = JSON.parse(packageJsonText); + + assert.equal(packageJson.main, "./dist/index.js"); + assert.equal(packageJson.bin["builder-agent"], "./dist/cli.js"); + assert.equal(packageJson.types, "./dist/index.d.ts"); + assert.match(cli, /^#!\/usr\/bin\/env node/); + assert.notEqual(cliStat.mode & 0o111, 0); + assert.match(entrypoint, /export type \{[^}]*BuildRequest[^}]*BuilderAdapter[^}]*\} from "\.\/types\/contracts\.js"/); + assert.match(contracts, /export interface BuilderAdapter/); + assert.match(contracts, /export interface TaskUnderstanding/); + assert.match(contracts, /taskUnderstanding: TaskUnderstanding/); + assert.match(contracts, /export interface KaizenLoopPayload/); + assert.match(buildRequest, /normalizeBuildRequest\(input: BuildRequestInput\): BuildRequest/); + assert.match(kaizenLoopPayload, /normalizeKaizenLoopPayload\(input: unknown\): KaizenLoopPayload/); + assert.match(builderAgent, /build\(input: BuildRequestInput\): Promise/); + assert.match(agentRunner, /runImplementationAgent\([^)]*AgentRunInput[^)]*\): Promise/); + assert.match(kaizenLoop, /runKaizenLoopBuilder\([^)]*KaizenLoopBuilderIO[^)]*\): Promise/); + }); +}); + +describe("builder handoff contract", () => { + it("documents builder handoff as reviewable evidence, not approval", async () => { + const [readme, skill, implementPrompt, selfReviewPrompt, improvePrompt] = await Promise.all([ + readFile("README.md", "utf8"), + readFile("SKILL.md", "utf8"), + readFile("prompts/implement.md", "utf8"), + readFile("prompts/self-review.md", "utf8"), + readFile("prompts/improve.md", "utf8") + ]); + const handoffGuidance = `${readme}\n${skill}\n${implementPrompt}\n${selfReviewPrompt}\n${improvePrompt}`; + + assert.doesNotMatch(readme, /approved task/i); + assert.match(readme, /accepted issue or scoped task/i); + assert.match(handoffGuidance, /what changed/i); + assert.match(handoffGuidance, /why/i); + assert.match(handoffGuidance, /verification run or skipped/i); + assert.match(handoffGuidance, /residual risk/i); + assert.match(handoffGuidance, /reviewer notes/i); + assert.match(handoffGuidance, /not approval/i); + }); +}); From 36d05030a514676968498ebabb38151fa854009e Mon Sep 17 00:00:00 2001 From: "hiraoku.shinichi" Date: Sun, 5 Jul 2026 11:25:45 +0900 Subject: [PATCH 2/2] test: run split TypeScript tests on Node 20 --- .gitignore | 1 + package.json | 2 +- scripts/run-tests.js | 96 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 scripts/run-tests.js diff --git a/.gitignore b/.gitignore index 2353474..bdf0f80 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .serena/ coverage/ node_modules/ +.test-build/ diff --git a/package.json b/package.json index a7d7da9..56f4abc 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "types": "./dist/index.d.ts", "scripts": { "build": "tsc -p tsconfig.json && node -e \"require('node:fs').chmodSync('dist/cli.js', 0o755)\"", - "test": "npm run build && node --test", + "test": "npm run build && node scripts/run-tests.js", "validate:json": "npm run build && node scripts/validate-json.js" }, "files": [ diff --git a/scripts/run-tests.js b/scripts/run-tests.js new file mode 100644 index 0000000..26da137 --- /dev/null +++ b/scripts/run-tests.js @@ -0,0 +1,96 @@ +import { spawn } from "node:child_process"; +import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import ts from "typescript"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const testRoot = join(repoRoot, "test"); +const buildRoot = join(repoRoot, ".test-build"); + +function rewriteSpecifier(specifier) { + if (!specifier.startsWith(".")) { + return specifier; + } + + if (/^(\.\.\/)+dist(\/|$)/.test(specifier)) { + return `../${specifier}`; + } + + if (specifier.endsWith(".ts")) { + return `${specifier.slice(0, -3)}.js`; + } + + return specifier; +} + +function rewriteImports(source) { + return source + .replace(/(from\s+["'])(\.[^"']+)(["'])/g, (_match, prefix, specifier, suffix) => { + return `${prefix}${rewriteSpecifier(specifier)}${suffix}`; + }) + .replace(/(import\(\s*["'])(\.[^"']+)(["']\s*\))/g, (_match, prefix, specifier, suffix) => { + return `${prefix}${rewriteSpecifier(specifier)}${suffix}`; + }); +} + +async function listTypeScriptFiles(root) { + const entries = await readdir(root, { withFileTypes: true }); + const files = []; + + for (const entry of entries) { + const path = join(root, entry.name); + if (entry.isDirectory()) { + files.push(...await listTypeScriptFiles(path)); + } else if (entry.name.endsWith(".ts")) { + files.push(path); + } + } + + return files; +} + +async function transpileTests() { + await rm(buildRoot, { force: true, recursive: true }); + const files = await listTypeScriptFiles(testRoot); + const testFiles = []; + + for (const file of files) { + const source = rewriteImports(await readFile(file, "utf8")); + const output = ts.transpileModule(source, { + compilerOptions: { + module: ts.ModuleKind.ES2022, + moduleResolution: ts.ModuleResolutionKind.NodeNext, + target: ts.ScriptTarget.ES2022 + }, + fileName: file + }); + const outFile = join(buildRoot, relative(repoRoot, file)).replace(/\.ts$/, ".js"); + await mkdir(dirname(outFile), { recursive: true }); + await writeFile(outFile, output.outputText, "utf8"); + if (outFile.endsWith(".test.js")) { + testFiles.push(outFile); + } + } + + return testFiles; +} + +const testFiles = await transpileTests(); +if (testFiles.length === 0) { + console.error("No test files found."); + process.exit(1); +} + +const child = spawn(process.execPath, ["--test", ...testFiles], { + cwd: repoRoot, + stdio: "inherit" +}); + +child.on("exit", (code, signal) => { + if (signal) { + process.kill(process.pid, signal); + return; + } + process.exit(code ?? 1); +});