From 371d2fc43d238cc6645df432373dc0ef8995f297 Mon Sep 17 00:00:00 2001 From: "hiraoku.shinichi" Date: Sun, 5 Jul 2026 10:44:12 +0900 Subject: [PATCH 1/3] =?UTF-8?q?kaizen:=20AgentRunner=E3=81=AE=E5=A4=B1?= =?UTF-8?q?=E6=95=97=E5=88=86=E9=A1=9E=E3=81=AB=E3=81=A4=E3=81=84=E3=81=A6?= =?UTF-8?q?=E3=80=81command=5Fmissing/timeout/rate=5Flimited=E3=81=AE?= =?UTF-8?q?=E6=9C=AA=E7=B6=B2=E7=BE=85=E3=82=B1=E3=83=BC=E3=82=B9=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E3=81=97=E3=81=BE=E3=81=97=E3=81=9F=E3=80=82?= =?UTF-8?q?=20(#76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/builder-agent.test.js | 143 +++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/test/builder-agent.test.js b/test/builder-agent.test.js index 77ea753..3df90a5 100644 --- a/test/builder-agent.test.js +++ b/test/builder-agent.test.js @@ -1069,6 +1069,37 @@ console.log(JSON.stringify({ assert.match(result.notes, /Selected backend: claude/); }); + it("falls back for command-missing, timeout, and rate-limited failures when fallbackOn opts in", async () => { + for (const failureCase of failureClassificationCases) { + const { result } = await runFailureClassificationFixture({ + failureCase, + fallbackOn: [failureCase.failureClass] + }); + + assert.equal(result.status, "fixed", failureCase.name); + assert.equal(result.summary, "implemented after classified fallback", failureCase.name); + assert.match(result.notes, failureCase.expectedEvidence, failureCase.name); + assert.match(result.notes, /hermes-agent: exitCode=1, status=fallback/, failureCase.name); + assert.match(result.notes, /claude: exitCode=0, status=selected, failureClass=none/, failureCase.name); + assert.match(result.notes, /Selected backend: claude/, failureCase.name); + } + }); + + it("stops fallback for command-missing, timeout, and rate-limited failures when fallbackOn opts out", async () => { + for (const failureCase of failureClassificationCases) { + const { result, error } = await runFailureClassificationFixture({ + failureCase, + fallbackOn: [] + }); + + assert.match(error.message, /Command exited with 2/, failureCase.name); + assert.equal(result.status, "blocked", failureCase.name); + assert.match(result.notes, failureCase.expectedEvidence, failureCase.name); + assert.match(result.notes, /hermes-agent: exitCode=1, status=stopped/, failureCase.name); + assert.doesNotMatch(result.notes, /implemented after classified fallback/, failureCase.name); + } + }); + it("stops fallback for provider-blocked failures unless the provider opts in", async () => { const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); const binDir = join(dir, "bin"); @@ -1304,6 +1335,57 @@ console.log(JSON.stringify({ }); }); +const failureClassificationCases = [ + { + name: "ENOENT error code", + failureClass: "command_missing", + missingCommand: true, + expectedEvidence: /hermes-agent: exitCode=1, status=(fallback|stopped), failureClass=command_missing/ + }, + { + name: "command not found raw output", + failureClass: "command_missing", + stderr: "shell: command not found: missing-provider", + expectedEvidence: /hermes-agent: exitCode=1, status=(fallback|stopped), failureClass=command_missing/ + }, + { + name: "timed out raw output", + failureClass: "timeout", + stderr: "agent timed out while waiting for a response", + expectedEvidence: /hermes-agent: exitCode=1, status=(fallback|stopped), failureClass=timeout/ + }, + { + name: "timeout raw output", + failureClass: "timeout", + stderr: "agent timeout exceeded", + expectedEvidence: /hermes-agent: exitCode=1, status=(fallback|stopped), failureClass=timeout/ + }, + { + name: "429 raw output", + failureClass: "rate_limited", + stderr: "provider returned HTTP 429", + expectedEvidence: /hermes-agent: exitCode=1, status=(fallback|stopped), failureClass=rate_limited/ + }, + { + name: "rate limit raw output", + failureClass: "rate_limited", + stderr: "provider rate limit reached", + expectedEvidence: /hermes-agent: exitCode=1, status=(fallback|stopped), failureClass=rate_limited/ + }, + { + name: "too many requests raw output", + failureClass: "rate_limited", + stderr: "too many requests for this account", + expectedEvidence: /hermes-agent: exitCode=1, status=(fallback|stopped), failureClass=rate_limited/ + }, + { + name: "quota exceeded raw output", + failureClass: "rate_limited", + stderr: "quota exceeded for this project", + expectedEvidence: /hermes-agent: exitCode=1, status=(fallback|stopped), failureClass=rate_limited/ + } +]; + function createAdapter({ reviews }) { const calls = { improve: 0 @@ -1341,6 +1423,67 @@ function createAdapter({ reviews }) { }; } +async function runFailureClassificationFixture({ failureCase, fallbackOn }) { + const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); + const binDir = join(dir, "bin"); + const resultPath = join(dir, "build-result.json"); + await mkdir(binDir); + + const providerCommand = failureCase.missingCommand ? "missing-builder-agent-command" : "hermes-agent"; + if (!failureCase.missingCommand) { + const fakeHermesPath = join(binDir, "hermes-agent"); + await writeFile( + fakeHermesPath, + `#!/usr/bin/env node +console.error(${JSON.stringify(failureCase.stderr)}); +process.exit(1); +`, + "utf8" + ); + await chmod(fakeHermesPath, 0o755); + } + + const fakeClaudePath = join(binDir, "claude"); + await writeFile( + fakeClaudePath, + `#!/usr/bin/env node +console.log(JSON.stringify({ + result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"implemented after classified fallback\",\"notes\":\"checked\"}\n```")} +})); +`, + "utf8" + ); + await chmod(fakeClaudePath, 0o755); + + let stdout = ""; + let error; + try { + const output = await spawnWithInput(process.execPath, ["dist/cli.js"], "Fix issue #1", { + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}`, + KAIZEN_BUILD_RESULT_PATH: resultPath, + KAIZEN_WORKSPACE_DIR: dir, + KAIZEN_PREFERRED_AGENT: "hermes-agent,claude", + KAIZEN_AGENT_PROVIDERS: JSON.stringify({ + "hermes-agent": { + command: providerCommand, + args: ["run", "{{prompt}}"], + fallbackOn, + output: "stdout" + } + }) + } + }); + stdout = output.stdout; + } catch (caught) { + error = caught; + } + + const result = JSON.parse(await readFile(resultPath, "utf8")); + return { stdout, result, error }; +} + function spawnWithInput(command, args, input, options) { return new Promise((resolve, reject) => { const child = spawn(command, args, { From 75c2aa72f34db3ebe33dfba078b36bf3890a321b Mon Sep 17 00:00:00 2001 From: "hiraoku.shinichi" Date: Sun, 5 Jul 2026 11:26:50 +0900 Subject: [PATCH 2/3] test: assert fallback success exits cleanly --- test/builder-agent.test.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/builder-agent.test.js b/test/builder-agent.test.js index 3df90a5..5ba4c2a 100644 --- a/test/builder-agent.test.js +++ b/test/builder-agent.test.js @@ -1071,11 +1071,12 @@ console.log(JSON.stringify({ it("falls back for command-missing, timeout, and rate-limited failures when fallbackOn opts in", async () => { for (const failureCase of failureClassificationCases) { - const { result } = await runFailureClassificationFixture({ + const { result, error } = await runFailureClassificationFixture({ failureCase, fallbackOn: [failureCase.failureClass] }); + assert.equal(error, undefined, failureCase.name); assert.equal(result.status, "fixed", failureCase.name); assert.equal(result.summary, "implemented after classified fallback", failureCase.name); assert.match(result.notes, failureCase.expectedEvidence, failureCase.name); From aba2d84b1096292fbcfecce06683bc42eeee9b8f Mon Sep 17 00:00:00 2001 From: "hiraoku.shinichi" Date: Sun, 5 Jul 2026 13:09:57 +0900 Subject: [PATCH 3/3] test: clean up fallback classification fixtures --- test/agents/AgentRunner.test.ts | 89 ++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 41 deletions(-) diff --git a/test/agents/AgentRunner.test.ts b/test/agents/AgentRunner.test.ts index 434f6e0..4a6c13a 100644 --- a/test/agents/AgentRunner.test.ts +++ b/test/agents/AgentRunner.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, it } from "node:test"; @@ -407,8 +407,7 @@ console.log(JSON.stringify({ assert.equal(result.exitCode, 0, failureCase.name); assert.equal(result.payload.status, "fixed", failureCase.name); assert.equal(result.payload.summary, "implemented after classified fallback", failureCase.name); - assert.match(result.payload.notes, failureCase.expectedEvidence, failureCase.name); - assert.match(result.payload.notes, /hermes-agent: exitCode=1, status=fallback/, failureCase.name); + assertClassifiedProviderEvidence(result.payload.notes, failureCase, "fallback"); assert.match(result.payload.notes, /claude: exitCode=0, status=selected, failureClass=none/, failureCase.name); assert.match(result.payload.notes, /Selected backend: claude/, failureCase.name); } @@ -422,8 +421,7 @@ console.log(JSON.stringify({ }); assert.equal(result.payload, undefined, failureCase.name); - assert.match(result.providerEvidence, failureCase.expectedEvidence, failureCase.name); - assert.match(result.providerEvidence, /hermes-agent: exitCode=1, status=stopped/, failureCase.name); + assertClassifiedProviderEvidence(result.providerEvidence, failureCase, "stopped"); assert.doesNotMatch(result.raw, /implemented after classified fallback/, failureCase.name); } }); @@ -668,49 +666,58 @@ const failureClassificationCases = [ async function runFailureClassificationFixture({ failureCase, fallbackOn }) { const dir = await mkdtemp(join(tmpdir(), "builder-agent-")); const binDir = join(dir, "bin"); - await mkdir(binDir); + try { + await mkdir(binDir); - const providerCommand = failureCase.missingCommand ? "missing-builder-agent-command" : "hermes-agent"; - if (!failureCase.missingCommand) { - const fakeHermesPath = join(binDir, "hermes-agent"); - await writeFile( - fakeHermesPath, - `#!/usr/bin/env node + const providerCommand = failureCase.missingCommand ? "missing-builder-agent-command" : "hermes-agent"; + if (!failureCase.missingCommand) { + const fakeHermesPath = join(binDir, "hermes-agent"); + await writeFile( + fakeHermesPath, + `#!/usr/bin/env node console.error(${JSON.stringify(failureCase.stderr)}); process.exit(1); `, - "utf8" - ); - await chmod(fakeHermesPath, 0o755); - } + "utf8" + ); + await chmod(fakeHermesPath, 0o755); + } - const fakeClaudePath = join(binDir, "claude"); - await writeFile( - fakeClaudePath, - `#!/usr/bin/env node + const fakeClaudePath = join(binDir, "claude"); + await writeFile( + fakeClaudePath, + `#!/usr/bin/env node console.log(JSON.stringify({ result: ${JSON.stringify("```json\n{\"status\":\"fixed\",\"summary\":\"implemented after classified fallback\",\"notes\":\"checked\"}\n```")} })); `, - "utf8" - ); - await chmod(fakeClaudePath, 0o755); - - return runImplementationAgent({ - agent: "hermes-agent,claude", - prompt: "Fix issue #1", - workspaceDir: dir, - env: { - ...process.env, - PATH: `${binDir}:${process.env.PATH}`, - KAIZEN_AGENT_PROVIDERS: JSON.stringify({ - "hermes-agent": { - command: providerCommand, - args: ["run", "{{prompt}}"], - fallbackOn, - output: "stdout" - } - }) - } - }); + "utf8" + ); + await chmod(fakeClaudePath, 0o755); + + return await runImplementationAgent({ + agent: "hermes-agent,claude", + prompt: "Fix issue #1", + workspaceDir: dir, + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}`, + KAIZEN_AGENT_PROVIDERS: JSON.stringify({ + "hermes-agent": { + command: providerCommand, + args: ["run", "{{prompt}}"], + fallbackOn, + output: "stdout" + } + }) + } + }); + } finally { + await rm(dir, { force: true, recursive: true }); + } +} + +function assertClassifiedProviderEvidence(evidence, failureCase, status) { + assert.match(evidence, failureCase.expectedEvidence, failureCase.name); + assert.match(evidence, new RegExp(`hermes-agent: exitCode=1, status=${status}`), failureCase.name); }