From 8cb8858ef210829b1807294a70b0b18c6a3782e1 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 23 Jul 2026 21:36:11 +0200 Subject: [PATCH 1/4] FE-1211: Clean up comparison summaries --- .pi/prompts/compare-execution.md | 9 +- src/dev/execution-comparison-operator.ts | 12 +- .../compare-execution-prompt.test.ts | 9 ++ .../__tests__/execution-summary.test.ts | 153 ++++++++++++++++++ .../execution-comparison/execution-summary.ts | 136 ++++++++++++++++ 5 files changed, 317 insertions(+), 2 deletions(-) create mode 100644 src/dev/execution-comparison/__tests__/execution-summary.test.ts create mode 100644 src/dev/execution-comparison/execution-summary.ts diff --git a/.pi/prompts/compare-execution.md b/.pi/prompts/compare-execution.md index 5b22aff0a..d09f145ef 100644 --- a/.pi/prompts/compare-execution.md +++ b/.pi/prompts/compare-execution.md @@ -174,4 +174,11 @@ After all selected lanes terminate and cleanup is proven, write `report.md` besi Keep common evidence and Brunch-only diagnostics visibly separate. Use `not_assessable` for unavailable common evidence. Do not score, rank, choose a winner, claim reliability, infer parity from unavailable or product-private evidence, or let Brunch-only run/Petri/debug data improve its common result. Do not turn ordinary visual hierarchy, clarity, or drag feel into an automatic mechanical verdict. -Finish with the case id, run id, `provenance.json` path, executor order, each attempt id and terminal/validity state, cleanup result, retained output paths, browser report paths, attempt-record paths, and `report.md`. State any remaining human witness plainly. +After `report.md` is complete, generate the terminal summary from the immutable attempt records: + +```sh +npx tsx src/dev/execution-comparison-operator.ts summary \ + --run-directory .fixtures/scratch/execution-comparisons/ +``` + +Return that command's stdout verbatim as the final response. Do not prepend or append commentary, recreate the summary from memory, or finish with only transcript or directory filenames. The deterministic summary must include the case and run ids, each attempt's terminal and validity state, invalidity reason when applicable, command and browser results, cleanup result, and absolute report, oracle, and attempt-record paths. Keep remaining human-witness details and retained output paths in `report.md`. diff --git a/src/dev/execution-comparison-operator.ts b/src/dev/execution-comparison-operator.ts index 24d397adf..304297f5f 100644 --- a/src/dev/execution-comparison-operator.ts +++ b/src/dev/execution-comparison-operator.ts @@ -11,6 +11,10 @@ import { runPetriEditorBrowserOracle, type BrowserOracleReport, } from './execution-comparison/browser-oracle.js'; +import { + formatExecutionComparisonSummary, + loadExecutionComparisonSummary, +} from './execution-comparison/execution-summary.js'; import { runBrunchHostLandingOracle, type HostLandingOracleReport, @@ -280,9 +284,15 @@ export async function runExecutionComparisonOperatorCli(args: readonly string[]) process.stdout.write(`${JSON.stringify({ stored })}\n`); return; } + case 'summary': { + assertOnlyOptions(options, ['run-directory']); + const summary = await loadExecutionComparisonSummary(required(options, 'run-directory')); + process.stdout.write(formatExecutionComparisonSummary(summary, process.cwd())); + return; + } default: throw new Error( - 'Usage: execution-comparison-operator [options]', + 'Usage: execution-comparison-operator [options]', ); } } diff --git a/src/dev/execution-comparison/__tests__/compare-execution-prompt.test.ts b/src/dev/execution-comparison/__tests__/compare-execution-prompt.test.ts index 92fb97a56..d98607609 100644 --- a/src/dev/execution-comparison/__tests__/compare-execution-prompt.test.ts +++ b/src/dev/execution-comparison/__tests__/compare-execution-prompt.test.ts @@ -59,4 +59,13 @@ describe('/compare-execution operator prompt', () => { expect(prompt).toContain('Do not score'); expect(prompt).toContain('diagnostic-only'); }); + + it('finishes with the deterministic retained-attempt summary only', async () => { + const prompt = await readFile(promptPath, 'utf8'); + + expect(prompt).toContain('execution-comparison-operator.ts summary'); + expect(prompt).toContain("Return that command's stdout verbatim as the final response"); + expect(prompt).toContain('invalidity reason when applicable'); + expect(prompt).toContain('Do not prepend or append commentary'); + }); }); diff --git a/src/dev/execution-comparison/__tests__/execution-summary.test.ts b/src/dev/execution-comparison/__tests__/execution-summary.test.ts new file mode 100644 index 000000000..e13241c6c --- /dev/null +++ b/src/dev/execution-comparison/__tests__/execution-summary.test.ts @@ -0,0 +1,153 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { runExecutionComparisonOperatorCli } from '../../execution-comparison-operator.js'; +import type { ExecutionAttempt } from '../artifact-contract.js'; +import { formatExecutionComparisonSummary, loadExecutionComparisonSummary } from '../execution-summary.js'; + +const roots: string[] = []; + +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(roots.splice(0).map(async (root) => await rm(root, { recursive: true, force: true }))); +}); + +describe('execution comparison final summary', () => { + it('loads immutable attempts and explains invalidity in a concise final summary', async () => { + const runDirectory = await createRun([ + attempt({ + attemptId: 'brunch-attempt-1', + lane: 'brunch', + startedAt: '2026-07-23T10:00:00.000Z', + validity: { status: 'invalid', reasons: ['The run was landed after promotion preparation.'] }, + terminal: { + outcome: 'invalid', + reason: 'landed out of bounds', + productStatus: 'promotion_prepared', + }, + commands: [command('test', 'passed'), command('build', 'failed')], + }), + attempt({ + attemptId: 'claude-code-attempt-1', + lane: 'claude_code', + startedAt: '2026-07-23T11:00:00.000Z', + }), + ]); + + const summary = await loadExecutionComparisonSummary(runDirectory); + const rendered = formatExecutionComparisonSummary(summary, runDirectory); + + expect(rendered).toContain('Execution comparison complete'); + expect(rendered).toContain('Case: minimal-petri-net-editor-v1'); + expect(rendered).toContain('Brunch\n Status: invalid\n Terminal: promotion_prepared'); + expect(rendered).toContain('Checks: test passed, build failed, browser not run'); + expect(rendered).toContain('Reason: The run was landed after promotion preparation.'); + expect(rendered).toContain('Claude Code\n Status: valid\n Terminal: promotion_prepared'); + expect(rendered).not.toContain('cleanup clean'); + expect(rendered).toContain('Cleanup: done'); + expect(rendered).toContain(`- Report: ${join(runDirectory, 'report.md')}`); + expect(rendered).toContain(join(runDirectory, 'attempt-records/brunch-attempt-1/attempt.json')); + }); + + it('prints the deterministic summary through the operator command', async () => { + const runDirectory = await createRun([attempt({ attemptId: 'claude-code-attempt-2' })]); + const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + + await runExecutionComparisonOperatorCli(['summary', '--run-directory', runDirectory]); + + expect(stdout).toHaveBeenCalledOnce(); + expect(stdout.mock.calls[0]?.[0]).toContain('Execution comparison complete'); + expect(stdout.mock.calls[0]?.[0]).toContain('Claude Code\n Status: valid'); + }); + + it('refuses to summarize before the final report exists', async () => { + const runDirectory = await mkdtemp(join(tmpdir(), 'brunch-execution-summary-')); + roots.push(runDirectory); + + await expect(loadExecutionComparisonSummary(runDirectory)).rejects.toThrow( + 'execution comparison report is missing', + ); + }); +}); + +async function createRun(attempts: readonly ExecutionAttempt[]): Promise { + const runDirectory = await mkdtemp(join(tmpdir(), 'brunch-execution-summary-')); + roots.push(runDirectory); + await writeFile(join(runDirectory, 'report.md'), '# Report\n'); + for (const selected of attempts) { + const attemptDirectory = join(runDirectory, 'attempt-records', selected.attemptId); + await mkdir(attemptDirectory, { recursive: true }); + await writeFile(join(attemptDirectory, 'attempt.json'), `${JSON.stringify(selected)}\n`); + } + return runDirectory; +} + +function attempt(overrides: Partial): ExecutionAttempt { + return { + schemaVersion: 1, + attemptId: 'claude-code-attempt-1', + caseId: 'minimal-petri-net-editor-v1', + lane: 'claude_code', + publicPacketSha256: `sha256:${'a'.repeat(64)}`, + oraclePackSha256: `sha256:${'b'.repeat(64)}`, + startedAt: '2026-07-23T12:00:00.000Z', + endedAt: '2026-07-23T12:10:00.000Z', + budget: { + elapsedMinutes: 90, + mechanicalInterventions: 2, + substantiveHumanInterventions: 0, + }, + versions: { + product: '@hashintel/brunch@1.0.0-alpha.11', + provider: 'anthropic', + model: 'claude-opus-4-8', + harness: 'execution-comparison/v1', + actorRecipe: 'claude-code/v1', + node: '24.18.0', + npm: '11.6.2', + os: 'darwin', + architecture: 'arm64', + }, + repository: { + baseSha: '1'.repeat(40), + reviewSha: '2'.repeat(40), + finalGitRange: `${'1'.repeat(40)}..${'2'.repeat(40)}`, + }, + terminal: { + outcome: 'success', + reason: 'promotion prepared', + productStatus: 'promotion_prepared', + }, + validity: { status: 'valid', reasons: [] }, + commands: [command('test', 'passed'), command('build', 'passed')], + browser: { status: 'not_run', reportPath: 'browser/report.json' }, + interventions: [], + commonMetrics: { + elapsedMs: 600_000, + inputTokens: 'not_assessable', + outputTokens: 'not_assessable', + costUsd: 'not_assessable', + permissionPrompts: 'not_assessable', + }, + evidence: { + finalTreePath: 'repository/final-tree.txt', + finalDiffPath: 'repository/final.diff', + visibleProcessPath: 'process/visible.jsonl', + }, + cleanup: { status: 'clean', liveProcesses: 0, liveSessions: 0 }, + ...overrides, + }; +} + +function command(id: string, status: 'passed' | 'failed' | 'not_run'): ExecutionAttempt['commands'][number] { + return { + id, + status, + exitCode: status === 'passed' ? 0 : status === 'failed' ? 1 : 'not_assessable', + stdoutPath: `commands/${id}.stdout.txt`, + stderrPath: `commands/${id}.stderr.txt`, + }; +} diff --git a/src/dev/execution-comparison/execution-summary.ts b/src/dev/execution-comparison/execution-summary.ts new file mode 100644 index 000000000..28fdc3824 --- /dev/null +++ b/src/dev/execution-comparison/execution-summary.ts @@ -0,0 +1,136 @@ +import { lstat, readFile, readdir } from 'node:fs/promises'; +import { basename, join, resolve } from 'node:path'; + +import { parseExecutionAttempt, type ExecutionAttempt } from './artifact-contract.js'; + +export interface ExecutionComparisonAttemptSummary { + readonly attempt: ExecutionAttempt; + readonly recordPath: string; +} + +export interface ExecutionComparisonSummary { + readonly runDirectory: string; + readonly runId: string; + readonly caseId: string; + readonly reportPath: string; + readonly attempts: readonly ExecutionComparisonAttemptSummary[]; +} + +export async function loadExecutionComparisonSummary( + runDirectoryInput: string, +): Promise { + const runDirectory = resolve(runDirectoryInput); + const reportPath = join(runDirectory, 'report.md'); + if (!(await isRegularFile(reportPath))) { + throw new Error(`execution comparison report is missing: ${reportPath}`); + } + + const attemptsRoot = join(runDirectory, 'attempt-records'); + const entries = await readdir(attemptsRoot, { withFileTypes: true }); + const attempts: ExecutionComparisonAttemptSummary[] = []; + for (const entry of entries.sort((left, right) => codePointCompare(left.name, right.name))) { + if (!entry.isDirectory() || entry.isSymbolicLink()) continue; + const recordPath = join(attemptsRoot, entry.name, 'attempt.json'); + const value = JSON.parse(await readFile(recordPath, 'utf8')) as unknown; + const attempt = parseExecutionAttempt(value); + if (attempt.attemptId !== entry.name) { + throw new Error(`execution attempt directory does not match attempt id: ${entry.name}`); + } + attempts.push({ attempt, recordPath }); + } + if (attempts.length === 0) { + throw new Error(`execution comparison has no retained attempts: ${attemptsRoot}`); + } + + attempts.sort((left, right) => { + const byStart = codePointCompare(left.attempt.startedAt, right.attempt.startedAt); + return byStart === 0 ? codePointCompare(left.attempt.attemptId, right.attempt.attemptId) : byStart; + }); + const caseId = attempts[0]!.attempt.caseId; + const publicPacketSha256 = attempts[0]!.attempt.publicPacketSha256; + const oraclePackSha256 = attempts[0]!.attempt.oraclePackSha256; + if ( + attempts.some( + ({ attempt }) => + attempt.caseId !== caseId || + attempt.publicPacketSha256 !== publicPacketSha256 || + attempt.oraclePackSha256 !== oraclePackSha256, + ) + ) { + throw new Error('execution comparison attempts do not share one case and frozen evidence set'); + } + + return { + runDirectory, + runId: basename(runDirectory), + caseId, + reportPath, + attempts, + }; +} + +export function formatExecutionComparisonSummary( + summary: ExecutionComparisonSummary, + baseDirectory: string, +): string { + const lines = [ + 'Execution comparison complete', + '=============================', + '', + `Case: ${summary.caseId}`, + `Run: ${summary.runId}`, + '', + 'Results', + '-------', + ]; + + for (const { attempt } of summary.attempts) { + const commands = attempt.commands + .map((command) => `${command.id} ${displayStatus(command.status)}`) + .join(', '); + lines.push('', laneLabel(attempt.lane)); + lines.push(` Status: ${attempt.validity.status}`); + lines.push(` Terminal: ${attempt.terminal.productStatus}`); + lines.push(` Checks: ${commands}, browser ${displayStatus(attempt.browser.status)}`); + if (attempt.validity.status === 'invalid') { + lines.push(` Reason: ${attempt.validity.reasons[0]}`); + } + } + + const cleanup = summary.attempts.every(({ attempt }) => attempt.cleanup.status === 'clean') + ? 'done' + : 'residue'; + lines.push('', `Cleanup: ${cleanup}`, '', 'Artifacts', '---------'); + lines.push(`- Report: ${absolutePath(summary.reportPath, baseDirectory)}`); + for (const { attempt, recordPath } of summary.attempts) { + lines.push(`- ${laneLabel(attempt.lane)} attempt: ${absolutePath(recordPath, baseDirectory)}`); + lines.push( + `- ${laneLabel(attempt.lane)} oracle: ${absolutePath(attempt.browser.reportPath, baseDirectory)}`, + ); + } + return `${lines.join('\n')}\n`; +} + +function displayStatus(status: 'passed' | 'failed' | 'not_run'): string { + return status === 'not_run' ? 'not run' : status; +} + +function laneLabel(lane: ExecutionAttempt['lane']): string { + return lane === 'brunch' ? 'Brunch' : 'Claude Code'; +} + +function absolutePath(path: string, baseDirectory: string): string { + return resolve(baseDirectory, path); +} + +async function isRegularFile(path: string): Promise { + try { + return (await lstat(path)).isFile(); + } catch { + return false; + } +} + +function codePointCompare(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} From ef93f7478bb6bac4bc094735b2b77c6f631914db Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 23 Jul 2026 21:48:29 +0200 Subject: [PATCH 2/4] FE-1211: Record no-release intent --- .changeset/calm-summaries-finish.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .changeset/calm-summaries-finish.md diff --git a/.changeset/calm-summaries-finish.md b/.changeset/calm-summaries-finish.md new file mode 100644 index 000000000..070fea22a --- /dev/null +++ b/.changeset/calm-summaries-finish.md @@ -0,0 +1,4 @@ +--- +--- + +Improve the project-local execution comparison summary. From 9a64e0c68627f60df213abc9121b26ac953d697a Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 24 Jul 2026 10:15:23 +0200 Subject: [PATCH 3/4] FE-1211: Resolve retained oracle paths --- .../__tests__/execution-summary.test.ts | 41 ++++++++++++++++--- .../execution-comparison/execution-summary.ts | 27 ++++++++++-- 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/src/dev/execution-comparison/__tests__/execution-summary.test.ts b/src/dev/execution-comparison/__tests__/execution-summary.test.ts index e13241c6c..6ad0164fb 100644 --- a/src/dev/execution-comparison/__tests__/execution-summary.test.ts +++ b/src/dev/execution-comparison/__tests__/execution-summary.test.ts @@ -1,6 +1,6 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { join, relative } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; @@ -38,7 +38,7 @@ describe('execution comparison final summary', () => { ]); const summary = await loadExecutionComparisonSummary(runDirectory); - const rendered = formatExecutionComparisonSummary(summary, runDirectory); + const rendered = formatExecutionComparisonSummary(summary, join(runDirectory, '..')); expect(rendered).toContain('Execution comparison complete'); expect(rendered).toContain('Case: minimal-petri-net-editor-v1'); @@ -50,6 +50,33 @@ describe('execution comparison final summary', () => { expect(rendered).toContain('Cleanup: done'); expect(rendered).toContain(`- Report: ${join(runDirectory, 'report.md')}`); expect(rendered).toContain(join(runDirectory, 'attempt-records/brunch-attempt-1/attempt.json')); + expect(rendered).toContain( + `- Brunch oracle: ${join(runDirectory, 'lanes/brunch/attempt-staging/browser/report.json')}`, + ); + expect(rendered).toContain( + `- Claude Code oracle: ${join(runDirectory, 'lanes/claude_code/attempt-staging/browser/report.json')}`, + ); + }); + + it('preserves a repository-relative oracle path already rooted inside the run', async () => { + const runDirectory = await createRun([]); + const baseDirectory = join(runDirectory, '..'); + const oraclePath = join(runDirectory, 'lanes/claude_code/attempt-staging/browser/report.json'); + await writeAttempt( + runDirectory, + attempt({ + attemptId: 'claude-code-attempt-3', + browser: { + status: 'passed', + reportPath: relative(baseDirectory, oraclePath), + }, + }), + ); + + const summary = await loadExecutionComparisonSummary(runDirectory); + const rendered = formatExecutionComparisonSummary(summary, baseDirectory); + + expect(rendered).toContain(`- Claude Code oracle: ${oraclePath}`); }); it('prints the deterministic summary through the operator command', async () => { @@ -78,13 +105,17 @@ async function createRun(attempts: readonly ExecutionAttempt[]): Promise roots.push(runDirectory); await writeFile(join(runDirectory, 'report.md'), '# Report\n'); for (const selected of attempts) { - const attemptDirectory = join(runDirectory, 'attempt-records', selected.attemptId); - await mkdir(attemptDirectory, { recursive: true }); - await writeFile(join(attemptDirectory, 'attempt.json'), `${JSON.stringify(selected)}\n`); + await writeAttempt(runDirectory, selected); } return runDirectory; } +async function writeAttempt(runDirectory: string, selected: ExecutionAttempt): Promise { + const attemptDirectory = join(runDirectory, 'attempt-records', selected.attemptId); + await mkdir(attemptDirectory, { recursive: true }); + await writeFile(join(attemptDirectory, 'attempt.json'), `${JSON.stringify(selected)}\n`); +} + function attempt(overrides: Partial): ExecutionAttempt { return { schemaVersion: 1, diff --git a/src/dev/execution-comparison/execution-summary.ts b/src/dev/execution-comparison/execution-summary.ts index 28fdc3824..1ee6d050d 100644 --- a/src/dev/execution-comparison/execution-summary.ts +++ b/src/dev/execution-comparison/execution-summary.ts @@ -1,5 +1,5 @@ import { lstat, readFile, readdir } from 'node:fs/promises'; -import { basename, join, resolve } from 'node:path'; +import { basename, isAbsolute, join, normalize, relative, resolve, sep } from 'node:path'; import { parseExecutionAttempt, type ExecutionAttempt } from './artifact-contract.js'; @@ -104,9 +104,7 @@ export function formatExecutionComparisonSummary( lines.push(`- Report: ${absolutePath(summary.reportPath, baseDirectory)}`); for (const { attempt, recordPath } of summary.attempts) { lines.push(`- ${laneLabel(attempt.lane)} attempt: ${absolutePath(recordPath, baseDirectory)}`); - lines.push( - `- ${laneLabel(attempt.lane)} oracle: ${absolutePath(attempt.browser.reportPath, baseDirectory)}`, - ); + lines.push(`- ${laneLabel(attempt.lane)} oracle: ${absoluteOraclePath(summary, attempt, baseDirectory)}`); } return `${lines.join('\n')}\n`; } @@ -123,6 +121,27 @@ function absolutePath(path: string, baseDirectory: string): string { return resolve(baseDirectory, path); } +function absoluteOraclePath( + summary: ExecutionComparisonSummary, + attempt: ExecutionAttempt, + baseDirectory: string, +): string { + const reportPath = normalize(attempt.browser.reportPath); + if (reportPath.startsWith(`browser${sep}`)) { + return resolve(summary.runDirectory, 'lanes', attempt.lane, 'attempt-staging', reportPath); + } + + const repositoryRelative = absolutePath(reportPath, baseDirectory); + return inside(summary.runDirectory, repositoryRelative) + ? repositoryRelative + : resolve(summary.runDirectory, reportPath); +} + +function inside(parent: string, child: string): boolean { + const selected = relative(parent, child); + return selected === '' || (!isAbsolute(selected) && selected !== '..' && !selected.startsWith(`..${sep}`)); +} + async function isRegularFile(path: string): Promise { try { return (await lstat(path)).isFile(); From 38c6bd94f882bab8d068e5b641e36161eaff9171 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 24 Jul 2026 10:28:29 +0200 Subject: [PATCH 4/4] FE-1211: Report comparison outcomes --- .../__tests__/execution-summary.test.ts | 29 +++++++++++++++++-- .../execution-comparison/execution-summary.ts | 1 + 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/dev/execution-comparison/__tests__/execution-summary.test.ts b/src/dev/execution-comparison/__tests__/execution-summary.test.ts index 6ad0164fb..b7091d02f 100644 --- a/src/dev/execution-comparison/__tests__/execution-summary.test.ts +++ b/src/dev/execution-comparison/__tests__/execution-summary.test.ts @@ -42,10 +42,14 @@ describe('execution comparison final summary', () => { expect(rendered).toContain('Execution comparison complete'); expect(rendered).toContain('Case: minimal-petri-net-editor-v1'); - expect(rendered).toContain('Brunch\n Status: invalid\n Terminal: promotion_prepared'); + expect(rendered).toContain( + 'Brunch\n Status: invalid\n Outcome: invalid\n Terminal: promotion_prepared', + ); expect(rendered).toContain('Checks: test passed, build failed, browser not run'); expect(rendered).toContain('Reason: The run was landed after promotion preparation.'); - expect(rendered).toContain('Claude Code\n Status: valid\n Terminal: promotion_prepared'); + expect(rendered).toContain( + 'Claude Code\n Status: valid\n Outcome: success\n Terminal: promotion_prepared', + ); expect(rendered).not.toContain('cleanup clean'); expect(rendered).toContain('Cleanup: done'); expect(rendered).toContain(`- Report: ${join(runDirectory, 'report.md')}`); @@ -79,6 +83,25 @@ describe('execution comparison final summary', () => { expect(rendered).toContain(`- Claude Code oracle: ${oraclePath}`); }); + it('distinguishes a valid failed attempt from a successful attempt', async () => { + const runDirectory = await createRun([ + attempt({ + attemptId: 'claude-code-failure-1', + terminal: { + outcome: 'failure', + reason: 'executor stopped before delivery', + productStatus: 'not_assessable', + }, + }), + ]); + + const summary = await loadExecutionComparisonSummary(runDirectory); + const rendered = formatExecutionComparisonSummary(summary, join(runDirectory, '..')); + + expect(rendered).toContain('Status: valid\n Outcome: failure\n Terminal: not_assessable'); + expect(rendered).not.toContain('Outcome: success'); + }); + it('prints the deterministic summary through the operator command', async () => { const runDirectory = await createRun([attempt({ attemptId: 'claude-code-attempt-2' })]); const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); @@ -87,7 +110,7 @@ describe('execution comparison final summary', () => { expect(stdout).toHaveBeenCalledOnce(); expect(stdout.mock.calls[0]?.[0]).toContain('Execution comparison complete'); - expect(stdout.mock.calls[0]?.[0]).toContain('Claude Code\n Status: valid'); + expect(stdout.mock.calls[0]?.[0]).toContain('Claude Code\n Status: valid\n Outcome: success'); }); it('refuses to summarize before the final report exists', async () => { diff --git a/src/dev/execution-comparison/execution-summary.ts b/src/dev/execution-comparison/execution-summary.ts index 1ee6d050d..3b39f8825 100644 --- a/src/dev/execution-comparison/execution-summary.ts +++ b/src/dev/execution-comparison/execution-summary.ts @@ -90,6 +90,7 @@ export function formatExecutionComparisonSummary( .join(', '); lines.push('', laneLabel(attempt.lane)); lines.push(` Status: ${attempt.validity.status}`); + lines.push(` Outcome: ${attempt.terminal.outcome}`); lines.push(` Terminal: ${attempt.terminal.productStatus}`); lines.push(` Checks: ${commands}, browser ${displayStatus(attempt.browser.status)}`); if (attempt.validity.status === 'invalid') {