diff --git a/.changeset/calm-summaries-finish.md b/.changeset/calm-summaries-finish.md new file mode 100644 index 000000000..070fea22a --- /dev/null +++ b/.changeset/calm-summaries-finish.md @@ -0,0 +1,4 @@ +--- +--- + +Improve the project-local execution comparison summary. diff --git a/.pi/prompts/compare-execution.md b/.pi/prompts/compare-execution.md index 5b22aff0a..d09f145ef 100644 --- a/.pi/prompts/compare-execution.md +++ b/.pi/prompts/compare-execution.md @@ -174,4 +174,11 @@ After all selected lanes terminate and cleanup is proven, write `report.md` besi Keep common evidence and Brunch-only diagnostics visibly separate. Use `not_assessable` for unavailable common evidence. Do not score, rank, choose a winner, claim reliability, infer parity from unavailable or product-private evidence, or let Brunch-only run/Petri/debug data improve its common result. Do not turn ordinary visual hierarchy, clarity, or drag feel into an automatic mechanical verdict. -Finish with the case id, run id, `provenance.json` path, executor order, each attempt id and terminal/validity state, cleanup result, retained output paths, browser report paths, attempt-record paths, and `report.md`. State any remaining human witness plainly. +After `report.md` is complete, generate the terminal summary from the immutable attempt records: + +```sh +npx tsx src/dev/execution-comparison-operator.ts summary \ + --run-directory .fixtures/scratch/execution-comparisons/ +``` + +Return that command's stdout verbatim as the final response. Do not prepend or append commentary, recreate the summary from memory, or finish with only transcript or directory filenames. The deterministic summary must include the case and run ids, each attempt's terminal and validity state, invalidity reason when applicable, command and browser results, cleanup result, and absolute report, oracle, and attempt-record paths. Keep remaining human-witness details and retained output paths in `report.md`. diff --git a/src/dev/execution-comparison-operator.ts b/src/dev/execution-comparison-operator.ts index 24d397adf..304297f5f 100644 --- a/src/dev/execution-comparison-operator.ts +++ b/src/dev/execution-comparison-operator.ts @@ -11,6 +11,10 @@ import { runPetriEditorBrowserOracle, type BrowserOracleReport, } from './execution-comparison/browser-oracle.js'; +import { + formatExecutionComparisonSummary, + loadExecutionComparisonSummary, +} from './execution-comparison/execution-summary.js'; import { runBrunchHostLandingOracle, type HostLandingOracleReport, @@ -280,9 +284,15 @@ export async function runExecutionComparisonOperatorCli(args: readonly string[]) process.stdout.write(`${JSON.stringify({ stored })}\n`); return; } + case 'summary': { + assertOnlyOptions(options, ['run-directory']); + const summary = await loadExecutionComparisonSummary(required(options, 'run-directory')); + process.stdout.write(formatExecutionComparisonSummary(summary, process.cwd())); + return; + } default: throw new Error( - 'Usage: execution-comparison-operator [options]', + 'Usage: execution-comparison-operator [options]', ); } } diff --git a/src/dev/execution-comparison/__tests__/compare-execution-prompt.test.ts b/src/dev/execution-comparison/__tests__/compare-execution-prompt.test.ts index 92fb97a56..d98607609 100644 --- a/src/dev/execution-comparison/__tests__/compare-execution-prompt.test.ts +++ b/src/dev/execution-comparison/__tests__/compare-execution-prompt.test.ts @@ -59,4 +59,13 @@ describe('/compare-execution operator prompt', () => { expect(prompt).toContain('Do not score'); expect(prompt).toContain('diagnostic-only'); }); + + it('finishes with the deterministic retained-attempt summary only', async () => { + const prompt = await readFile(promptPath, 'utf8'); + + expect(prompt).toContain('execution-comparison-operator.ts summary'); + expect(prompt).toContain("Return that command's stdout verbatim as the final response"); + expect(prompt).toContain('invalidity reason when applicable'); + expect(prompt).toContain('Do not prepend or append commentary'); + }); }); diff --git a/src/dev/execution-comparison/__tests__/execution-summary.test.ts b/src/dev/execution-comparison/__tests__/execution-summary.test.ts new file mode 100644 index 000000000..b7091d02f --- /dev/null +++ b/src/dev/execution-comparison/__tests__/execution-summary.test.ts @@ -0,0 +1,207 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, relative } from 'node:path'; + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { runExecutionComparisonOperatorCli } from '../../execution-comparison-operator.js'; +import type { ExecutionAttempt } from '../artifact-contract.js'; +import { formatExecutionComparisonSummary, loadExecutionComparisonSummary } from '../execution-summary.js'; + +const roots: string[] = []; + +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(roots.splice(0).map(async (root) => await rm(root, { recursive: true, force: true }))); +}); + +describe('execution comparison final summary', () => { + it('loads immutable attempts and explains invalidity in a concise final summary', async () => { + const runDirectory = await createRun([ + attempt({ + attemptId: 'brunch-attempt-1', + lane: 'brunch', + startedAt: '2026-07-23T10:00:00.000Z', + validity: { status: 'invalid', reasons: ['The run was landed after promotion preparation.'] }, + terminal: { + outcome: 'invalid', + reason: 'landed out of bounds', + productStatus: 'promotion_prepared', + }, + commands: [command('test', 'passed'), command('build', 'failed')], + }), + attempt({ + attemptId: 'claude-code-attempt-1', + lane: 'claude_code', + startedAt: '2026-07-23T11:00:00.000Z', + }), + ]); + + const summary = await loadExecutionComparisonSummary(runDirectory); + const rendered = formatExecutionComparisonSummary(summary, join(runDirectory, '..')); + + expect(rendered).toContain('Execution comparison complete'); + expect(rendered).toContain('Case: minimal-petri-net-editor-v1'); + expect(rendered).toContain( + 'Brunch\n Status: invalid\n Outcome: invalid\n Terminal: promotion_prepared', + ); + expect(rendered).toContain('Checks: test passed, build failed, browser not run'); + expect(rendered).toContain('Reason: The run was landed after promotion preparation.'); + expect(rendered).toContain( + 'Claude Code\n Status: valid\n Outcome: success\n Terminal: promotion_prepared', + ); + expect(rendered).not.toContain('cleanup clean'); + expect(rendered).toContain('Cleanup: done'); + expect(rendered).toContain(`- Report: ${join(runDirectory, 'report.md')}`); + expect(rendered).toContain(join(runDirectory, 'attempt-records/brunch-attempt-1/attempt.json')); + expect(rendered).toContain( + `- Brunch oracle: ${join(runDirectory, 'lanes/brunch/attempt-staging/browser/report.json')}`, + ); + expect(rendered).toContain( + `- Claude Code oracle: ${join(runDirectory, 'lanes/claude_code/attempt-staging/browser/report.json')}`, + ); + }); + + it('preserves a repository-relative oracle path already rooted inside the run', async () => { + const runDirectory = await createRun([]); + const baseDirectory = join(runDirectory, '..'); + const oraclePath = join(runDirectory, 'lanes/claude_code/attempt-staging/browser/report.json'); + await writeAttempt( + runDirectory, + attempt({ + attemptId: 'claude-code-attempt-3', + browser: { + status: 'passed', + reportPath: relative(baseDirectory, oraclePath), + }, + }), + ); + + const summary = await loadExecutionComparisonSummary(runDirectory); + const rendered = formatExecutionComparisonSummary(summary, baseDirectory); + + expect(rendered).toContain(`- Claude Code oracle: ${oraclePath}`); + }); + + it('distinguishes a valid failed attempt from a successful attempt', async () => { + const runDirectory = await createRun([ + attempt({ + attemptId: 'claude-code-failure-1', + terminal: { + outcome: 'failure', + reason: 'executor stopped before delivery', + productStatus: 'not_assessable', + }, + }), + ]); + + const summary = await loadExecutionComparisonSummary(runDirectory); + const rendered = formatExecutionComparisonSummary(summary, join(runDirectory, '..')); + + expect(rendered).toContain('Status: valid\n Outcome: failure\n Terminal: not_assessable'); + expect(rendered).not.toContain('Outcome: success'); + }); + + it('prints the deterministic summary through the operator command', async () => { + const runDirectory = await createRun([attempt({ attemptId: 'claude-code-attempt-2' })]); + const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + + await runExecutionComparisonOperatorCli(['summary', '--run-directory', runDirectory]); + + expect(stdout).toHaveBeenCalledOnce(); + expect(stdout.mock.calls[0]?.[0]).toContain('Execution comparison complete'); + expect(stdout.mock.calls[0]?.[0]).toContain('Claude Code\n Status: valid\n Outcome: success'); + }); + + it('refuses to summarize before the final report exists', async () => { + const runDirectory = await mkdtemp(join(tmpdir(), 'brunch-execution-summary-')); + roots.push(runDirectory); + + await expect(loadExecutionComparisonSummary(runDirectory)).rejects.toThrow( + 'execution comparison report is missing', + ); + }); +}); + +async function createRun(attempts: readonly ExecutionAttempt[]): Promise { + const runDirectory = await mkdtemp(join(tmpdir(), 'brunch-execution-summary-')); + roots.push(runDirectory); + await writeFile(join(runDirectory, 'report.md'), '# Report\n'); + for (const selected of attempts) { + await writeAttempt(runDirectory, selected); + } + return runDirectory; +} + +async function writeAttempt(runDirectory: string, selected: ExecutionAttempt): Promise { + const attemptDirectory = join(runDirectory, 'attempt-records', selected.attemptId); + await mkdir(attemptDirectory, { recursive: true }); + await writeFile(join(attemptDirectory, 'attempt.json'), `${JSON.stringify(selected)}\n`); +} + +function attempt(overrides: Partial): ExecutionAttempt { + return { + schemaVersion: 1, + attemptId: 'claude-code-attempt-1', + caseId: 'minimal-petri-net-editor-v1', + lane: 'claude_code', + publicPacketSha256: `sha256:${'a'.repeat(64)}`, + oraclePackSha256: `sha256:${'b'.repeat(64)}`, + startedAt: '2026-07-23T12:00:00.000Z', + endedAt: '2026-07-23T12:10:00.000Z', + budget: { + elapsedMinutes: 90, + mechanicalInterventions: 2, + substantiveHumanInterventions: 0, + }, + versions: { + product: '@hashintel/brunch@1.0.0-alpha.11', + provider: 'anthropic', + model: 'claude-opus-4-8', + harness: 'execution-comparison/v1', + actorRecipe: 'claude-code/v1', + node: '24.18.0', + npm: '11.6.2', + os: 'darwin', + architecture: 'arm64', + }, + repository: { + baseSha: '1'.repeat(40), + reviewSha: '2'.repeat(40), + finalGitRange: `${'1'.repeat(40)}..${'2'.repeat(40)}`, + }, + terminal: { + outcome: 'success', + reason: 'promotion prepared', + productStatus: 'promotion_prepared', + }, + validity: { status: 'valid', reasons: [] }, + commands: [command('test', 'passed'), command('build', 'passed')], + browser: { status: 'not_run', reportPath: 'browser/report.json' }, + interventions: [], + commonMetrics: { + elapsedMs: 600_000, + inputTokens: 'not_assessable', + outputTokens: 'not_assessable', + costUsd: 'not_assessable', + permissionPrompts: 'not_assessable', + }, + evidence: { + finalTreePath: 'repository/final-tree.txt', + finalDiffPath: 'repository/final.diff', + visibleProcessPath: 'process/visible.jsonl', + }, + cleanup: { status: 'clean', liveProcesses: 0, liveSessions: 0 }, + ...overrides, + }; +} + +function command(id: string, status: 'passed' | 'failed' | 'not_run'): ExecutionAttempt['commands'][number] { + return { + id, + status, + exitCode: status === 'passed' ? 0 : status === 'failed' ? 1 : 'not_assessable', + stdoutPath: `commands/${id}.stdout.txt`, + stderrPath: `commands/${id}.stderr.txt`, + }; +} diff --git a/src/dev/execution-comparison/execution-summary.ts b/src/dev/execution-comparison/execution-summary.ts new file mode 100644 index 000000000..3b39f8825 --- /dev/null +++ b/src/dev/execution-comparison/execution-summary.ts @@ -0,0 +1,156 @@ +import { lstat, readFile, readdir } from 'node:fs/promises'; +import { basename, isAbsolute, join, normalize, relative, resolve, sep } from 'node:path'; + +import { parseExecutionAttempt, type ExecutionAttempt } from './artifact-contract.js'; + +export interface ExecutionComparisonAttemptSummary { + readonly attempt: ExecutionAttempt; + readonly recordPath: string; +} + +export interface ExecutionComparisonSummary { + readonly runDirectory: string; + readonly runId: string; + readonly caseId: string; + readonly reportPath: string; + readonly attempts: readonly ExecutionComparisonAttemptSummary[]; +} + +export async function loadExecutionComparisonSummary( + runDirectoryInput: string, +): Promise { + const runDirectory = resolve(runDirectoryInput); + const reportPath = join(runDirectory, 'report.md'); + if (!(await isRegularFile(reportPath))) { + throw new Error(`execution comparison report is missing: ${reportPath}`); + } + + const attemptsRoot = join(runDirectory, 'attempt-records'); + const entries = await readdir(attemptsRoot, { withFileTypes: true }); + const attempts: ExecutionComparisonAttemptSummary[] = []; + for (const entry of entries.sort((left, right) => codePointCompare(left.name, right.name))) { + if (!entry.isDirectory() || entry.isSymbolicLink()) continue; + const recordPath = join(attemptsRoot, entry.name, 'attempt.json'); + const value = JSON.parse(await readFile(recordPath, 'utf8')) as unknown; + const attempt = parseExecutionAttempt(value); + if (attempt.attemptId !== entry.name) { + throw new Error(`execution attempt directory does not match attempt id: ${entry.name}`); + } + attempts.push({ attempt, recordPath }); + } + if (attempts.length === 0) { + throw new Error(`execution comparison has no retained attempts: ${attemptsRoot}`); + } + + attempts.sort((left, right) => { + const byStart = codePointCompare(left.attempt.startedAt, right.attempt.startedAt); + return byStart === 0 ? codePointCompare(left.attempt.attemptId, right.attempt.attemptId) : byStart; + }); + const caseId = attempts[0]!.attempt.caseId; + const publicPacketSha256 = attempts[0]!.attempt.publicPacketSha256; + const oraclePackSha256 = attempts[0]!.attempt.oraclePackSha256; + if ( + attempts.some( + ({ attempt }) => + attempt.caseId !== caseId || + attempt.publicPacketSha256 !== publicPacketSha256 || + attempt.oraclePackSha256 !== oraclePackSha256, + ) + ) { + throw new Error('execution comparison attempts do not share one case and frozen evidence set'); + } + + return { + runDirectory, + runId: basename(runDirectory), + caseId, + reportPath, + attempts, + }; +} + +export function formatExecutionComparisonSummary( + summary: ExecutionComparisonSummary, + baseDirectory: string, +): string { + const lines = [ + 'Execution comparison complete', + '=============================', + '', + `Case: ${summary.caseId}`, + `Run: ${summary.runId}`, + '', + 'Results', + '-------', + ]; + + for (const { attempt } of summary.attempts) { + const commands = attempt.commands + .map((command) => `${command.id} ${displayStatus(command.status)}`) + .join(', '); + lines.push('', laneLabel(attempt.lane)); + lines.push(` Status: ${attempt.validity.status}`); + lines.push(` Outcome: ${attempt.terminal.outcome}`); + lines.push(` Terminal: ${attempt.terminal.productStatus}`); + lines.push(` Checks: ${commands}, browser ${displayStatus(attempt.browser.status)}`); + if (attempt.validity.status === 'invalid') { + lines.push(` Reason: ${attempt.validity.reasons[0]}`); + } + } + + const cleanup = summary.attempts.every(({ attempt }) => attempt.cleanup.status === 'clean') + ? 'done' + : 'residue'; + lines.push('', `Cleanup: ${cleanup}`, '', 'Artifacts', '---------'); + lines.push(`- Report: ${absolutePath(summary.reportPath, baseDirectory)}`); + for (const { attempt, recordPath } of summary.attempts) { + lines.push(`- ${laneLabel(attempt.lane)} attempt: ${absolutePath(recordPath, baseDirectory)}`); + lines.push(`- ${laneLabel(attempt.lane)} oracle: ${absoluteOraclePath(summary, attempt, baseDirectory)}`); + } + return `${lines.join('\n')}\n`; +} + +function displayStatus(status: 'passed' | 'failed' | 'not_run'): string { + return status === 'not_run' ? 'not run' : status; +} + +function laneLabel(lane: ExecutionAttempt['lane']): string { + return lane === 'brunch' ? 'Brunch' : 'Claude Code'; +} + +function absolutePath(path: string, baseDirectory: string): string { + return resolve(baseDirectory, path); +} + +function absoluteOraclePath( + summary: ExecutionComparisonSummary, + attempt: ExecutionAttempt, + baseDirectory: string, +): string { + const reportPath = normalize(attempt.browser.reportPath); + if (reportPath.startsWith(`browser${sep}`)) { + return resolve(summary.runDirectory, 'lanes', attempt.lane, 'attempt-staging', reportPath); + } + + const repositoryRelative = absolutePath(reportPath, baseDirectory); + return inside(summary.runDirectory, repositoryRelative) + ? repositoryRelative + : resolve(summary.runDirectory, reportPath); +} + +function inside(parent: string, child: string): boolean { + const selected = relative(parent, child); + return selected === '' || (!isAbsolute(selected) && selected !== '..' && !selected.startsWith(`..${sep}`)); +} + +async function isRegularFile(path: string): Promise { + try { + return (await lstat(path)).isFile(); + } catch { + return false; + } +} + +function codePointCompare(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +}