From 361f3edbd5519576a3b182c96f3cded847bf417f Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 24 Jul 2026 17:13:30 +0200 Subject: [PATCH 1/2] FE-1267: Record comparison controller takeovers --- .changeset/eleven-words-feel.md | 2 + .pi/prompts/compare-execution.md | 11 ++++- src/dev/execution-comparison-operator.ts | 9 +++- .../__tests__/artifact-contract.test.ts | 26 +++++++++++ .../__tests__/operator-cli.test.ts | 14 ++++++ .../execution-comparison/artifact-contract.ts | 46 +++++++++++++++++++ 6 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 .changeset/eleven-words-feel.md diff --git a/.changeset/eleven-words-feel.md b/.changeset/eleven-words-feel.md new file mode 100644 index 000000000..a845151cc --- /dev/null +++ b/.changeset/eleven-words-feel.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.pi/prompts/compare-execution.md b/.pi/prompts/compare-execution.md index d09f145ef..ed944f855 100644 --- a/.pi/prompts/compare-execution.md +++ b/.pi/prompts/compare-execution.md @@ -133,6 +133,12 @@ Send only the approved shared framing. Do not substitute a raw binary, another m After every lane terminates, first capture its final process status and all target-visible interaction evidence. Kill any still-running executor process, query the shell to a final status, dismiss the completed shell record, and verify that no executor process or interactive session remains. +Record every controller takeover immediately in `/intervention-ledger.json`, including a +mechanical selection of an already-proposed action. The ledger has schema `{ "schemaVersion": 1, +"interventions": [...] }`; each intervention uses the `ExecutionAttempt` index, kind, description, and +timestamp fields. The visible-interaction summary must be derived from this ledger rather than independently +authored. No unledgered controller takeover is permitted. + Then run the unchanged controller-owned oracle identity returned by `inspect` against the retained output: ```sh @@ -163,10 +169,13 @@ Stage evidence under the fresh attempt path, construct an `ExecutionAttempt` JSO ```sh npx tsx src/dev/execution-comparison-operator.ts retain-attempt \ --attempt-file /attempt.json \ + --intervention-ledger /intervention-ledger.json \ --attempts-root .fixtures/scratch/execution-comparisons//attempt-records ``` -The immutable writer rejects an existing attempt id. Never overwrite, delete, replace, or silently retry a poor-output attempt. A replacement is permitted only for provider, adapter, or mechanical invalidity under the unchanged packet, and both attempts remain retained. +The immutable writer rejects an existing attempt id and rejects a ledger that does not exactly match the attempt's +intervention list. Never overwrite, delete, replace, or silently retry a poor-output attempt. A replacement is +permitted only for provider, adapter, or mechanical invalidity under the unchanged packet, and both attempts remain retained. ## Report diff --git a/src/dev/execution-comparison-operator.ts b/src/dev/execution-comparison-operator.ts index 304297f5f..81810166e 100644 --- a/src/dev/execution-comparison-operator.ts +++ b/src/dev/execution-comparison-operator.ts @@ -4,6 +4,7 @@ import process from 'node:process'; import { fileURLToPath } from 'node:url'; import { + assertExecutionAttemptInterventionLedger, parseExecutionAttempt, writeExecutionAttemptImmutable, } from './execution-comparison/artifact-contract.js'; @@ -275,9 +276,13 @@ export async function runExecutionComparisonOperatorCli(args: readonly string[]) return; } case 'retain-attempt': { - assertOnlyOptions(options, ['attempt-file', 'attempts-root']); - const value = JSON.parse(await readFile(resolve(required(options, 'attempt-file')), 'utf8')) as unknown; + assertOnlyOptions(options, ['attempt-file', 'attempts-root', 'intervention-ledger']); + const attemptPath = resolve(required(options, 'attempt-file')); + const interventionLedgerPath = resolve(required(options, 'intervention-ledger')); + const value = JSON.parse(await readFile(attemptPath, 'utf8')) as unknown; const attempt = parseExecutionAttempt(value); + const interventionLedger = JSON.parse(await readFile(interventionLedgerPath, 'utf8')) as unknown; + assertExecutionAttemptInterventionLedger(attempt, interventionLedger); const attemptsRoot = resolve(required(options, 'attempts-root')); await mkdir(attemptsRoot, { recursive: true }); const stored = await writeExecutionAttemptImmutable(attemptsRoot, attempt); diff --git a/src/dev/execution-comparison/__tests__/artifact-contract.test.ts b/src/dev/execution-comparison/__tests__/artifact-contract.test.ts index f58243699..124b86e74 100644 --- a/src/dev/execution-comparison/__tests__/artifact-contract.test.ts +++ b/src/dev/execution-comparison/__tests__/artifact-contract.test.ts @@ -5,6 +5,7 @@ import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; import { + assertExecutionAttemptInterventionLedger, parseExecutionAttempt, writeExecutionAttemptImmutable, type ExecutionAttempt, @@ -119,4 +120,29 @@ describe('execution comparison attempt artifact contract', () => { }), ).toThrow('invalid execution attempt'); }); + + it('rejects an intervention ledger that does not exactly match the immutable attempt', () => { + const selected = attempt('success'); + const ledger = { + schemaVersion: 1, + interventions: [ + { + index: 1, + kind: 'mechanical', + description: 'controller selected the confirmed action', + at: '2026-07-20T12:05:00.000Z', + }, + ], + } as const; + + expect(() => assertExecutionAttemptInterventionLedger(selected, ledger)).toThrow( + 'execution intervention ledger does not match attempt', + ); + expect( + assertExecutionAttemptInterventionLedger(selected, { schemaVersion: 1, interventions: [] }), + ).toEqual({ + schemaVersion: 1, + interventions: [], + }); + }); }); diff --git a/src/dev/execution-comparison/__tests__/operator-cli.test.ts b/src/dev/execution-comparison/__tests__/operator-cli.test.ts index af1e398cd..521dd23aa 100644 --- a/src/dev/execution-comparison/__tests__/operator-cli.test.ts +++ b/src/dev/execution-comparison/__tests__/operator-cli.test.ts @@ -250,3 +250,17 @@ describe('execution comparison oracle CLI', () => { } }); }); + +describe('execution comparison attempt retention CLI', () => { + it('requires an intervention ledger before reading or retaining an attempt', async () => { + await expect( + runExecutionComparisonOperatorCli([ + 'retain-attempt', + '--attempt-file', + 'missing-attempt.json', + '--attempts-root', + 'attempt-records', + ]), + ).rejects.toThrow('missing required option --intervention-ledger'); + }); +}); diff --git a/src/dev/execution-comparison/artifact-contract.ts b/src/dev/execution-comparison/artifact-contract.ts index 08434713f..0465efac0 100644 --- a/src/dev/execution-comparison/artifact-contract.ts +++ b/src/dev/execution-comparison/artifact-contract.ts @@ -80,6 +80,16 @@ export interface ExecutionAttempt { }; } +export interface ExecutionInterventionLedger { + readonly schemaVersion: 1; + readonly interventions: readonly { + readonly index: number; + readonly kind: 'mechanical'; + readonly description: string; + readonly at: string; + }[]; +} + export function parseExecutionAttempt(value: unknown): ExecutionAttempt { if (!record(value)) invalid(); const budget = child(value, 'budget'); @@ -144,6 +154,29 @@ export function parseExecutionAttempt(value: unknown): ExecutionAttempt { return value as unknown as ExecutionAttempt; } +export function assertExecutionAttemptInterventionLedger( + attempt: ExecutionAttempt, + value: unknown, +): ExecutionInterventionLedger { + if ( + !record(value) || + value['schemaVersion'] !== 1 || + !interventionRecords(value['interventions'], Infinity) + ) { + throw new Error('invalid execution intervention ledger'); + } + const ledger = value as unknown as ExecutionInterventionLedger; + if ( + ledger.interventions.length !== attempt.interventions.length || + !ledger.interventions.every((intervention, index) => + sameIntervention(intervention, attempt.interventions[index]), + ) + ) { + throw new Error('execution intervention ledger does not match attempt'); + } + return ledger; +} + export async function writeExecutionAttemptImmutable( attemptsRoot: string, value: ExecutionAttempt, @@ -211,6 +244,19 @@ function interventionRecords(value: unknown, budget: number): boolean { ); } +function sameIntervention( + left: ExecutionInterventionLedger['interventions'][number], + right: ExecutionAttempt['interventions'][number] | undefined, +): boolean { + return ( + right !== undefined && + left.index === right.index && + left.kind === right.kind && + left.description === right.description && + left.at === right.at + ); +} + function gitRange(range: unknown, baseSha: unknown, reviewSha: unknown): boolean { if (range === 'not_assessable') return reviewSha === 'not_assessable'; return ( From c55f2866f38dccc1034728b10f21acda811c4c73 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 24 Jul 2026 17:15:39 +0200 Subject: [PATCH 2/2] FE-1267: Release comparison evidence guard --- .changeset/eleven-words-feel.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.changeset/eleven-words-feel.md b/.changeset/eleven-words-feel.md index a845151cc..914fc53ee 100644 --- a/.changeset/eleven-words-feel.md +++ b/.changeset/eleven-words-feel.md @@ -1,2 +1,5 @@ --- +"@hashintel/brunch": patch --- + +Record controller takeovers before retaining immutable comparison evidence.