diff --git a/.changeset/recover-prepared-execution-journals.md b/.changeset/recover-prepared-execution-journals.md new file mode 100644 index 000000000..860c70fd8 --- /dev/null +++ b/.changeset/recover-prepared-execution-journals.md @@ -0,0 +1,5 @@ +--- +"@hashintel/brunch": patch +--- + +Recover prepared execution journals across direct lifecycle steps and superseding runs. diff --git a/src/.pi/extensions/__tests__/registry.test.ts b/src/.pi/extensions/__tests__/registry.test.ts index db3289664..e11d52b24 100644 --- a/src/.pi/extensions/__tests__/registry.test.ts +++ b/src/.pi/extensions/__tests__/registry.test.ts @@ -1371,6 +1371,11 @@ describe('Brunch explicit Pi extension registry', () => { sideEffects: [ { kind: 'mkdir', path: dirname(newMetadataPath) }, { kind: 'write_file', path: newMetadataPath, ifExists: 'overwrite' }, + { kind: 'mkdir', path: join(dirname(newMetadataPath), 'petrinaut') }, + { kind: 'write_file', path: join(dirname(newMetadataPath), 'petrinaut', 'plan.json') }, + { kind: 'write_file', path: join(dirname(newMetadataPath), 'petrinaut', 'net.json') }, + { kind: 'write_file', path: join(dirname(newMetadataPath), 'petrinaut', 'net.sdcpn.json') }, + { kind: 'write_file', path: join(dirname(newMetadataPath), 'petrinaut', 'events.jsonl') }, ], }); await expect(readFile(oldMetadataPath, 'utf8')).resolves.not.toContain('supersedesRunId'); diff --git a/src/.pi/extensions/executor/execute-report-init/index.ts b/src/.pi/extensions/executor/execute-report-init/index.ts index f7cb51b41..20b6822c5 100644 --- a/src/.pi/extensions/executor/execute-report-init/index.ts +++ b/src/.pi/extensions/executor/execute-report-init/index.ts @@ -24,7 +24,7 @@ export function createExecuteReportInitTool() { name: BRUNCH_EXECUTE_REPORT_INIT_TOOL, label: 'execute_report_init', description: - 'Initialize reports.jsonl for a source-copied cook run. Does not execute slices or create Petri artifacts.', + 'Initialize reports.jsonl for a source-copied cook run and reconcile an existing Petri observation journal. Does not execute slices or create observer definitions.', parameters: toolParameters(ExecuteReportInitParams), async execute(_toolCallId, params, _signal, _onUpdate, ctx) { const cwd = ctx?.cwd; diff --git a/src/.pi/extensions/executor/execute-slice-start/index.ts b/src/.pi/extensions/executor/execute-slice-start/index.ts index 6ac3c92ce..8fe23fd79 100644 --- a/src/.pi/extensions/executor/execute-slice-start/index.ts +++ b/src/.pi/extensions/executor/execute-slice-start/index.ts @@ -25,7 +25,7 @@ export function createExecuteSliceStartTool() { name: BRUNCH_EXECUTE_SLICE_START_TOOL, label: 'execute_slice_start', description: - 'Append a slice-start marker for a ready cook run. Does not execute agents, tests, or Petri transitions.', + 'Append a slice-start marker and reconcile its prepared Petri observation. Does not execute agents or tests.', parameters: toolParameters(ExecuteSliceStartParams), async execute(_toolCallId, params, _signal, _onUpdate, ctx) { const cwd = ctx?.cwd; diff --git a/src/executor/TOPOLOGY.md b/src/executor/TOPOLOGY.md index 63c9cd699..b8159edf3 100644 --- a/src/executor/TOPOLOGY.md +++ b/src/executor/TOPOLOGY.md @@ -22,6 +22,7 @@ executor/ │ └── types.ts private batch/effect contracts ├── petri-events.ts crash-durable `petrinaut/events.jsonl` authority + parse helpers and process-local observer wake-ups ├── petri-journal-authority.ts complete journal + lifecycle history -> topology-validated multiset residual and slice claims +├── petri-lifecycle-reconciliation.ts prepared serial lifecycle prefix -> explicit synchronized/not-prepared/blocked result + repaired journal/marking ├── petri-plan-snapshot.ts immutable run-local plan snapshot path + atomic first-write helper ├── petri-projection.ts shared Petri projection contract + parser/normalizer for snapshot/read/live-update boundaries ├── petri-runtime-plan.ts shared populated-plan fallback resolution for driver and observer Petri materialization @@ -138,7 +139,7 @@ Planner rounds publish lifecycle-only progress, stop after a fixed per-round dea `run-auto-replan-policy.ts` is the conservative automation policy over a precomputed recommendation. It auto-delegates only fresh-run `retry_current_step` when retry budget remains and stale-early `regenerate_plan`; stale started/missing runs require human start-new-run, and terminal/blocked runs stay inspect-only. It owns no filesystem, driver, tool, or graph effects: retry/regenerate are injected delegates, and the policy never auto-supersedes. -`run-supersession.ts` is the first bounded mutation helper in the HITL replanning family. Given an existing run and a fresh launch-ready plan, it creates a new `created` run with `supersedesRunId` pointing to the prior run. It refuses missing prior runs, stale/non-ready plans, and target run id collisions, and it never mutates the prior run. +`run-supersession.ts` is the first bounded mutation helper in the HITL replanning family. Given an existing run and a fresh launch-ready plan, it creates a new `created` run with `supersedesRunId` pointing to the prior run. Creation, Petrinaut observation preparation, and failure rollback remain under one target-run authority lease, so no other run mutation can observe or act on a half-prepared replacement. It refuses missing prior runs, stale/non-ready plans, and target run id collisions, and it never mutates the prior run. `worktree.ts` creates the run workspace from the metadata substrate. The default `git_worktree` substrate creates a real git worktree through the injected `GitWorktreePort` (app-layer `git worktree add --detach HEAD`) and updates `run.json` to `status:"worktree_created"`. The `empty_dir` substrate creates an isolated run directory, initializes a run-local git repository with an empty base commit, and never invokes the host git-worktree port; exact-root checks prevent host-linked `.git` markers from being accepted. Both substrates record the durable `runBaseSha` (the commit the worktree was created at, or the empty base commit) in the same `worktree_created` metadata advance; every later promotion/landing range is `runBaseSha..tip`. If the selected port operation, base resolution, or run-local initialization fails, run metadata is not advanced (`status:"worktree_create_failed"`). Source population, scaffold templating, agent execution, Petri artifacts, report logs, promotion refs, and land branches remain deferred. @@ -148,9 +149,9 @@ Planner rounds publish lifecycle-only progress, stop after a fixed per-round dea `source-copy.ts` performs bounded host source copying for `host_source_deferred`: it copies top-level source entries into the worktree while excluding `.brunch`, `.git`, `node_modules`, `dist`, and `build`, then records `status:"source_copied"`. Slice execution, Petri artifacts, report logs, promotion refs, and land branches remain deferred. -`report.ts` initializes `reports.jsonl` for a source-copied run with a single `run_ready` event and records `status:"reports_initialized"`. It creates the report log carrier but still does not execute slices or produce Petri artifacts. +`report.ts` initializes `reports.jsonl` for a source-copied run with a single `run_ready` event and records `status:"reports_initialized"`. For an observation-prepared run, it first reconciles the source-copied prefix, then reconciles the `report_init` transition after the report/metadata writes. A durable Petri terminal blocks before those writes; if post-transition reconciliation blocks, the result exposes the Petri reason, advanced run status, and completed writes instead of reporting success. It does not execute slices or create observer definitions. -`slice-start.ts` appends a `slice_started` marker for one plan slice and records the active slice/epic in `run.json`. It is not agent execution: no tools/tests run and no Petri transitions or promotion artifacts are created. +`slice-start.ts` appends a `slice_started` marker for one plan slice and records the active slice/epic in `run.json`. Prepared standalone starts use the same serial lifecycle reconciler before and after the metadata transition, so they neither diverge from `drive()` nor invent parallel authority: an unrecoverable lifecycle-ahead journal returns `petri_journal_gap`, unreadable or non-slice journal authority returns `petri_input_unreadable`, a durable terminal returns `petri_terminal_recorded`, and only persisted or journal-ahead slice claims return `parallel_batch_active`. It is not agent execution: no tools/tests run and no promotion artifacts are created. `slice-execute.ts` asks the injected `GitSliceIntegrationPort` to create one stable workspace at `slice-workspaces//worktree`, rooted at the run workspace's current HEAD, then creates the execution request under `agent-output//request.json` and records the workspace/base identity. It requires the active slice to be readable from the populated plan before advancing, then carries definition, verification criteria, and scope/design/verification context; missing or malformed plan data returns `plan_slice_invalid` with no side effects. A pre-existing path is reusable only when git reports the same common repository, the path is registered as that repository's linked worktree, and its HEAD still equals the current run base; integration rechecks repository identity and recorded-base ancestry. A foreign repository at the predictable path fails before agent dispatch and is never mutated. Retries reuse the already accepted workspace; there are no per-attempt worktrees. @@ -168,6 +169,8 @@ The standalone `execute_slice_complete` tool composes those same two explicit he With `petriScheduler` + `frontierFiringPolicy`, a co-firable slice-start frontier delegates to `parallel-slice-batch.ts` instead of racing active-slice lifecycle helpers. Every selected `slice_start` is appended to the journal and reflected in an atomic marking snapshot before any workspace/agent effect begins. Slice workspace preparation, agent attempts, and verification may then overlap; each effect persists an ordered per-slice success or failure settlement (including failure step/reason) as soon as it settles. Thrown workspace preparation, request-artifact, agent, verifier, and integration effects become typed failed settlements; authority-carrier failures still reject fail-closed. Serial bound-effect throws use the same reason formatter and become journaled `net_halted` outcomes with terminal marking; Petri journal/marking carrier failures retain their dedicated fail-closed behavior. Fan-in consumes the concurrently started effect promises in claimed order, so a clean sibling can remain durable/integrable while another fails or blocks. `run.json` stays unchanged during effects and advances only as a serial summary after successful integration; the still-present batch remains observer authority during the summary-write-before-marking-refresh window. Before every subsequent lifecycle or epic dispatch, `drive()` requires a readable journal independently of `net.json`, then replays both durable and lifecycle transition sequences through the compiled executor topology. Equal/ahead/behind histories must also have matching transition multisets or subset authority; this admits only causally valid independent interleavings, not per-lane reorderings that violate slice dependencies or epic membership. Missing, unavailable, unreadable, unreplayable, or mismatched journals halt before dispatch as `petri_input_unreadable`; lifecycle-ahead readable journals halt as `petri_journal_gap`. Only active parallel batch/journal claims and transitioned epic claims with matching journal authority may be journal-ahead. Persisted parallel authority still returns `parallel_slice_replan_required`, including the final-summary-before-clear crash window. +`petri-lifecycle-reconciliation.ts` repairs only a causally valid journal history whose lifecycle-only residual is a trailing serial suffix through `report_init` or one standalone `slice_start`; causally valid independent interleavings that already equal lifecycle authority remain synchronized. It derives those already-completed firings from D112-L `run.json` authority, appends canonical topology events, and refreshes marking provenance before dispatch. Equal histories with missing or provenance-stale marking replay and rewrite it, so a journal append that outlived a failed marking write repairs on retry; a matching claim-bearing marking remains authoritative. Journal-append and marking-persist failures return dedicated blocked reasons instead of escaping the driver or standalone tool. Its named result distinguishes synchronized authority, an unprepared observation, and blocked authority with a concrete reason; a reconciling handler never treats the blocked result as permission for a fallback append. Unreadable, terminal, reordered, parallel, epic, attempt, or later slice histories remain fail-closed. + Parallel admission persists a run summary with stale serial active slice/epic, workspace/base, request, and result fields removed before claims or effects; all settlement and fan-in summaries derive from that cleared state. The first claim then journals before persisting the authoritative marking and its matching cleared lifecycle provenance. All scheduler, step-halt, and parallel terminal paths share one complete-journal append-once authority. A journal-ahead terminal is reused with its original timestamp and catches up a lagging marking snapshot; multiple terminals fail closed. Durable terminal kind/status/reason/failed-slice evidence remains final across restart and later metadata abandonment. `epic-lifecycle.ts` is the one serial epic lane used after either serial slice execution or a parallel batch converges. `epic_integrate` becomes enabled only after every member emits its integration/completion token. Empty verification follows the explicit integrate -> complete path without invoking a runner. Non-empty verification first appends `epic_verification_claimed` and persists a marking claim before invoking the injected `TestRunnerPort` against the integrated run worktree with the run's configured `VerifyTarget`; plan criterion targets are report provenance only. Claimed and transitioned verification authority suppresses observer `epic_verify` readiness and appears as an explicit blocker. A claimed effect with no durable result halts/replans and never reruns. A durable passing report can resume without rerunning; `epic_verify` journal + transitioned marking persist before `verifiedEpicIds`, and a transitioned marking catches a lagging summary up only when shared journal inspection finds that exact transition. A transitioned marking without the durable transition interrupts without runner, verified summary, completion, or rerun. A thrown runner becomes a durable failed `epic_test_result` and terminal marking. Carrier append/snapshot failures invoke no runner and advance no summary. Epic transition history is reconciled in exact order from the durable journal; malformed or torn journal content halts with `petri_input_unreadable` instead of falling back to metadata. No epic gate is inferred from completed slices and parallel authority never auto-fires epic transitions. @@ -176,6 +179,8 @@ Parallel authority journals result ingestion followed by the chosen cycle-qualif `run-complete.ts` appends `run_completed` only after every plan slice and epic is completed, every slice has latest passing verification evidence, and every non-empty epic verification has latest passing `epic_test_result` evidence. Failed or missing verification leaves metadata unchanged. Promotion therefore follows verification of the same integrated run worktree. +`run-supersession.ts` prepares the same immutable Petrinaut observer snapshot as product `execute_run_create` before returning a replacement run, so supersession cannot publish structurally incomplete execution input. + `petri-plan-snapshot.ts` atomically freezes the source plan at `petrinaut/plan.json`. Product `execute_run_create` prepares that snapshot, `net.json`, `net.sdcpn.json`, and a create-only empty `events.jsonl` before returning, so clients can attach before orchestration. `drive()` verifies preparation before its first lifecycle effect and never recreates a missing journal after preparation or on a later dispatch gate, even when definition artifacts are missing too. A preparation failure caused by a missing or unavailable journal publishes only the process-local failure wake-up and returns fail-closed; it does not append a halt event through the broken carrier. Worktree population and Petri runtime resolution prefer the same frozen snapshot, preventing published topology from diverging from execution if the source plan changes. `petri.ts` atomically creates immutable definition files; later preparation/export accepts byte-identical content and rejects rewrites. `petri_export` records `status:"petri_exported"` without changing the published definition. `net.json` remains the complete raw executor topology. The SDCPN/view projection mechanically omits only places with no incident arc and no initial token; every connected ID and arc survives. It derives human-readable contextual labels and globally unique finite coordinates from subnet identity, using fixed execution-phase columns, locale-independent natural-ID ordering, deterministic sorted slice bands with compact collision offsets, a run lane, and an epic lane. The current identity projection keeps every per-slice attempt expanded; above roughly 12 slices, revisit standardized subnet grouping/folding. This is not color-fold parity. These are Brunch-owned observer projections, not runtime authority. Compatibility with Petrinaut's loader remains an external consumer oracle, now mirrored locally by the strict staging definition contract but still requiring the external consumer check. `petrinaut/replay-export.ts`, `petrinaut/stream-frames.ts`, and `petrinaut/sse.ts` validate and reduce `net.sdcpn.json` + `events.jsonl` into replay-equivalent ordered frames and SSE chunks only after the complete event history successfully replays through raw `net.json`; impossible causality or any fact after a terminal event yields no export or stream. Replay definitions preserve SDCPN names and coordinates; legacy coordinate fallback reserves every explicit coordinate first, then allocates globally unique deterministic finite positions, and synthetic run-status nodes occupy the run lane. Coordinates are required by the public RPC schema after projection. The SDCPN read boundary enforces positive version, non-empty projected labels/identities, finite present coordinates, valid arcs/scenarios, and accepts legitimate full-root type/equation/parameter/metric arrays that replay does not interpret. `petri-events.ts` stamps each event once with an ISO timestamp before durable append, rejects timestamp-less or invalidly timestamped journals, and publishes that same stamped object to same-process observers only after append. Petrinaut firings retain the origin/main `{transitionId,input,output,ts}` contract; synthetic `run:finish` reuses the terminal event timestamp. Terminal journal and marking evidence carry ordered failed-slice identity through `PetriProjection`, status/terminal frames, SSE, and both `execute.runs`/`execute.run`; run detail retains metadata-only abandonment failures, but a durable prior terminal remains authoritative. A failed append or dispatch-gate detection of broken journal authority wakes run-scoped journal-failure listeners; `drive()` halts before further lifecycle advancement, and active streams close so clients reconnect against whatever remained durable rather than waiting on a wake-up that cannot come. Normal completion closes only after its journal terminal, preserving final transition order. `run.ts` metadata listeners wake active streams for `abandoned`, the sole metadata-only terminal; its synthetic `run:finish` uses `abandonedAt` only when no durable terminal exists. These surfaces never own lifecycle truth, mutate `run.json`, or grant parallel side-effect authority. Process-local wake-ups are the declared first ceiling; split-process execution requires file watching or a durable broker. diff --git a/src/executor/__tests__/orchestrate.test.ts b/src/executor/__tests__/orchestrate.test.ts index 6ae11b84b..8b53cd527 100644 --- a/src/executor/__tests__/orchestrate.test.ts +++ b/src/executor/__tests__/orchestrate.test.ts @@ -1725,7 +1725,12 @@ describe('drive', () => { reason: 'petri_input_unreadable', }); } else { - await expect(driving).rejects.toThrow(); + await expect(driving).resolves.toEqual({ + status: 'halted', + step: 'epic_verify', + runStatus: 'slice_completed', + reason: 'petri_marking_persist_failed', + }); } expect(runnerCalls).toBe(0); expect((await readRunMetadata(runMetadataPath(cwd, 'run-1')))?.verifiedEpicIds).toBeUndefined(); @@ -3576,7 +3581,7 @@ describe('drive', () => { expect(snapshot?.terminalEventKind).toBeUndefined(); }); - it('does not repeat an effect when its metadata advanced before the transition append failed', async () => { + it('repairs a lifecycle journal gap without repeating the completed effect', async () => { const cwd = await mkdtemp(join(tmpdir(), 'brunch-drive-journal-gap-restart-')); await createRunAtCreated(cwd, ['task-1']); await preparePetriObservation({ cwd, runId: 'run-1' }); @@ -3609,14 +3614,129 @@ describe('drive', () => { await rename(preservedJournalPath, journalPath); await expect(drive({ cwd, runId: 'run-1', ports })).resolves.toEqual({ + status: 'completed', + runStatus: 'promotion_prepared', + }); + expect(worktreeCalls).toBe(1); + await expect(readRunMetadata(runMetadataPath(cwd, 'run-1'))).resolves.toMatchObject({ + status: 'promotion_prepared', + }); + }); + + it('halts and repairs when lifecycle catch-up cannot persist its marking', async () => { + const cwd = await mkdtemp(join(tmpdir(), 'brunch-drive-reconcile-marking-failure-')); + await createRunAtCreated(cwd, ['task-1']); + await preparePetriObservation({ cwd, runId: 'run-1' }); + await createWorktree({ cwd, runId: 'run-1', gitWorktree: createFakeGitWorktreePort() }); + const markingPath = petriMarkingPath(cwd, 'run-1'); + await mkdir(markingPath); + + await expect(drive({ cwd, runId: 'run-1', ports: fakePorts() })).resolves.toEqual({ status: 'halted', step: 'populate', runStatus: 'worktree_created', - reason: 'petri_journal_gap', + reason: 'petri_marking_persist_failed', + }); + expect( + (await readPetriEvents(cwd)).flatMap((event) => + event.kind === 'transition_fired' ? [event.transitionId] : [], + ), + ).toEqual(['worktree_create']); + + await rm(markingPath, { recursive: true }); + await expect(drive({ cwd, runId: 'run-1', ports: fakePorts() })).resolves.toEqual({ + status: 'completed', + runStatus: 'promotion_prepared', + }); + await expect(readPetriMarkingSnapshot({ cwd, runId: 'run-1' })).resolves.toMatchObject({ + lifecycleProvenance: { runStatus: 'promotion_prepared' }, + }); + }); + + it('does not append when Petri authority becomes unreadable during a reconciling step', async () => { + const cwd = await mkdtemp(join(tmpdir(), 'brunch-drive-reconcile-fail-closed-')); + await createRunAtCreated(cwd, ['task-1']); + const journalPath = petriEventsPath(cwd, 'run-1'); + const corruptJournal = '{not-json\n'; + let failureWakeUps = 0; + const unsubscribe = subscribePetriJournalFailures({ + cwd, + runId: 'run-1', + listener: () => { + failureWakeUps += 1; + }, + }); + + let outcome: Awaited>; + try { + outcome = await drive({ + cwd, + runId: 'run-1', + ports: fakePorts(), + onStepStart(step) { + if (step !== 'report_init') return; + writeFileSync(journalPath, corruptJournal, 'utf8'); + }, + }); + } finally { + unsubscribe(); + } + + expect(await readFile(journalPath, 'utf8')).toBe(corruptJournal); + expect(failureWakeUps).toBe(1); + expect(outcome).toEqual({ + status: 'halted', + step: 'report_init', + runStatus: 'source_copied', + reason: 'petri_input_unreadable', }); - expect(worktreeCalls).toBe(1); await expect(readRunMetadata(runMetadataPath(cwd, 'run-1'))).resolves.toMatchObject({ - status: 'worktree_created', + status: 'source_copied', + }); + }); + + it('reconciles direct mechanical lifecycle steps into the prepared Petri journal', async () => { + const cwd = await mkdtemp(join(tmpdir(), 'brunch-drive-direct-lifecycle-journal-')); + await createRunAtCreated(cwd, ['task-1']); + await preparePetriObservation({ cwd, runId: 'run-1' }); + await createWorktree({ cwd, runId: 'run-1', gitWorktree: createFakeGitWorktreePort() }); + await populateWorktree({ cwd, runId: 'run-1' }); + await selectSourcePolicy({ cwd, runId: 'run-1', policy: 'plan_only' }); + await copyHostSource({ cwd, runId: 'run-1' }); + await initializeReports({ cwd, runId: 'run-1' }); + + expect( + (await readPetriEvents(cwd)).flatMap((event) => + event.kind === 'transition_fired' ? [event.transitionId] : [], + ), + ).toEqual(['worktree_create', 'populate', 'source_policy', 'source_copy', 'report_init']); + expect(await readPetriMarkingSnapshot({ cwd, runId: 'run-1' })).toMatchObject({ + currentMarking: { 'slice:task-1:claim': 1 }, + firedTransitionCount: 5, + lifecycleProvenance: { runStatus: 'reports_initialized' }, + }); + + await expect( + drive({ cwd, runId: 'run-1', ports: fakePorts() }, linearScheduler, serialFiringPolicy, { + maxFirings: 1, + }), + ).resolves.toEqual({ status: 'completed', runStatus: 'slice_started' }); + expect( + (await readPetriEvents(cwd)).flatMap((event) => + event.kind === 'transition_fired' ? [event.transitionId] : [], + ), + ).toEqual([ + 'worktree_create', + 'populate', + 'source_policy', + 'source_copy', + 'report_init', + 'slice_start:task-1', + ]); + expect(await readPetriMarkingSnapshot({ cwd, runId: 'run-1' })).toMatchObject({ + currentMarking: { 'slice:task-1:started': 1 }, + firedTransitionCount: 6, + lifecycleProvenance: { runStatus: 'slice_started', activeSliceId: 'task-1' }, }); }); @@ -6227,7 +6347,7 @@ describe('petriScheduler', () => { }); }); - it('refuses standalone start after one parallel claim journals before batch marking', async () => { + it('halts before a parallel claim when equal marking repair cannot persist', async () => { const cwd = await mkdtemp(join(tmpdir(), 'brunch-petri-single-journal-claim-')); await createRunAtCreated(cwd, ['task-1', 'task-2']); await expect( @@ -6241,19 +6361,13 @@ describe('petriScheduler', () => { await expect( drive({ cwd, runId: 'run-1', ports: fakePorts() }, petriScheduler, frontierFiringPolicy), ).resolves.toMatchObject({ status: 'halted', reason: 'petri_marking_persist_failed' }); - await expect(readRunDetail(cwd, 'run-1')).resolves.toMatchObject({ - petriReadySteps: [], - petriBlockedSteps: [ - { kind: 'authority_unreadable', blockers: [{ kind: 'parallel_authority_unreadable' }] }, - { - kind: 'slice_start', - sliceId: 'task-1', - blockers: [{ kind: 'parallel_authority_unreadable' }], - }, - ], - }); + expect( + (await readPetriEvents(cwd)).filter( + (event) => event.kind === 'transition_fired' && event.transitionId.startsWith('slice_start:'), + ), + ).toEqual([]); await expect(startSlice({ cwd, runId: 'run-1', sliceId: 'task-2' })).resolves.toMatchObject({ - status: 'parallel_batch_active', + status: 'petri_marking_persist_failed', sideEffects: [], }); await expect(readRunMetadata(runMetadataPath(cwd, 'run-1'))).resolves.toMatchObject({ @@ -6277,7 +6391,7 @@ describe('petriScheduler', () => { }); await expect(startSlice({ cwd, runId: 'run-1', sliceId: 'task-2' })).resolves.toEqual({ - status: 'parallel_batch_active', + status: 'petri_journal_gap', runStatus: 'slice_completed', runId: 'run-1', metadataPath: runMetadataPath(cwd, 'run-1'), diff --git a/src/executor/__tests__/report.test.ts b/src/executor/__tests__/report.test.ts index 9ec3feef6..0ba6b8f4c 100644 --- a/src/executor/__tests__/report.test.ts +++ b/src/executor/__tests__/report.test.ts @@ -2,12 +2,15 @@ import { access, mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; +import { appendPetriEvent, petriEventsPath } from '../petri-events.js'; +import * as petriLifecycleReconciliation from '../petri-lifecycle-reconciliation.js'; +import { preparePetriObservation } from '../petri.js'; import { planFilePath } from '../plan-file.js'; import { populateWorktree } from '../populate.js'; import { initializeReports, reportsPath } from '../report.js'; -import { runDirPath, runMetadataPath, createRun } from '../run.js'; +import { runDirPath, runMetadataPath, readRunMetadata, createRun } from '../run.js'; import { copyHostSource } from '../source-copy.js'; import { selectSourcePolicy } from '../source-policy.js'; import { createWorktree } from '../worktree.js'; @@ -102,4 +105,64 @@ describe('initializeReports', () => { }); expect(await pathExists(join(runDirPath(cwd, 'run-1'), 'petrinaut'))).toBe(false); }); + + it('reports a blocked prepared-journal reconciliation after persisting report initialization', async () => { + const cwd = await mkdtemp(join(tmpdir(), 'brunch-report-blocked-journal-')); + await createSourceCopiedRun(cwd); + const reconcile = vi + .spyOn(petriLifecycleReconciliation, 'reconcilePreparedLifecycleJournal') + .mockResolvedValueOnce({ status: 'synchronized' }) + .mockResolvedValueOnce({ status: 'blocked', reason: 'petri_input_unreadable' }); + + try { + await expect(initializeReports({ cwd, runId: 'run-1' })).resolves.toEqual({ + status: 'petri_input_unreadable', + runStatus: 'reports_initialized', + runId: 'run-1', + metadataPath: runMetadataPath(cwd, 'run-1'), + reportsPath: reportsPath(cwd, 'run-1'), + sideEffects: [ + { kind: 'write_file', path: reportsPath(cwd, 'run-1'), ifExists: 'overwrite' }, + { kind: 'write_file', path: runMetadataPath(cwd, 'run-1'), ifExists: 'overwrite' }, + ], + }); + } finally { + reconcile.mockRestore(); + } + await expect(readRunMetadata(runMetadataPath(cwd, 'run-1'))).resolves.toMatchObject({ + status: 'reports_initialized', + }); + }); + + it('does not initialize reports after the prepared journal records a terminal', async () => { + const cwd = await mkdtemp(join(tmpdir(), 'brunch-report-terminal-journal-')); + await createSourceCopiedRun(cwd); + await preparePetriObservation({ cwd, runId: 'run-1' }); + await appendPetriEvent({ + cwd, + runId: 'run-1', + event: { + kind: 'net_halted', + runId: 'run-1', + runStatus: 'source_copied', + step: 'report_init', + reason: 'operator_halt', + failedSliceIds: [], + }, + }); + const journalBefore = await readFile(petriEventsPath(cwd, 'run-1'), 'utf8'); + + await expect(initializeReports({ cwd, runId: 'run-1' })).resolves.toEqual({ + status: 'petri_terminal_recorded', + runStatus: 'source_copied', + runId: 'run-1', + metadataPath: runMetadataPath(cwd, 'run-1'), + sideEffects: [], + }); + await expect(readRunMetadata(runMetadataPath(cwd, 'run-1'))).resolves.toMatchObject({ + status: 'source_copied', + }); + await expect(pathExists(reportsPath(cwd, 'run-1'))).resolves.toBe(false); + await expect(readFile(petriEventsPath(cwd, 'run-1'), 'utf8')).resolves.toBe(journalBefore); + }); }); diff --git a/src/executor/__tests__/run-supersession.test.ts b/src/executor/__tests__/run-supersession.test.ts index 59d034714..dc174cdbb 100644 --- a/src/executor/__tests__/run-supersession.test.ts +++ b/src/executor/__tests__/run-supersession.test.ts @@ -5,10 +5,19 @@ import { dirname, join } from 'node:path'; import { describe, expect, it } from 'vitest'; import type { LaunchCurrentProjection } from '../launch.js'; +import { petriEventsPath } from '../petri-events.js'; +import { petriPlanSnapshotPath } from '../petri-plan-snapshot.js'; +import { petriNetPath, petriSdcpnPath } from '../petri.js'; import { planFilePath, planProvenancePath } from '../plan-file.js'; import { withRunExecutionAuthority } from '../run-execution-authority.js'; import { createSupersedingRun } from '../run-supersession.js'; -import { runDirPath, runMetadataPath, type RunMetadata } from '../run.js'; +import { + readRunMetadata, + runDirPath, + runMetadataPath, + subscribeRunMetadata, + type RunMetadata, +} from '../run.js'; const current: LaunchCurrentProjection = { specId: '42', @@ -211,6 +220,11 @@ describe('createSupersedingRun', () => { sideEffects: [ { kind: 'mkdir', path: runDirPath(cwd, 'run-new') }, { kind: 'write_file', path: runMetadataPath(cwd, 'run-new'), ifExists: 'overwrite' }, + { kind: 'mkdir', path: dirname(petriNetPath(cwd, 'run-new')) }, + { kind: 'write_file', path: petriPlanSnapshotPath(cwd, 'run-new') }, + { kind: 'write_file', path: petriNetPath(cwd, 'run-new') }, + { kind: 'write_file', path: petriSdcpnPath(cwd, 'run-new') }, + { kind: 'write_file', path: petriEventsPath(cwd, 'run-new') }, ], }); await expect(readFile(runMetadataPath(cwd, 'run-old'), 'utf8')).resolves.toBe( @@ -219,6 +233,68 @@ describe('createSupersedingRun', () => { await expect(readFile(runMetadataPath(cwd, 'run-new'), 'utf8')).resolves.toContain( '"supersedesRunId": "run-old"', ); + await expect(readRunMetadata(runMetadataPath(cwd, 'run-new'))).resolves.toMatchObject({ + petriObservationPrepared: true, + }); + await expect(pathExists(petriPlanSnapshotPath(cwd, 'run-new'))).resolves.toBe(true); + await expect(pathExists(petriNetPath(cwd, 'run-new'))).resolves.toBe(true); + await expect(pathExists(petriSdcpnPath(cwd, 'run-new'))).resolves.toBe(true); + await expect(readFile(petriEventsPath(cwd, 'run-new'), 'utf8')).resolves.toBe(''); + }); + + it('holds target-run authority through observer preparation', async () => { + const cwd = await mkdtemp(join(tmpdir(), 'brunch-run-supersede-target-authority-')); + await writePlan(cwd); + await writeRun(cwd, 'run-old'); + let initialMetadataPersisted!: () => void; + const initialMetadata = new Promise((resolve) => { + initialMetadataPersisted = resolve; + }); + const unsubscribe = subscribeRunMetadata({ + cwd, + runId: 'run-new', + listener(metadata) { + if (metadata.petriObservationPrepared !== true) initialMetadataPersisted(); + }, + }); + const targetClaim = initialMetadata.then(() => + withRunExecutionAuthority({ + cwd, + runId: 'run-new', + execute: async () => 'acquired' as const, + onContended: () => 'contended' as const, + }), + ); + + try { + const [result, claim] = await Promise.all([ + createSupersedingRun({ + cwd, + previousRunId: 'run-old', + runId: 'run-new', + current, + }), + targetClaim, + ]); + + expect(result.status).toBe('created'); + expect(claim).toBe('contended'); + } finally { + unsubscribe(); + } + }); + + it('removes an unpublished replacement run when observer preparation fails', async () => { + const cwd = await mkdtemp(join(tmpdir(), 'brunch-run-supersede-observer-failure-')); + await writePlan(cwd); + await writeRun(cwd, 'run-old'); + await writeFile(planFilePath(cwd, '42'), '{', 'utf8'); + + await expect( + createSupersedingRun({ cwd, previousRunId: 'run-old', runId: 'run-new', current }), + ).rejects.toThrow(); + await expect(pathExists(runDirPath(cwd, 'run-new'))).resolves.toBe(false); + await expect(pathExists(runMetadataPath(cwd, 'run-old'))).resolves.toBe(true); }); it('preserves the previous run environment policy on the superseding run', async () => { diff --git a/src/executor/__tests__/slice-start.test.ts b/src/executor/__tests__/slice-start.test.ts index fa2f801bf..9c2511101 100644 --- a/src/executor/__tests__/slice-start.test.ts +++ b/src/executor/__tests__/slice-start.test.ts @@ -2,14 +2,26 @@ import { access, mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { describe, expect, it } from 'vitest'; - -import { writePetriMarkingSnapshot } from '../petri-marking.js'; +import { describe, expect, it, vi } from 'vitest'; + +import * as petriEvents from '../petri-events.js'; +import * as petriLifecycleReconciliation from '../petri-lifecycle-reconciliation.js'; +import * as petriMarking from '../petri-marking.js'; +import { readPetriRuntimePlan } from '../petri-runtime-plan.js'; +import { projectExecutorPetriTransitionHistory } from '../petri-runtime.js'; +import { preparePetriObservation } from '../petri.js'; import { planFilePath } from '../plan-file.js'; import { populateWorktree } from '../populate.js'; import { initializeReports, reportsPath } from '../report.js'; import { withRunExecutionAuthority } from '../run-execution-authority.js'; -import { runDirPath, runMetadataPath, persistRunMetadata, readRunMetadata, createRun } from '../run.js'; +import { + runDirPath, + runMetadataPath, + persistRunMetadata, + readRunMetadata, + createRun, + type RunMetadata, +} from '../run.js'; import { startSlice, startSliceWithExecutionAuthority } from '../slice-start.js'; import { copyHostSource } from '../source-copy.js'; import { selectSourcePolicy } from '../source-policy.js'; @@ -25,7 +37,7 @@ async function pathExists(path: string): Promise { } } -async function createReportReadyRun(cwd: string): Promise { +async function createReportReadyRun(cwd: string, prepareObservation = false): Promise { const planPath = planFilePath(cwd, '42'); await mkdir(join(cwd, 'src'), { recursive: true }); await writeFile(join(cwd, 'src', 'app.ts'), 'export const app = true;\n', 'utf8'); @@ -49,6 +61,7 @@ async function createReportReadyRun(cwd: string): Promise { 'utf8', ); await createRun({ cwd, specId: '42', runId: 'run-1' }); + if (prepareObservation) await preparePetriObservation({ cwd, runId: 'run-1' }); await createWorktree({ cwd, runId: 'run-1', gitWorktree: createFakeGitWorktreePort() }); await populateWorktree({ cwd, runId: 'run-1' }); await selectSourcePolicy({ cwd, runId: 'run-1', policy: 'host_source_deferred' }); @@ -56,6 +69,23 @@ async function createReportReadyRun(cwd: string): Promise { await initializeReports({ cwd, runId: 'run-1' }); } +async function advancePreparedRunToSliceStarted(cwd: string) { + await createReportReadyRun(cwd, true); + const current = await readRunMetadata(runMetadataPath(cwd, 'run-1')); + if (!current) throw new Error('expected prepared run metadata'); + const state: RunMetadata = { + ...current, + status: 'slice_started', + activeSliceId: 'task-1', + activeEpicId: 'frontier-1', + }; + await persistRunMetadata(runMetadataPath(cwd, 'run-1'), state); + const plan = await readPetriRuntimePlan(cwd, state); + const lifecycleTransitionIds = projectExecutorPetriTransitionHistory(state, plan)?.transitionIds; + if (!plan || !lifecycleTransitionIds) throw new Error('expected prepared lifecycle projection'); + return { state, plan, lifecycleTransitionIds }; +} + async function createTwoSliceReportReadyRun(cwd: string): Promise { const planPath = planFilePath(cwd, '42'); await mkdir(join(cwd, 'src'), { recursive: true }); @@ -193,6 +223,174 @@ describe('startSlice', () => { expect(await pathExists(join(runDirPath(cwd, 'run-1'), 'agent-output'))).toBe(false); }); + it('starts directly from a reconciled prepared journal instead of reporting a parallel batch', async () => { + const cwd = await mkdtemp(join(tmpdir(), 'brunch-slice-start-prepared-journal-')); + await createReportReadyRun(cwd, true); + + await expect(startSliceWithExecutionAuthority({ cwd, runId: 'run-1' })).resolves.toMatchObject({ + status: 'slice_started', + runStatus: 'slice_started', + sliceId: 'task-1', + }); + const transitionIds = (await readFile(petriEvents.petriEventsPath(cwd, 'run-1'), 'utf8')) + .trim() + .split('\n') + .map((line) => JSON.parse(line)) + .flatMap((event) => (event.kind === 'transition_fired' ? [event.transitionId] : [])); + expect(transitionIds).toEqual([ + 'worktree_create', + 'populate', + 'source_policy', + 'source_copy', + 'report_init', + 'slice_start:task-1', + ]); + }); + + it('reports a blocked reconciliation after persisting the slice-start transition', async () => { + const cwd = await mkdtemp(join(tmpdir(), 'brunch-slice-start-post-reconciliation-block-')); + await createReportReadyRun(cwd); + const reconcile = vi + .spyOn(petriLifecycleReconciliation, 'reconcilePreparedLifecycleJournal') + .mockResolvedValueOnce({ status: 'synchronized' }) + .mockResolvedValueOnce({ status: 'blocked', reason: 'petri_input_unreadable' }); + + try { + await expect(startSliceWithExecutionAuthority({ cwd, runId: 'run-1' })).resolves.toEqual({ + status: 'petri_input_unreadable', + runStatus: 'slice_started', + runId: 'run-1', + sliceId: 'task-1', + epicId: 'frontier-1', + metadataPath: runMetadataPath(cwd, 'run-1'), + reportsPath: reportsPath(cwd, 'run-1'), + sideEffects: [ + { kind: 'append_file', path: reportsPath(cwd, 'run-1') }, + { kind: 'write_file', path: runMetadataPath(cwd, 'run-1'), ifExists: 'overwrite' }, + ], + }); + } finally { + reconcile.mockRestore(); + } + await expect(readRunMetadata(runMetadataPath(cwd, 'run-1'))).resolves.toMatchObject({ + status: 'slice_started', + activeSliceId: 'task-1', + }); + }); + + it('returns a blocked result when lifecycle journal catch-up cannot append', async () => { + const cwd = await mkdtemp(join(tmpdir(), 'brunch-lifecycle-reconcile-append-failure-')); + const { state, plan, lifecycleTransitionIds } = await advancePreparedRunToSliceStarted(cwd); + const append = vi.spyOn(petriEvents, 'appendPetriEvent').mockRejectedValueOnce(new Error('disk full')); + + try { + await expect( + petriLifecycleReconciliation.reconcilePreparedLifecycleJournal({ + cwd, + runId: 'run-1', + state, + lifecycleTransitionIds, + plan, + }), + ).resolves.toEqual({ + status: 'blocked', + reason: 'petri_journal_append_failed', + }); + } finally { + append.mockRestore(); + } + }); + + it('repairs an equal journal after a marking write failure', async () => { + const cwd = await mkdtemp(join(tmpdir(), 'brunch-lifecycle-reconcile-marking-repair-')); + const { state, plan, lifecycleTransitionIds } = await advancePreparedRunToSliceStarted(cwd); + const writeMarking = vi + .spyOn(petriMarking, 'writePetriMarkingSnapshot') + .mockRejectedValueOnce(new Error('disk full')); + + try { + await expect( + petriLifecycleReconciliation.reconcilePreparedLifecycleJournal({ + cwd, + runId: 'run-1', + state, + lifecycleTransitionIds, + plan, + }), + ).resolves.toEqual({ + status: 'blocked', + reason: 'petri_marking_persist_failed', + }); + } finally { + writeMarking.mockRestore(); + } + + await expect(petriMarking.readPetriMarkingSnapshot({ cwd, runId: 'run-1' })).resolves.toMatchObject({ + lifecycleProvenance: { runStatus: 'reports_initialized' }, + }); + await expect( + petriLifecycleReconciliation.reconcilePreparedLifecycleJournal({ + cwd, + runId: 'run-1', + state, + lifecycleTransitionIds, + plan, + }), + ).resolves.toEqual({ status: 'synchronized' }); + await expect(petriMarking.readPetriMarkingSnapshot({ cwd, runId: 'run-1' })).resolves.toMatchObject({ + lifecycleProvenance: { + runStatus: 'slice_started', + activeSliceId: 'task-1', + }, + }); + }); + + it('reports unreadable Petri input instead of inventing parallel authority', async () => { + const cwd = await mkdtemp(join(tmpdir(), 'brunch-slice-start-unreadable-journal-')); + await createReportReadyRun(cwd, true); + await writeFile(petriEvents.petriEventsPath(cwd, 'run-1'), '{', 'utf8'); + + await expect(startSliceWithExecutionAuthority({ cwd, runId: 'run-1' })).resolves.toEqual({ + status: 'petri_input_unreadable', + runStatus: 'reports_initialized', + runId: 'run-1', + metadataPath: runMetadataPath(cwd, 'run-1'), + sideEffects: [], + }); + }); + + it('does not start a slice after the prepared journal records a terminal', async () => { + const cwd = await mkdtemp(join(tmpdir(), 'brunch-slice-start-terminal-journal-')); + await createReportReadyRun(cwd, true); + await petriEvents.appendPetriEvent({ + cwd, + runId: 'run-1', + event: { + kind: 'net_halted', + runId: 'run-1', + runStatus: 'reports_initialized', + step: 'slice_start', + reason: 'operator_halt', + failedSliceIds: [], + }, + }); + const journalBefore = await readFile(petriEvents.petriEventsPath(cwd, 'run-1'), 'utf8'); + const reportsBefore = await readFile(reportsPath(cwd, 'run-1'), 'utf8'); + + await expect(startSliceWithExecutionAuthority({ cwd, runId: 'run-1' })).resolves.toEqual({ + status: 'petri_terminal_recorded', + runStatus: 'reports_initialized', + runId: 'run-1', + metadataPath: runMetadataPath(cwd, 'run-1'), + sideEffects: [], + }); + const metadata = await readRunMetadata(runMetadataPath(cwd, 'run-1')); + expect(metadata).toMatchObject({ status: 'reports_initialized' }); + expect(metadata?.activeSliceId).toBeUndefined(); + await expect(readFile(reportsPath(cwd, 'run-1'), 'utf8')).resolves.toBe(reportsBefore); + await expect(readFile(petriEvents.petriEventsPath(cwd, 'run-1'), 'utf8')).resolves.toBe(journalBefore); + }); + it('starts the next incomplete slice after a previous slice has completed', async () => { const cwd = await mkdtemp(join(tmpdir(), 'brunch-slice-start-next-')); await createTwoSliceReportReadyRun(cwd); @@ -305,7 +503,7 @@ describe('startSlice', () => { it('refuses a standalone start when durable parallel batch authority is active', async () => { const cwd = await mkdtemp(join(tmpdir(), 'brunch-slice-start-parallel-authority-')); await createTwoSliceReportReadyRun(cwd); - await writePetriMarkingSnapshot({ + await petriMarking.writePetriMarkingSnapshot({ cwd, runId: 'run-1', snapshot: { diff --git a/src/executor/orchestrate.ts b/src/executor/orchestrate.ts index 5f73f493c..228a83d9c 100644 --- a/src/executor/orchestrate.ts +++ b/src/executor/orchestrate.ts @@ -26,6 +26,7 @@ import { inspectPetriJournalAuthority, type PetriJournalAuthorityInspection, } from './petri-journal-authority.js'; +import { reconcilePreparedLifecycleJournal } from './petri-lifecycle-reconciliation.js'; import { petriMarkingLifecycleProvenance, petriMarkingSnapshotMatchesRunMetadata, @@ -465,6 +466,25 @@ async function driveOwned( return (await settleDriveTerminal(ctx, terminal)).outcome; } } + const lifecycleReconciliation = await reconcilePreparedLifecycleJournal({ + cwd: ctx.cwd, + runId: ctx.runId, + state, + lifecycleTransitionIds: projectExecutorPetriTransitionHistory(state, plan)?.transitionIds, + plan, + }); + if ( + lifecycleReconciliation.status === 'blocked' && + (lifecycleReconciliation.reason === 'petri_journal_append_failed' || + lifecycleReconciliation.reason === 'petri_marking_persist_failed') + ) { + return { + status: 'halted', + step: reconciliationStep(state, plan), + runStatus: state.status, + reason: lifecycleReconciliation.reason, + }; + } const authoritySnapshot = await readPetriMarkingSnapshot({ cwd: ctx.cwd, runId: ctx.runId }); const journal = await inspectPetriJournalAuthority({ cwd: ctx.cwd, @@ -805,6 +825,17 @@ async function driveOwned( } return settled.outcome; } + if ( + result.status === 'petri_journal_append_failed' || + result.status === 'petri_marking_persist_failed' + ) { + return { + status: 'halted', + step: next.kind, + runStatus: result.runStatus === 'not_started' ? currentState.status : result.runStatus, + reason: result.status, + }; + } if (result.skipTransition && result.runStatus !== 'not_started') { try { ctx.onStepComplete?.( @@ -864,6 +895,9 @@ async function driveOwned( } if (attemptOutcome.status === 'retry') continue; } + if (result.status === 'petri_input_unreadable' || result.status === 'petri_journal_gap') { + publishPetriJournalFailure(ctx); + } const terminal = classifyDriveTerminal({ kind: 'step_halted', runId: ctx.runId, @@ -900,29 +934,63 @@ async function driveOwned( const transition = boundTransition?.transition; if (transition) { firedTransitions += 1; - const emitted = await emitNetEvent(ctx, { - kind: 'transition_fired', - runId: ctx.runId, - runStatus: result.runStatus, - transitionId: transition.id, - subnetId: transition.subnetId, - ...(transition.epicId === undefined ? {} : { epicId: transition.epicId }), - ...(transition.derivedFrom === undefined ? {} : { derivedFrom: transition.derivedFrom }), - step: next.kind, - contract: transition.contract, - consumed: transition.inputArcs.map((arc) => arc.placeId), - produced: transition.outputArcs.map((arc) => arc.placeId), - fromStatus: currentState.status, - toStatus: result.runStatus, - ...(currentState.activeSliceAttempts ? { attempt: currentState.activeSliceAttempts + 1 } : {}), - }); - if (!emitted.journaled) { - return { - status: 'halted', - step: next.kind, + const transitionedState = await readRunMetadata(metadataPath); + const transitionReconcilesInHandler = + transition.id === 'report_init' || transition.id.startsWith('slice_start:'); + if (transitionReconcilesInHandler) { + const reconciliation = transitionedState + ? await reconcilePreparedLifecycleJournal({ + cwd: ctx.cwd, + runId: ctx.runId, + state: transitionedState, + lifecycleTransitionIds: projectExecutorPetriTransitionHistory( + transitionedState, + currentPlan, + )?.transitionIds, + plan: currentPlan, + }) + : { status: 'blocked' as const, reason: 'petri_input_unreadable' as const }; + if (reconciliation.status !== 'synchronized') { + if ( + reconciliation.status === 'blocked' && + (reconciliation.reason === 'petri_input_unreadable' || + reconciliation.reason === 'petri_journal_gap') + ) { + publishPetriJournalFailure(ctx); + } + return { + status: 'halted', + step: next.kind, + runStatus: result.runStatus, + reason: + reconciliation.status === 'blocked' ? reconciliation.reason : 'petri_input_unreadable', + }; + } + } else { + const emitted = await emitNetEvent(ctx, { + kind: 'transition_fired', + runId: ctx.runId, runStatus: result.runStatus, - reason: 'petri_journal_append_failed', - }; + transitionId: transition.id, + subnetId: transition.subnetId, + ...(transition.epicId === undefined ? {} : { epicId: transition.epicId }), + ...(transition.derivedFrom === undefined ? {} : { derivedFrom: transition.derivedFrom }), + step: next.kind, + contract: transition.contract, + consumed: transition.inputArcs.map((arc) => arc.placeId), + produced: transition.outputArcs.map((arc) => arc.placeId), + fromStatus: currentState.status, + toStatus: result.runStatus, + ...(currentState.activeSliceAttempts ? { attempt: currentState.activeSliceAttempts + 1 } : {}), + }); + if (!emitted.journaled) { + return { + status: 'halted', + step: next.kind, + runStatus: result.runStatus, + reason: 'petri_journal_append_failed', + }; + } } if (result.epicVerificationPassed) { const transitioned = replayTransitionHistory( diff --git a/src/executor/petri-lifecycle-reconciliation.ts b/src/executor/petri-lifecycle-reconciliation.ts new file mode 100644 index 000000000..fecf55266 --- /dev/null +++ b/src/executor/petri-lifecycle-reconciliation.ts @@ -0,0 +1,180 @@ +import { compileExecutorTopology, type SchedulerPlan } from './orchestrate-topology.js'; +import { appendPetriEvent } from './petri-events.js'; +import { inspectPetriJournalAuthority } from './petri-journal-authority.js'; +import { + petriMarkingLifecycleProvenance, + petriMarkingSnapshotMatchesRunMetadata, + readPetriMarkingSnapshot, + writePetriMarkingSnapshot, +} from './petri-marking.js'; +import { replayTransitionHistory } from './petri-replay.js'; +import { readPetriRuntimePlan } from './petri-runtime-plan.js'; +import type { RunMetadata } from './run.js'; + +export type PetriLifecycleReconciliationBlockReason = + | 'parallel_batch_active' + | 'petri_input_unreadable' + | 'petri_journal_append_failed' + | 'petri_journal_gap' + | 'petri_marking_persist_failed' + | 'petri_terminal_recorded'; + +export type PetriLifecycleReconciliation = + | { readonly status: 'not_prepared' } + | { readonly status: 'synchronized' } + | { + readonly status: 'blocked'; + readonly reason: PetriLifecycleReconciliationBlockReason; + }; + +export async function reconcilePreparedLifecycleJournal(args: { + readonly cwd: string; + readonly runId: string; + readonly state: RunMetadata; + readonly lifecycleTransitionIds: readonly string[] | undefined; + readonly plan?: SchedulerPlan | undefined; +}): Promise { + const existing = await readPetriMarkingSnapshot({ cwd: args.cwd, runId: args.runId }); + if (existing?.parallelSliceBatch) { + return { status: 'blocked', reason: 'parallel_batch_active' }; + } + if (args.state.petriObservationPrepared !== true) return { status: 'not_prepared' }; + const plan = args.plan ?? (await readPetriRuntimePlan(args.cwd, args.state)); + if (!args.lifecycleTransitionIds) { + return { status: 'blocked', reason: 'petri_input_unreadable' }; + } + const authority = await inspectPetriJournalAuthority({ + cwd: args.cwd, + runId: args.runId, + lifecycleTransitionIds: args.lifecycleTransitionIds, + plan, + }); + if (authority.status !== 'readable') { + return { status: 'blocked', reason: 'petri_input_unreadable' }; + } + if ( + authority.events.some( + (event) => + event.kind === 'net_completed' || event.kind === 'net_halted' || event.kind === 'net_deadlocked', + ) + ) { + return { status: 'blocked', reason: 'petri_terminal_recorded' }; + } + if ( + authority.relation === 'equal' && + existing && + petriMarkingSnapshotMatchesRunMetadata(existing, args.state) + ) { + return { status: 'synchronized' }; + } + if (authority.relation === 'journal_ahead') { + const parallelClaimOnly = + authority.residualTransitionIds.length > 0 && + authority.residualTransitionIds.every((transitionId) => transitionId.startsWith('slice_start:')); + return { + status: 'blocked', + reason: parallelClaimOnly ? 'parallel_batch_active' : 'petri_input_unreadable', + }; + } + + const topology = compileExecutorTopology(plan); + if (authority.relation === 'lifecycle_ahead') { + const missingTransitionIds = authority.residualTransitionIds; + if (!isSuffix(missingTransitionIds, args.lifecycleTransitionIds)) { + return { status: 'blocked', reason: 'petri_input_unreadable' }; + } + if (missingTransitionIds.some((id) => !isRecoverableLifecycleTransition(id))) { + return { status: 'blocked', reason: 'petri_journal_gap' }; + } + for (const transitionId of missingTransitionIds) { + const transition = topology.transitions.find((candidate) => candidate.id === transitionId); + const statuses = lifecycleStatuses(transitionId, args.state); + if (!transition || !statuses) { + return { status: 'blocked', reason: 'petri_journal_gap' }; + } + try { + await appendPetriEvent({ + cwd: args.cwd, + runId: args.runId, + event: { + kind: 'transition_fired', + runId: args.runId, + runStatus: statuses.toStatus, + transitionId, + subnetId: transition.subnetId, + ...(transition.epicId === undefined ? {} : { epicId: transition.epicId }), + ...(transition.derivedFrom === undefined ? {} : { derivedFrom: transition.derivedFrom }), + step: transition.step!.kind, + contract: transition.contract, + consumed: transition.inputArcs.map((arc) => arc.placeId), + produced: transition.outputArcs.map((arc) => arc.placeId), + fromStatus: statuses.fromStatus, + toStatus: statuses.toStatus, + }, + }); + } catch { + return { status: 'blocked', reason: 'petri_journal_append_failed' }; + } + } + } + + const replayed = replayTransitionHistory(topology, args.lifecycleTransitionIds); + if (!replayed) return { status: 'blocked', reason: 'petri_input_unreadable' }; + try { + await writePetriMarkingSnapshot({ + cwd: args.cwd, + runId: args.runId, + snapshot: { + ...replayed, + lifecycleProvenance: petriMarkingLifecycleProvenance(args.state), + ...(existing?.epicVerificationClaims + ? { epicVerificationClaims: existing.epicVerificationClaims } + : {}), + }, + }); + } catch { + return { status: 'blocked', reason: 'petri_marking_persist_failed' }; + } + return { status: 'synchronized' }; +} + +function isSuffix(suffix: readonly string[], value: readonly string[]): boolean { + const offset = value.length - suffix.length; + return offset >= 0 && suffix.every((entry, index) => value[offset + index] === entry); +} + +function isRecoverableLifecycleTransition(transitionId: string): boolean { + return ( + transitionId === 'worktree_create' || + transitionId === 'populate' || + transitionId === 'source_policy' || + transitionId === 'source_copy' || + transitionId === 'report_init' || + transitionId.startsWith('slice_start:') + ); +} + +function lifecycleStatuses( + transitionId: string, + state: RunMetadata, +): { readonly fromStatus: RunMetadata['status']; readonly toStatus: RunMetadata['status'] } | undefined { + const fixed = RUN_LIFECYCLE_STATUSES[transitionId]; + if (fixed) return fixed; + if (transitionId.startsWith('slice_start:')) { + return { + fromStatus: (state.completedSliceIds?.length ?? 0) > 0 ? 'slice_completed' : 'reports_initialized', + toStatus: 'slice_started', + }; + } + return undefined; +} + +const RUN_LIFECYCLE_STATUSES: Readonly< + Record +> = { + worktree_create: { fromStatus: 'created', toStatus: 'worktree_created' }, + populate: { fromStatus: 'worktree_created', toStatus: 'worktree_populated' }, + source_policy: { fromStatus: 'worktree_populated', toStatus: 'source_policy_selected' }, + source_copy: { fromStatus: 'source_policy_selected', toStatus: 'source_copied' }, + report_init: { fromStatus: 'source_copied', toStatus: 'reports_initialized' }, +}; diff --git a/src/executor/report.ts b/src/executor/report.ts index 5f14e20a8..2317103fb 100644 --- a/src/executor/report.ts +++ b/src/executor/report.ts @@ -1,6 +1,10 @@ import { writeFile } from 'node:fs/promises'; import { join } from 'node:path'; +import { + reconcilePreparedLifecycleJournal, + type PetriLifecycleReconciliationBlockReason, +} from './petri-lifecycle-reconciliation.js'; import { runExecutionActive, withRunExecutionAuthority, @@ -24,6 +28,24 @@ export type ReportInitResult = readonly metadataPath: string; readonly sideEffects: readonly []; } + | { + readonly status: PetriLifecycleReconciliationBlockReason; + readonly runStatus: 'source_copied'; + readonly runId: string; + readonly metadataPath: string; + readonly sideEffects: readonly []; + } + | { + readonly status: PetriLifecycleReconciliationBlockReason; + readonly runStatus: 'reports_initialized'; + readonly runId: string; + readonly metadataPath: string; + readonly reportsPath: string; + readonly sideEffects: readonly [ + { readonly kind: 'write_file'; readonly path: string; readonly ifExists: 'overwrite' }, + { readonly kind: 'write_file'; readonly path: string; readonly ifExists: 'overwrite' }, + ]; + } | { readonly status: 'reports_initialized'; readonly runStatus: 'reports_initialized'; @@ -78,12 +100,45 @@ async function initializeReportsOwned(args: { }; } + const sourceReconciliation = await reconcilePreparedLifecycleJournal({ + cwd: args.cwd, + runId: args.runId, + state: metadata, + lifecycleTransitionIds: ['worktree_create', 'populate', 'source_policy', 'source_copy'], + }); + if (sourceReconciliation.status === 'blocked') { + return { + status: sourceReconciliation.reason, + runStatus: 'source_copied', + runId: args.runId, + metadataPath, + sideEffects: [], + }; + } + const path = reportsPath(args.cwd, args.runId); const updated: RunMetadata = { ...metadata, status: 'reports_initialized', reportsPath: path }; const event = { event: 'run_ready', runId: args.runId, status: 'reports_initialized' }; await writeFile(path, `${JSON.stringify(event)}\n`, 'utf8'); const metadataEffect = await persistRunMetadata(metadataPath, updated); + const sideEffects = [{ kind: 'write_file', path, ifExists: 'overwrite' }, metadataEffect] as const; + const reconciliation = await reconcilePreparedLifecycleJournal({ + cwd: args.cwd, + runId: args.runId, + state: updated, + lifecycleTransitionIds: ['worktree_create', 'populate', 'source_policy', 'source_copy', 'report_init'], + }); + if (reconciliation.status === 'blocked') { + return { + status: reconciliation.reason, + runStatus: 'reports_initialized', + runId: args.runId, + metadataPath, + reportsPath: path, + sideEffects, + }; + } return { status: 'reports_initialized', @@ -91,6 +146,6 @@ async function initializeReportsOwned(args: { runId: args.runId, metadataPath, reportsPath: path, - sideEffects: [{ kind: 'write_file', path, ifExists: 'overwrite' }, metadataEffect], + sideEffects, }; } diff --git a/src/executor/run-supersession.ts b/src/executor/run-supersession.ts index 8c9c2b6a2..5b9d65b74 100644 --- a/src/executor/run-supersession.ts +++ b/src/executor/run-supersession.ts @@ -1,6 +1,10 @@ -import { access, mkdir } from 'node:fs/promises'; +import { access, mkdir, rm } from 'node:fs/promises'; +import { dirname } from 'node:path'; import { prepareLaunch, type LaunchCurrentProjection, type LaunchResult } from './launch.js'; +import { petriEventsPath } from './petri-events.js'; +import { petriPlanSnapshotPath } from './petri-plan-snapshot.js'; +import { petriNetPath, petriSdcpnPath, preparePetriObservation } from './petri.js'; import { runExecutionActive, withRunExecutionAuthority, @@ -41,10 +45,10 @@ export type RunSupersessionResult = readonly runDir: string; readonly metadataPath: string; readonly planPath: string; - readonly sideEffects: readonly [ - { readonly kind: 'mkdir'; readonly path: string }, - { readonly kind: 'write_file'; readonly path: string; readonly ifExists: 'overwrite' }, - ]; + readonly sideEffects: readonly ( + | { readonly kind: 'mkdir'; readonly path: string } + | { readonly kind: 'write_file'; readonly path: string; readonly ifExists?: 'overwrite' } + )[]; }; export async function createSupersedingRun(args: { @@ -126,6 +130,12 @@ async function createSupersedingRunOwned(args: { if (await pathExists(runDir)) return { status: 'exists' as const }; await mkdir(runDir, { recursive: true }); const metadataEffect = await persistRunMetadata(metadataPath, metadata); + try { + await preparePetriObservation({ cwd: args.cwd, runId }); + } catch (error) { + await rm(runDir, { recursive: true, force: true }); + throw error; + } return { status: 'created' as const, metadataEffect }; }, onContended: () => ({ status: 'active' as const }), @@ -141,6 +151,7 @@ async function createSupersedingRunOwned(args: { sideEffects: [], }; } + const petriDir = dirname(petriNetPath(args.cwd, runId)); return { status: 'created', @@ -150,7 +161,15 @@ async function createSupersedingRunOwned(args: { runDir, metadataPath, planPath: launch.planPath, - sideEffects: [{ kind: 'mkdir', path: runDir }, created.metadataEffect], + sideEffects: [ + { kind: 'mkdir', path: runDir }, + created.metadataEffect, + { kind: 'mkdir', path: petriDir }, + { kind: 'write_file', path: petriPlanSnapshotPath(args.cwd, runId) }, + { kind: 'write_file', path: petriNetPath(args.cwd, runId) }, + { kind: 'write_file', path: petriSdcpnPath(args.cwd, runId) }, + { kind: 'write_file', path: petriEventsPath(args.cwd, runId) }, + ], }; } diff --git a/src/executor/slice-start.ts b/src/executor/slice-start.ts index 016bea5f8..717a715c8 100644 --- a/src/executor/slice-start.ts +++ b/src/executor/slice-start.ts @@ -7,8 +7,10 @@ import { type BlockedStep, type SchedulerPlan, } from './orchestrate-topology.js'; -import { inspectPetriJournalAuthority } from './petri-journal-authority.js'; -import { readPetriMarkingSnapshot } from './petri-marking.js'; +import { + reconcilePreparedLifecycleJournal, + type PetriLifecycleReconciliationBlockReason, +} from './petri-lifecycle-reconciliation.js'; import { projectExecutorPetriTransitionHistory } from './petri-runtime.js'; import { populatedPlanPath } from './populate.js'; import { reportsPath } from './report.js'; @@ -32,7 +34,7 @@ export type SliceStartResult = readonly sideEffects: readonly []; } | { - readonly status: 'run_execution_active' | 'parallel_batch_active'; + readonly status: 'run_execution_active' | PetriLifecycleReconciliationBlockReason; readonly runStatus: RunMetadata['status'] | 'not_started'; readonly runId: string; readonly metadataPath: string; @@ -54,6 +56,19 @@ export type SliceStartResult = readonly blockedSteps: readonly BlockedStep[]; readonly sideEffects: readonly []; } + | { + readonly status: PetriLifecycleReconciliationBlockReason; + readonly runStatus: 'slice_started'; + readonly runId: string; + readonly sliceId: string; + readonly epicId?: string; + readonly metadataPath: string; + readonly reportsPath: string; + readonly sideEffects: readonly [ + { readonly kind: 'append_file'; readonly path: string }, + { readonly kind: 'write_file'; readonly path: string; readonly ifExists: 'overwrite' }, + ]; + } | { readonly status: 'slice_started'; readonly runStatus: 'slice_started'; @@ -127,31 +142,17 @@ async function startSliceOwned(args: { }; } - const marking = await readPetriMarkingSnapshot({ cwd: args.cwd, runId: args.runId }); - if (marking?.parallelSliceBatch) { - return { - status: 'parallel_batch_active', - runStatus: metadata.status, - runId: args.runId, - metadataPath, - sideEffects: [], - }; - } - const plan = await readPlan(metadata.populatedPlanPath ?? populatedPlanPath(args.cwd, args.runId)); - const journalAuthority = await inspectPetriJournalAuthority({ + const reconciliation = await reconcilePreparedLifecycleJournal({ cwd: args.cwd, runId: args.runId, + state: metadata, lifecycleTransitionIds: projectExecutorPetriTransitionHistory(metadata, plan)?.transitionIds, plan, }); - if ( - journalAuthority.status === 'unreadable' || - (journalAuthority.status === 'missing' && metadata.petriObservationPrepared === true) || - (journalAuthority.status === 'readable' && journalAuthority.relation !== 'equal') - ) { + if (reconciliation.status === 'blocked') { return { - status: 'parallel_batch_active', + status: reconciliation.reason, runStatus: metadata.status, runId: args.runId, metadataPath, @@ -201,6 +202,26 @@ async function startSliceOwned(args: { await appendFile(reportPath, `${JSON.stringify(event)}\n`, 'utf8'); const metadataEffect = await persistRunMetadata(metadataPath, updated); + const sideEffects = [{ kind: 'append_file', path: reportPath }, metadataEffect] as const; + const transitionedReconciliation = await reconcilePreparedLifecycleJournal({ + cwd: args.cwd, + runId: args.runId, + state: updated, + lifecycleTransitionIds: projectExecutorPetriTransitionHistory(updated, plan)?.transitionIds, + plan, + }); + if (transitionedReconciliation.status === 'blocked') { + return { + status: transitionedReconciliation.reason, + runStatus: 'slice_started', + runId: args.runId, + sliceId: slice.id, + ...(slice.epic_id === undefined ? {} : { epicId: slice.epic_id }), + metadataPath, + reportsPath: reportPath, + sideEffects, + }; + } return { status: 'slice_started', @@ -210,7 +231,7 @@ async function startSliceOwned(args: { ...(slice.epic_id === undefined ? {} : { epicId: slice.epic_id }), metadataPath, reportsPath: reportPath, - sideEffects: [{ kind: 'append_file', path: reportPath }, metadataEffect], + sideEffects, }; }